drivers/nvme/host/core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVM Express device driver
   4  * Copyright (c) 2011-2014, Intel Corporation.
   5  */
   6
   7 #include <linux/blkdev.h>
   8 #include <linux/blk-mq.h>
   9 #include <linux/blk-integrity.h>
  10 #include <linux/compat.h>
  11 #include <linux/delay.h>
  12 #include <linux/errno.h>
  13 #include <linux/hdreg.h>
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/backing-dev.h>
  17 #include <linux/slab.h>
  18 #include <linux/types.h>
  19 #include <linux/pr.h>
  20 #include <linux/ptrace.h>
  21 #include <linux/nvme_ioctl.h>
  22 #include <linux/pm_qos.h>
  23 #include <asm/unaligned.h>
  24
  25 #include "nvme.h"
  26 #include "fabrics.h"
  27 #include <linux/nvme-auth.h>
  28
  29 #define CREATE_TRACE_POINTS
  30 #include "trace.h"
  31
  32 #define NVME_MINORS             (1U << MINORBITS)
  33
  34 struct nvme_ns_info {
  35         struct nvme_ns_ids ids;
  36         u32 nsid;
  37         __le32 anagrpid;
  38         bool is_shared;
  39         bool is_readonly;
  40         bool is_ready;
  41         bool is_removed;
  42 };
  43
  44 unsigned int admin_timeout = 60;
  45 module_param(admin_timeout, uint, 0644);
  46 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
  47 EXPORT_SYMBOL_GPL(admin_timeout);
  48
  49 unsigned int nvme_io_timeout = 30;
  50 module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
  51 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
  52 EXPORT_SYMBOL_GPL(nvme_io_timeout);
  53
  54 static unsigned char shutdown_timeout = 5;
  55 module_param(shutdown_timeout, byte, 0644);
  56 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
  57
  58 static u8 nvme_max_retries = 5;
  59 module_param_named(max_retries, nvme_max_retries, byte, 0644);
  60 MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
  61
  62 static unsigned long default_ps_max_latency_us = 100000;
  63 module_param(default_ps_max_latency_us, ulong, 0644);
  64 MODULE_PARM_DESC(default_ps_max_latency_us,
  65                  "max power saving latency for new devices; use PM QOS to change per device");
  66
  67 static bool force_apst;
  68 module_param(force_apst, bool, 0644);
  69 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
  70
  71 static unsigned long apst_primary_timeout_ms = 100;
  72 module_param(apst_primary_timeout_ms, ulong, 0644);
  73 MODULE_PARM_DESC(apst_primary_timeout_ms,
  74         "primary APST timeout in ms");
  75
  76 static unsigned long apst_secondary_timeout_ms = 2000;
  77 module_param(apst_secondary_timeout_ms, ulong, 0644);
  78 MODULE_PARM_DESC(apst_secondary_timeout_ms,
  79         "secondary APST timeout in ms");
  80
  81 static unsigned long apst_primary_latency_tol_us = 15000;
  82 module_param(apst_primary_latency_tol_us, ulong, 0644);
  83 MODULE_PARM_DESC(apst_primary_latency_tol_us,
  84         "primary APST latency tolerance in us");
  85
  86 static unsigned long apst_secondary_latency_tol_us = 100000;
  87 module_param(apst_secondary_latency_tol_us, ulong, 0644);
  88 MODULE_PARM_DESC(apst_secondary_latency_tol_us,
  89         "secondary APST latency tolerance in us");
  90
  91 /*
  92  * nvme_wq - hosts nvme related works that are not reset or delete
  93  * nvme_reset_wq - hosts nvme reset works
  94  * nvme_delete_wq - hosts nvme delete works
  95  *
  96  * nvme_wq will host works such as scan, aen handling, fw activation,
  97  * keep-alive, periodic reconnects etc. nvme_reset_wq
  98  * runs reset works which also flush works hosted on nvme_wq for
  99  * serialization purposes. nvme_delete_wq host controller deletion
 100  * works which flush reset works for serialization.
 101  */
 102 struct workqueue_struct *nvme_wq;
 103 EXPORT_SYMBOL_GPL(nvme_wq);
 104
 105 struct workqueue_struct *nvme_reset_wq;
 106 EXPORT_SYMBOL_GPL(nvme_reset_wq);
 107
 108 struct workqueue_struct *nvme_delete_wq;
 109 EXPORT_SYMBOL_GPL(nvme_delete_wq);
 110
 111 static LIST_HEAD(nvme_subsystems);
 112 static DEFINE_MUTEX(nvme_subsystems_lock);
 113
 114 static DEFINE_IDA(nvme_instance_ida);
 115 static dev_t nvme_ctrl_base_chr_devt;
 116 static struct class *nvme_class;
 117 static struct class *nvme_subsys_class;
 118
 119 static DEFINE_IDA(nvme_ns_chr_minor_ida);
 120 static dev_t nvme_ns_chr_devt;
 121 static struct class *nvme_ns_chr_class;
 122
 123 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
 124 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
 125                                            unsigned nsid);
 126 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
 127                                    struct nvme_command *cmd);
 128
 129 void nvme_queue_scan(struct nvme_ctrl *ctrl)
 130 {
 131         /*
 132          * Only new queue scan work when admin and IO queues are both alive
 133          */
 134         if (ctrl->state == NVME_CTRL_LIVE && ctrl->tagset)
 135                 queue_work(nvme_wq, &ctrl->scan_work);
 136 }
 137
 138 /*
 139  * Use this function to proceed with scheduling reset_work for a controller
 140  * that had previously been set to the resetting state. This is intended for
 141  * code paths that can't be interrupted by other reset attempts. A hot removal
 142  * may prevent this from succeeding.
 143  */
 144 int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
 145 {
 146         if (ctrl->state != NVME_CTRL_RESETTING)
 147                 return -EBUSY;
 148         if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
 149                 return -EBUSY;
 150         return 0;
 151 }
 152 EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
 153
 154 static void nvme_failfast_work(struct work_struct *work)
 155 {
 156         struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
 157                         struct nvme_ctrl, failfast_work);
 158
 159         if (ctrl->state != NVME_CTRL_CONNECTING)
 160                 return;
 161
 162         set_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
 163         dev_info(ctrl->device, "failfast expired\n");
 164         nvme_kick_requeue_lists(ctrl);
 165 }
 166
 167 static inline void nvme_start_failfast_work(struct nvme_ctrl *ctrl)
 168 {
 169         if (!ctrl->opts || ctrl->opts->fast_io_fail_tmo == -1)
 170                 return;
 171
 172         schedule_delayed_work(&ctrl->failfast_work,
 173                               ctrl->opts->fast_io_fail_tmo * HZ);
 174 }
 175
 176 static inline void nvme_stop_failfast_work(struct nvme_ctrl *ctrl)
 177 {
 178         if (!ctrl->opts)
 179                 return;
 180
 181         cancel_delayed_work_sync(&ctrl->failfast_work);
 182         clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
 183 }
 184
 185
 186 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
 187 {
 188         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
 189                 return -EBUSY;
 190         if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
 191                 return -EBUSY;
 192         return 0;
 193 }
 194 EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
 195
 196 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
 197 {
 198         int ret;
 199
 200         ret = nvme_reset_ctrl(ctrl);
 201         if (!ret) {
 202                 flush_work(&ctrl->reset_work);
 203                 if (ctrl->state != NVME_CTRL_LIVE)
 204                         ret = -ENETRESET;
 205         }
 206
 207         return ret;
 208 }
 209
 210 static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)
 211 {
 212         dev_info(ctrl->device,
 213                  "Removing ctrl: NQN \"%s\"\n", nvmf_ctrl_subsysnqn(ctrl));
 214
 215         flush_work(&ctrl->reset_work);
 216         nvme_stop_ctrl(ctrl);
 217         nvme_remove_namespaces(ctrl);
 218         ctrl->ops->delete_ctrl(ctrl);
 219         nvme_uninit_ctrl(ctrl);
 220 }
 221
 222 static void nvme_delete_ctrl_work(struct work_struct *work)
 223 {
 224         struct nvme_ctrl *ctrl =
 225                 container_of(work, struct nvme_ctrl, delete_work);
 226
 227         nvme_do_delete_ctrl(ctrl);
 228 }
 229
 230 int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
 231 {
 232         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 233                 return -EBUSY;
 234         if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
 235                 return -EBUSY;
 236         return 0;
 237 }
 238 EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
 239
 240 static void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
 241 {
 242         /*
 243          * Keep a reference until nvme_do_delete_ctrl() complete,
 244          * since ->delete_ctrl can free the controller.
 245          */
 246         nvme_get_ctrl(ctrl);
 247         if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 248                 nvme_do_delete_ctrl(ctrl);
 249         nvme_put_ctrl(ctrl);
 250 }
 251
 252 static blk_status_t nvme_error_status(u16 status)
 253 {
 254         switch (status & 0x7ff) {
 255         case NVME_SC_SUCCESS:
 256                 return BLK_STS_OK;
 257         case NVME_SC_CAP_EXCEEDED:
 258                 return BLK_STS_NOSPC;
 259         case NVME_SC_LBA_RANGE:
 260         case NVME_SC_CMD_INTERRUPTED:
 261         case NVME_SC_NS_NOT_READY:
 262                 return BLK_STS_TARGET;
 263         case NVME_SC_BAD_ATTRIBUTES:
 264         case NVME_SC_ONCS_NOT_SUPPORTED:
 265         case NVME_SC_INVALID_OPCODE:
 266         case NVME_SC_INVALID_FIELD:
 267         case NVME_SC_INVALID_NS:
 268                 return BLK_STS_NOTSUPP;
 269         case NVME_SC_WRITE_FAULT:
 270         case NVME_SC_READ_ERROR:
 271         case NVME_SC_UNWRITTEN_BLOCK:
 272         case NVME_SC_ACCESS_DENIED:
 273         case NVME_SC_READ_ONLY:
 274         case NVME_SC_COMPARE_FAILED:
 275                 return BLK_STS_MEDIUM;
 276         case NVME_SC_GUARD_CHECK:
 277         case NVME_SC_APPTAG_CHECK:
 278         case NVME_SC_REFTAG_CHECK:
 279         case NVME_SC_INVALID_PI:
 280                 return BLK_STS_PROTECTION;
 281         case NVME_SC_RESERVATION_CONFLICT:
 282                 return BLK_STS_RESV_CONFLICT;
 283         case NVME_SC_HOST_PATH_ERROR:
 284                 return BLK_STS_TRANSPORT;
 285         case NVME_SC_ZONE_TOO_MANY_ACTIVE:
 286                 return BLK_STS_ZONE_ACTIVE_RESOURCE;
 287         case NVME_SC_ZONE_TOO_MANY_OPEN:
 288                 return BLK_STS_ZONE_OPEN_RESOURCE;
 289         default:
 290                 return BLK_STS_IOERR;
 291         }
 292 }
 293
 294 static void nvme_retry_req(struct request *req)
 295 {
 296         unsigned long delay = 0;
 297         u16 crd;
 298
 299         /* The mask and shift result must be <= 3 */
 300         crd = (nvme_req(req)->status & NVME_SC_CRD) >> 11;
 301         if (crd)
 302                 delay = nvme_req(req)->ctrl->crdt[crd - 1] * 100;
 303
 304         nvme_req(req)->retries++;
 305         blk_mq_requeue_request(req, false);
 306         blk_mq_delay_kick_requeue_list(req->q, delay);
 307 }
 308
 309 static void nvme_log_error(struct request *req)
 310 {
 311         struct nvme_ns *ns = req->q->queuedata;
 312         struct nvme_request *nr = nvme_req(req);
 313
 314         if (ns) {
 315                 pr_err_ratelimited("%s: %s(0x%x) @ LBA %llu, %llu blocks, %s (sct 0x%x / sc 0x%x) %s%s\n",
 316                        ns->disk ? ns->disk->disk_name : "?",
 317                        nvme_get_opcode_str(nr->cmd->common.opcode),
 318                        nr->cmd->common.opcode,
 319                        (unsigned long long)nvme_sect_to_lba(ns, blk_rq_pos(req)),
 320                        (unsigned long long)blk_rq_bytes(req) >> ns->lba_shift,
 321                        nvme_get_error_status_str(nr->status),
 322                        nr->status >> 8 & 7,     /* Status Code Type */
 323                        nr->status & 0xff,       /* Status Code */
 324                        nr->status & NVME_SC_MORE ? "MORE " : "",
 325                        nr->status & NVME_SC_DNR  ? "DNR "  : "");
 326                 return;
 327         }
 328
 329         pr_err_ratelimited("%s: %s(0x%x), %s (sct 0x%x / sc 0x%x) %s%s\n",
 330                            dev_name(nr->ctrl->device),
 331                            nvme_get_admin_opcode_str(nr->cmd->common.opcode),
 332                            nr->cmd->common.opcode,
 333                            nvme_get_error_status_str(nr->status),
 334                            nr->status >> 8 & 7, /* Status Code Type */
 335                            nr->status & 0xff,   /* Status Code */
 336                            nr->status & NVME_SC_MORE ? "MORE " : "",
 337                            nr->status & NVME_SC_DNR  ? "DNR "  : "");
 338 }
 339
 340 enum nvme_disposition {
 341         COMPLETE,
 342         RETRY,
 343         FAILOVER,
 344         AUTHENTICATE,
 345 };
 346
 347 static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
 348 {
 349         if (likely(nvme_req(req)->status == 0))
 350                 return COMPLETE;
 351
 352         if ((nvme_req(req)->status & 0x7ff) == NVME_SC_AUTH_REQUIRED)
 353                 return AUTHENTICATE;
 354
 355         if (blk_noretry_request(req) ||
 356             (nvme_req(req)->status & NVME_SC_DNR) ||
 357             nvme_req(req)->retries >= nvme_max_retries)
 358                 return COMPLETE;
 359
 360         if (req->cmd_flags & REQ_NVME_MPATH) {
 361                 if (nvme_is_path_error(nvme_req(req)->status) ||
 362                     blk_queue_dying(req->q))
 363                         return FAILOVER;
 364         } else {
 365                 if (blk_queue_dying(req->q))
 366                         return COMPLETE;
 367         }
 368
 369         return RETRY;
 370 }
 371
 372 static inline void nvme_end_req_zoned(struct request *req)
 373 {
 374         if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
 375             req_op(req) == REQ_OP_ZONE_APPEND)
 376                 req->__sector = nvme_lba_to_sect(req->q->queuedata,
 377                         le64_to_cpu(nvme_req(req)->result.u64));
 378 }
 379
 380 static inline void nvme_end_req(struct request *req)
 381 {
 382         blk_status_t status = nvme_error_status(nvme_req(req)->status);
 383
 384         if (unlikely(nvme_req(req)->status && !(req->rq_flags & RQF_QUIET)))
 385                 nvme_log_error(req);
 386         nvme_end_req_zoned(req);
 387         nvme_trace_bio_complete(req);
 388         if (req->cmd_flags & REQ_NVME_MPATH)
 389                 nvme_mpath_end_request(req);
 390         blk_mq_end_request(req, status);
 391 }
 392
 393 void nvme_complete_rq(struct request *req)
 394 {
 395         struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
 396
 397         trace_nvme_complete_rq(req);
 398         nvme_cleanup_cmd(req);
 399
 400         if (ctrl->kas)
 401                 ctrl->comp_seen = true;
 402
 403         switch (nvme_decide_disposition(req)) {
 404         case COMPLETE:
 405                 nvme_end_req(req);
 406                 return;
 407         case RETRY:
 408                 nvme_retry_req(req);
 409                 return;
 410         case FAILOVER:
 411                 nvme_failover_req(req);
 412                 return;
 413         case AUTHENTICATE:
 414 #ifdef CONFIG_NVME_AUTH
 415                 queue_work(nvme_wq, &ctrl->dhchap_auth_work);
 416                 nvme_retry_req(req);
 417 #else
 418                 nvme_end_req(req);
 419 #endif
 420                 return;
 421         }
 422 }
 423 EXPORT_SYMBOL_GPL(nvme_complete_rq);
 424
 425 void nvme_complete_batch_req(struct request *req)
 426 {
 427         trace_nvme_complete_rq(req);
 428         nvme_cleanup_cmd(req);
 429         nvme_end_req_zoned(req);
 430 }
 431 EXPORT_SYMBOL_GPL(nvme_complete_batch_req);
 432
 433 /*
 434  * Called to unwind from ->queue_rq on a failed command submission so that the
 435  * multipathing code gets called to potentially failover to another path.
 436  * The caller needs to unwind all transport specific resource allocations and
 437  * must return propagate the return value.
 438  */
 439 blk_status_t nvme_host_path_error(struct request *req)
 440 {
 441         nvme_req(req)->status = NVME_SC_HOST_PATH_ERROR;
 442         blk_mq_set_request_complete(req);
 443         nvme_complete_rq(req);
 444         return BLK_STS_OK;
 445 }
 446 EXPORT_SYMBOL_GPL(nvme_host_path_error);
 447
 448 bool nvme_cancel_request(struct request *req, void *data)
 449 {
 450         dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
 451                                 "Cancelling I/O %d", req->tag);
 452
 453         /* don't abort one completed or idle request */
 454         if (blk_mq_rq_state(req) != MQ_RQ_IN_FLIGHT)
 455                 return true;
 456
 457         nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
 458         nvme_req(req)->flags |= NVME_REQ_CANCELLED;
 459         blk_mq_complete_request(req);
 460         return true;
 461 }
 462 EXPORT_SYMBOL_GPL(nvme_cancel_request);
 463
 464 void nvme_cancel_tagset(struct nvme_ctrl *ctrl)
 465 {
 466         if (ctrl->tagset) {
 467                 blk_mq_tagset_busy_iter(ctrl->tagset,
 468                                 nvme_cancel_request, ctrl);
 469                 blk_mq_tagset_wait_completed_request(ctrl->tagset);
 470         }
 471 }
 472 EXPORT_SYMBOL_GPL(nvme_cancel_tagset);
 473
 474 void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)
 475 {
 476         if (ctrl->admin_tagset) {
 477                 blk_mq_tagset_busy_iter(ctrl->admin_tagset,
 478                                 nvme_cancel_request, ctrl);
 479                 blk_mq_tagset_wait_completed_request(ctrl->admin_tagset);
 480         }
 481 }
 482 EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset);
 483
 484 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 485                 enum nvme_ctrl_state new_state)
 486 {
 487         enum nvme_ctrl_state old_state;
 488         unsigned long flags;
 489         bool changed = false;
 490
 491         spin_lock_irqsave(&ctrl->lock, flags);
 492
 493         old_state = ctrl->state;
 494         switch (new_state) {
 495         case NVME_CTRL_LIVE:
 496                 switch (old_state) {
 497                 case NVME_CTRL_NEW:
 498                 case NVME_CTRL_RESETTING:
 499                 case NVME_CTRL_CONNECTING:
 500                         changed = true;
 501                         fallthrough;
 502                 default:
 503                         break;
 504                 }
 505                 break;
 506         case NVME_CTRL_RESETTING:
 507                 switch (old_state) {
 508                 case NVME_CTRL_NEW:
 509                 case NVME_CTRL_LIVE:
 510                         changed = true;
 511                         fallthrough;
 512                 default:
 513                         break;
 514                 }
 515                 break;
 516         case NVME_CTRL_CONNECTING:
 517                 switch (old_state) {
 518                 case NVME_CTRL_NEW:
 519                 case NVME_CTRL_RESETTING:
 520                         changed = true;
 521                         fallthrough;
 522                 default:
 523                         break;
 524                 }
 525                 break;
 526         case NVME_CTRL_DELETING:
 527                 switch (old_state) {
 528                 case NVME_CTRL_LIVE:
 529                 case NVME_CTRL_RESETTING:
 530                 case NVME_CTRL_CONNECTING:
 531                         changed = true;
 532                         fallthrough;
 533                 default:
 534                         break;
 535                 }
 536                 break;
 537         case NVME_CTRL_DELETING_NOIO:
 538                 switch (old_state) {
 539                 case NVME_CTRL_DELETING:
 540                 case NVME_CTRL_DEAD:
 541                         changed = true;
 542                         fallthrough;
 543                 default:
 544                         break;
 545                 }
 546                 break;
 547         case NVME_CTRL_DEAD:
 548                 switch (old_state) {
 549                 case NVME_CTRL_DELETING:
 550                         changed = true;
 551                         fallthrough;
 552                 default:
 553                         break;
 554                 }
 555                 break;
 556         default:
 557                 break;
 558         }
 559
 560         if (changed) {
 561                 ctrl->state = new_state;
 562                 wake_up_all(&ctrl->state_wq);
 563         }
 564
 565         spin_unlock_irqrestore(&ctrl->lock, flags);
 566         if (!changed)
 567                 return false;
 568
 569         if (ctrl->state == NVME_CTRL_LIVE) {
 570                 if (old_state == NVME_CTRL_CONNECTING)
 571                         nvme_stop_failfast_work(ctrl);
 572                 nvme_kick_requeue_lists(ctrl);
 573         } else if (ctrl->state == NVME_CTRL_CONNECTING &&
 574                 old_state == NVME_CTRL_RESETTING) {
 575                 nvme_start_failfast_work(ctrl);
 576         }
 577         return changed;
 578 }
 579 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
 580
 581 /*
 582  * Returns true for sink states that can't ever transition back to live.
 583  */
 584 static bool nvme_state_terminal(struct nvme_ctrl *ctrl)
 585 {
 586         switch (ctrl->state) {
 587         case NVME_CTRL_NEW:
 588         case NVME_CTRL_LIVE:
 589         case NVME_CTRL_RESETTING:
 590         case NVME_CTRL_CONNECTING:
 591                 return false;
 592         case NVME_CTRL_DELETING:
 593         case NVME_CTRL_DELETING_NOIO:
 594         case NVME_CTRL_DEAD:
 595                 return true;
 596         default:
 597                 WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);
 598                 return true;
 599         }
 600 }
 601
 602 /*
 603  * Waits for the controller state to be resetting, or returns false if it is
 604  * not possible to ever transition to that state.
 605  */
 606 bool nvme_wait_reset(struct nvme_ctrl *ctrl)
 607 {
 608         wait_event(ctrl->state_wq,
 609                    nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) ||
 610                    nvme_state_terminal(ctrl));
 611         return ctrl->state == NVME_CTRL_RESETTING;
 612 }
 613 EXPORT_SYMBOL_GPL(nvme_wait_reset);
 614
 615 static void nvme_free_ns_head(struct kref *ref)
 616 {
 617         struct nvme_ns_head *head =
 618                 container_of(ref, struct nvme_ns_head, ref);
 619
 620         nvme_mpath_remove_disk(head);
 621         ida_free(&head->subsys->ns_ida, head->instance);
 622         cleanup_srcu_struct(&head->srcu);
 623         nvme_put_subsystem(head->subsys);
 624         kfree(head);
 625 }
 626
 627 bool nvme_tryget_ns_head(struct nvme_ns_head *head)
 628 {
 629         return kref_get_unless_zero(&head->ref);
 630 }
 631
 632 void nvme_put_ns_head(struct nvme_ns_head *head)
 633 {
 634         kref_put(&head->ref, nvme_free_ns_head);
 635 }
 636
 637 static void nvme_free_ns(struct kref *kref)
 638 {
 639         struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
 640
 641         put_disk(ns->disk);
 642         nvme_put_ns_head(ns->head);
 643         nvme_put_ctrl(ns->ctrl);
 644         kfree(ns);
 645 }
 646
 647 static inline bool nvme_get_ns(struct nvme_ns *ns)
 648 {
 649         return kref_get_unless_zero(&ns->kref);
 650 }
 651
 652 void nvme_put_ns(struct nvme_ns *ns)
 653 {
 654         kref_put(&ns->kref, nvme_free_ns);
 655 }
 656 EXPORT_SYMBOL_NS_GPL(nvme_put_ns, NVME_TARGET_PASSTHRU);
 657
 658 static inline void nvme_clear_nvme_request(struct request *req)
 659 {
 660         nvme_req(req)->status = 0;
 661         nvme_req(req)->retries = 0;
 662         nvme_req(req)->flags = 0;
 663         req->rq_flags |= RQF_DONTPREP;
 664 }
 665
 666 /* initialize a passthrough request */
 667 void nvme_init_request(struct request *req, struct nvme_command *cmd)
 668 {
 669         if (req->q->queuedata)
 670                 req->timeout = NVME_IO_TIMEOUT;
 671         else /* no queuedata implies admin queue */
 672                 req->timeout = NVME_ADMIN_TIMEOUT;
 673
 674         /* passthru commands should let the driver set the SGL flags */
 675         cmd->common.flags &= ~NVME_CMD_SGL_ALL;
 676
 677         req->cmd_flags |= REQ_FAILFAST_DRIVER;
 678         if (req->mq_hctx->type == HCTX_TYPE_POLL)
 679                 req->cmd_flags |= REQ_POLLED;
 680         nvme_clear_nvme_request(req);
 681         req->rq_flags |= RQF_QUIET;
 682         memcpy(nvme_req(req)->cmd, cmd, sizeof(*cmd));
 683 }
 684 EXPORT_SYMBOL_GPL(nvme_init_request);
 685
 686 /*
 687  * For something we're not in a state to send to the device the default action
 688  * is to busy it and retry it after the controller state is recovered.  However,
 689  * if the controller is deleting or if anything is marked for failfast or
 690  * nvme multipath it is immediately failed.
 691  *
 692  * Note: commands used to initialize the controller will be marked for failfast.
 693  * Note: nvme cli/ioctl commands are marked for failfast.
 694  */
 695 blk_status_t nvme_fail_nonready_command(struct nvme_ctrl *ctrl,
 696                 struct request *rq)
 697 {
 698         if (ctrl->state != NVME_CTRL_DELETING_NOIO &&
 699             ctrl->state != NVME_CTRL_DELETING &&
 700             ctrl->state != NVME_CTRL_DEAD &&
 701             !test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags) &&
 702             !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH))
 703                 return BLK_STS_RESOURCE;
 704         return nvme_host_path_error(rq);
 705 }
 706 EXPORT_SYMBOL_GPL(nvme_fail_nonready_command);
 707
 708 bool __nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq,
 709                 bool queue_live)
 710 {
 711         struct nvme_request *req = nvme_req(rq);
 712
 713         /*
 714          * currently we have a problem sending passthru commands
 715          * on the admin_q if the controller is not LIVE because we can't
 716          * make sure that they are going out after the admin connect,
 717          * controller enable and/or other commands in the initialization
 718          * sequence. until the controller will be LIVE, fail with
 719          * BLK_STS_RESOURCE so that they will be rescheduled.
 720          */
 721         if (rq->q == ctrl->admin_q && (req->flags & NVME_REQ_USERCMD))
 722                 return false;
 723
 724         if (ctrl->ops->flags & NVME_F_FABRICS) {
 725                 /*
 726                  * Only allow commands on a live queue, except for the connect
 727                  * command, which is require to set the queue live in the
 728                  * appropinquate states.
 729                  */
 730                 switch (ctrl->state) {
 731                 case NVME_CTRL_CONNECTING:
 732                         if (blk_rq_is_passthrough(rq) && nvme_is_fabrics(req->cmd) &&
 733                             (req->cmd->fabrics.fctype == nvme_fabrics_type_connect ||
 734                              req->cmd->fabrics.fctype == nvme_fabrics_type_auth_send ||
 735                              req->cmd->fabrics.fctype == nvme_fabrics_type_auth_receive))
 736                                 return true;
 737                         break;
 738                 default:
 739                         break;
 740                 case NVME_CTRL_DEAD:
 741                         return false;
 742                 }
 743         }
 744
 745         return queue_live;
 746 }
 747 EXPORT_SYMBOL_GPL(__nvme_check_ready);
 748
 749 static inline void nvme_setup_flush(struct nvme_ns *ns,
 750                 struct nvme_command *cmnd)
 751 {
 752         memset(cmnd, 0, sizeof(*cmnd));
 753         cmnd->common.opcode = nvme_cmd_flush;
 754         cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
 755 }
 756
 757 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
 758                 struct nvme_command *cmnd)
 759 {
 760         unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
 761         struct nvme_dsm_range *range;
 762         struct bio *bio;
 763
 764         /*
 765          * Some devices do not consider the DSM 'Number of Ranges' field when
 766          * determining how much data to DMA. Always allocate memory for maximum
 767          * number of segments to prevent device reading beyond end of buffer.
 768          */
 769         static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES;
 770
 771         range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN);
 772         if (!range) {
 773                 /*
 774                  * If we fail allocation our range, fallback to the controller
 775                  * discard page. If that's also busy, it's safe to return
 776                  * busy, as we know we can make progress once that's freed.
 777                  */
 778                 if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy))
 779                         return BLK_STS_RESOURCE;
 780
 781                 range = page_address(ns->ctrl->discard_page);
 782         }
 783
 784         if (queue_max_discard_segments(req->q) == 1) {
 785                 u64 slba = nvme_sect_to_lba(ns, blk_rq_pos(req));
 786                 u32 nlb = blk_rq_sectors(req) >> (ns->lba_shift - 9);
 787
 788                 range[0].cattr = cpu_to_le32(0);
 789                 range[0].nlb = cpu_to_le32(nlb);
 790                 range[0].slba = cpu_to_le64(slba);
 791                 n = 1;
 792         } else {
 793                 __rq_for_each_bio(bio, req) {
 794                         u64 slba = nvme_sect_to_lba(ns, bio->bi_iter.bi_sector);
 795                         u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
 796
 797                         if (n < segments) {
 798                                 range[n].cattr = cpu_to_le32(0);
 799                                 range[n].nlb = cpu_to_le32(nlb);
 800                                 range[n].slba = cpu_to_le64(slba);
 801                         }
 802                         n++;
 803                 }
 804         }
 805
 806         if (WARN_ON_ONCE(n != segments)) {
 807                 if (virt_to_page(range) == ns->ctrl->discard_page)
 808                         clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
 809                 else
 810                         kfree(range);
 811                 return BLK_STS_IOERR;
 812         }
 813
 814         memset(cmnd, 0, sizeof(*cmnd));
 815         cmnd->dsm.opcode = nvme_cmd_dsm;
 816         cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
 817         cmnd->dsm.nr = cpu_to_le32(segments - 1);
 818         cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
 819
 820         bvec_set_virt(&req->special_vec, range, alloc_size);
 821         req->rq_flags |= RQF_SPECIAL_PAYLOAD;
 822
 823         return BLK_STS_OK;
 824 }
 825
 826 static void nvme_set_ref_tag(struct nvme_ns *ns, struct nvme_command *cmnd,
 827                               struct request *req)
 828 {
 829         u32 upper, lower;
 830         u64 ref48;
 831
 832         /* both rw and write zeroes share the same reftag format */
 833         switch (ns->guard_type) {
 834         case NVME_NVM_NS_16B_GUARD:
 835                 cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
 836                 break;
 837         case NVME_NVM_NS_64B_GUARD:
 838                 ref48 = ext_pi_ref_tag(req);
 839                 lower = lower_32_bits(ref48);
 840                 upper = upper_32_bits(ref48);
 841
 842                 cmnd->rw.reftag = cpu_to_le32(lower);
 843                 cmnd->rw.cdw3 = cpu_to_le32(upper);
 844                 break;
 845         default:
 846                 break;
 847         }
 848 }
 849
 850 static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
 851                 struct request *req, struct nvme_command *cmnd)
 852 {
 853         memset(cmnd, 0, sizeof(*cmnd));
 854
 855         if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
 856                 return nvme_setup_discard(ns, req, cmnd);
 857
 858         cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
 859         cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
 860         cmnd->write_zeroes.slba =
 861                 cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
 862         cmnd->write_zeroes.length =
 863                 cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
 864
 865         if (!(req->cmd_flags & REQ_NOUNMAP) && (ns->features & NVME_NS_DEAC))
 866                 cmnd->write_zeroes.control |= cpu_to_le16(NVME_WZ_DEAC);
 867
 868         if (nvme_ns_has_pi(ns)) {
 869                 cmnd->write_zeroes.control |= cpu_to_le16(NVME_RW_PRINFO_PRACT);
 870
 871                 switch (ns->pi_type) {
 872                 case NVME_NS_DPS_PI_TYPE1:
 873                 case NVME_NS_DPS_PI_TYPE2:
 874                         nvme_set_ref_tag(ns, cmnd, req);
 875                         break;
 876                 }
 877         }
 878
 879         return BLK_STS_OK;
 880 }
 881
 882 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
 883                 struct request *req, struct nvme_command *cmnd,
 884                 enum nvme_opcode op)
 885 {
 886         u16 control = 0;
 887         u32 dsmgmt = 0;
 888
 889         if (req->cmd_flags & REQ_FUA)
 890                 control |= NVME_RW_FUA;
 891         if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
 892                 control |= NVME_RW_LR;
 893
 894         if (req->cmd_flags & REQ_RAHEAD)
 895                 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
 896
 897         cmnd->rw.opcode = op;
 898         cmnd->rw.flags = 0;
 899         cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
 900         cmnd->rw.cdw2 = 0;
 901         cmnd->rw.cdw3 = 0;
 902         cmnd->rw.metadata = 0;
 903         cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
 904         cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
 905         cmnd->rw.reftag = 0;
 906         cmnd->rw.apptag = 0;
 907         cmnd->rw.appmask = 0;
 908
 909         if (ns->ms) {
 910                 /*
 911                  * If formated with metadata, the block layer always provides a
 912                  * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled.  Else
 913                  * we enable the PRACT bit for protection information or set the
 914                  * namespace capacity to zero to prevent any I/O.
 915                  */
 916                 if (!blk_integrity_rq(req)) {
 917                         if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
 918                                 return BLK_STS_NOTSUPP;
 919                         control |= NVME_RW_PRINFO_PRACT;
 920                 }
 921
 922                 switch (ns->pi_type) {
 923                 case NVME_NS_DPS_PI_TYPE3:
 924                         control |= NVME_RW_PRINFO_PRCHK_GUARD;
 925                         break;
 926                 case NVME_NS_DPS_PI_TYPE1:
 927                 case NVME_NS_DPS_PI_TYPE2:
 928                         control |= NVME_RW_PRINFO_PRCHK_GUARD |
 929                                         NVME_RW_PRINFO_PRCHK_REF;
 930                         if (op == nvme_cmd_zone_append)
 931                                 control |= NVME_RW_APPEND_PIREMAP;
 932                         nvme_set_ref_tag(ns, cmnd, req);
 933                         break;
 934                 }
 935         }
 936
 937         cmnd->rw.control = cpu_to_le16(control);
 938         cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
 939         return 0;
 940 }
 941
 942 void nvme_cleanup_cmd(struct request *req)
 943 {
 944         if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
 945                 struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
 946
 947                 if (req->special_vec.bv_page == ctrl->discard_page)
 948                         clear_bit_unlock(0, &ctrl->discard_page_busy);
 949                 else
 950                         kfree(bvec_virt(&req->special_vec));
 951         }
 952 }
 953 EXPORT_SYMBOL_GPL(nvme_cleanup_cmd);
 954
 955 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req)
 956 {
 957         struct nvme_command *cmd = nvme_req(req)->cmd;
 958         blk_status_t ret = BLK_STS_OK;
 959
 960         if (!(req->rq_flags & RQF_DONTPREP))
 961                 nvme_clear_nvme_request(req);
 962
 963         switch (req_op(req)) {
 964         case REQ_OP_DRV_IN:
 965         case REQ_OP_DRV_OUT:
 966                 /* these are setup prior to execution in nvme_init_request() */
 967                 break;
 968         case REQ_OP_FLUSH:
 969                 nvme_setup_flush(ns, cmd);
 970                 break;
 971         case REQ_OP_ZONE_RESET_ALL:
 972         case REQ_OP_ZONE_RESET:
 973                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_RESET);
 974                 break;
 975         case REQ_OP_ZONE_OPEN:
 976                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_OPEN);
 977                 break;
 978         case REQ_OP_ZONE_CLOSE:
 979                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_CLOSE);
 980                 break;
 981         case REQ_OP_ZONE_FINISH:
 982                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_FINISH);
 983                 break;
 984         case REQ_OP_WRITE_ZEROES:
 985                 ret = nvme_setup_write_zeroes(ns, req, cmd);
 986                 break;
 987         case REQ_OP_DISCARD:
 988                 ret = nvme_setup_discard(ns, req, cmd);
 989                 break;
 990         case REQ_OP_READ:
 991                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_read);
 992                 break;
 993         case REQ_OP_WRITE:
 994                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_write);
 995                 break;
 996         case REQ_OP_ZONE_APPEND:
 997                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_zone_append);
 998                 break;
 999         default:
1000                 WARN_ON_ONCE(1);
1001                 return BLK_STS_IOERR;
1002         }
1003
1004         cmd->common.command_id = nvme_cid(req);
1005         trace_nvme_setup_cmd(req, cmd);
1006         return ret;
1007 }
1008 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
1009
1010 /*
1011  * Return values:
1012  * 0:  success
1013  * >0: nvme controller's cqe status response
1014  * <0: kernel error in lieu of controller response
1015  */
1016 int nvme_execute_rq(struct request *rq, bool at_head)
1017 {
1018         blk_status_t status;
1019
1020         status = blk_execute_rq(rq, at_head);
1021         if (nvme_req(rq)->flags & NVME_REQ_CANCELLED)
1022                 return -EINTR;
1023         if (nvme_req(rq)->status)
1024                 return nvme_req(rq)->status;
1025         return blk_status_to_errno(status);
1026 }
1027 EXPORT_SYMBOL_NS_GPL(nvme_execute_rq, NVME_TARGET_PASSTHRU);
1028
1029 /*
1030  * Returns 0 on success.  If the result is negative, it's a Linux error code;
1031  * if the result is positive, it's an NVM Express status code
1032  */
1033 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
1034                 union nvme_result *result, void *buffer, unsigned bufflen,
1035                 int qid, int at_head, blk_mq_req_flags_t flags)
1036 {
1037         struct request *req;
1038         int ret;
1039
1040         if (qid == NVME_QID_ANY)
1041                 req = blk_mq_alloc_request(q, nvme_req_op(cmd), flags);
1042         else
1043                 req = blk_mq_alloc_request_hctx(q, nvme_req_op(cmd), flags,
1044                                                 qid - 1);
1045
1046         if (IS_ERR(req))
1047                 return PTR_ERR(req);
1048         nvme_init_request(req, cmd);
1049
1050         if (buffer && bufflen) {
1051                 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
1052                 if (ret)
1053                         goto out;
1054         }
1055
1056         ret = nvme_execute_rq(req, at_head);
1057         if (result && ret >= 0)
1058                 *result = nvme_req(req)->result;
1059  out:
1060         blk_mq_free_request(req);
1061         return ret;
1062 }
1063 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
1064
1065 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
1066                 void *buffer, unsigned bufflen)
1067 {
1068         return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen,
1069                         NVME_QID_ANY, 0, 0);
1070 }
1071 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
1072
1073 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
1074 {
1075         u32 effects = 0;
1076
1077         if (ns) {
1078                 effects = le32_to_cpu(ns->head->effects->iocs[opcode]);
1079                 if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC))
1080                         dev_warn_once(ctrl->device,
1081                                 "IO command:%02x has unusual effects:%08x\n",
1082                                 opcode, effects);
1083
1084                 /*
1085                  * NVME_CMD_EFFECTS_CSE_MASK causes a freeze all I/O queues,
1086                  * which would deadlock when done on an I/O command.  Note that
1087                  * We already warn about an unusual effect above.
1088                  */
1089                 effects &= ~NVME_CMD_EFFECTS_CSE_MASK;
1090         } else {
1091                 effects = le32_to_cpu(ctrl->effects->acs[opcode]);
1092         }
1093
1094         return effects;
1095 }
1096 EXPORT_SYMBOL_NS_GPL(nvme_command_effects, NVME_TARGET_PASSTHRU);
1097
1098 u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
1099 {
1100         u32 effects = nvme_command_effects(ctrl, ns, opcode);
1101
1102         /*
1103          * For simplicity, IO to all namespaces is quiesced even if the command
1104          * effects say only one namespace is affected.
1105          */
1106         if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1107                 mutex_lock(&ctrl->scan_lock);
1108                 mutex_lock(&ctrl->subsys->lock);
1109                 nvme_mpath_start_freeze(ctrl->subsys);
1110                 nvme_mpath_wait_freeze(ctrl->subsys);
1111                 nvme_start_freeze(ctrl);
1112                 nvme_wait_freeze(ctrl);
1113         }
1114         return effects;
1115 }
1116 EXPORT_SYMBOL_NS_GPL(nvme_passthru_start, NVME_TARGET_PASSTHRU);
1117
1118 void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects,
1119                        struct nvme_command *cmd, int status)
1120 {
1121         if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1122                 nvme_unfreeze(ctrl);
1123                 nvme_mpath_unfreeze(ctrl->subsys);
1124                 mutex_unlock(&ctrl->subsys->lock);
1125                 mutex_unlock(&ctrl->scan_lock);
1126         }
1127         if (effects & NVME_CMD_EFFECTS_CCC) {
1128                 dev_info(ctrl->device,
1129 "controller capabilities changed, reset may be required to take effect.\n");
1130         }
1131         if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) {
1132                 nvme_queue_scan(ctrl);
1133                 flush_work(&ctrl->scan_work);
1134         }
1135
1136         switch (cmd->common.opcode) {
1137         case nvme_admin_set_features:
1138                 switch (le32_to_cpu(cmd->common.cdw10) & 0xFF) {
1139                 case NVME_FEAT_KATO:
1140                         /*
1141                          * Keep alive commands interval on the host should be
1142                          * updated when KATO is modified by Set Features
1143                          * commands.
1144                          */
1145                         if (!status)
1146                                 nvme_update_keep_alive(ctrl, cmd);
1147                         break;
1148                 default:
1149                         break;
1150                 }
1151                 break;
1152         default:
1153                 break;
1154         }
1155 }
1156 EXPORT_SYMBOL_NS_GPL(nvme_passthru_end, NVME_TARGET_PASSTHRU);
1157
1158 /*
1159  * Recommended frequency for KATO commands per NVMe 1.4 section 7.12.1:
1160  *
1161  *   The host should send Keep Alive commands at half of the Keep Alive Timeout
1162  *   accounting for transport roundtrip times [..].
1163  */
1164 static void nvme_queue_keep_alive_work(struct nvme_ctrl *ctrl)
1165 {
1166         queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ / 2);
1167 }
1168
1169 static enum rq_end_io_ret nvme_keep_alive_end_io(struct request *rq,
1170                                                  blk_status_t status)
1171 {
1172         struct nvme_ctrl *ctrl = rq->end_io_data;
1173         unsigned long flags;
1174         bool startka = false;
1175
1176         blk_mq_free_request(rq);
1177
1178         if (status) {
1179                 dev_err(ctrl->device,
1180                         "failed nvme_keep_alive_end_io error=%d\n",
1181                                 status);
1182                 return RQ_END_IO_NONE;
1183         }
1184
1185         ctrl->comp_seen = false;
1186         spin_lock_irqsave(&ctrl->lock, flags);
1187         if (ctrl->state == NVME_CTRL_LIVE ||
1188             ctrl->state == NVME_CTRL_CONNECTING)
1189                 startka = true;
1190         spin_unlock_irqrestore(&ctrl->lock, flags);
1191         if (startka)
1192                 nvme_queue_keep_alive_work(ctrl);
1193         return RQ_END_IO_NONE;
1194 }
1195
1196 static void nvme_keep_alive_work(struct work_struct *work)
1197 {
1198         struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
1199                         struct nvme_ctrl, ka_work);
1200         bool comp_seen = ctrl->comp_seen;
1201         struct request *rq;
1202
1203         if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) {
1204                 dev_dbg(ctrl->device,
1205                         "reschedule traffic based keep-alive timer\n");
1206                 ctrl->comp_seen = false;
1207                 nvme_queue_keep_alive_work(ctrl);
1208                 return;
1209         }
1210
1211         rq = blk_mq_alloc_request(ctrl->admin_q, nvme_req_op(&ctrl->ka_cmd),
1212                                   BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
1213         if (IS_ERR(rq)) {
1214                 /* allocation failure, reset the controller */
1215                 dev_err(ctrl->device, "keep-alive failed: %ld\n", PTR_ERR(rq));
1216                 nvme_reset_ctrl(ctrl);
1217                 return;
1218         }
1219         nvme_init_request(rq, &ctrl->ka_cmd);
1220
1221         rq->timeout = ctrl->kato * HZ;
1222         rq->end_io = nvme_keep_alive_end_io;
1223         rq->end_io_data = ctrl;
1224         blk_execute_rq_nowait(rq, false);
1225 }
1226
1227 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
1228 {
1229         if (unlikely(ctrl->kato == 0))
1230                 return;
1231
1232         nvme_queue_keep_alive_work(ctrl);
1233 }
1234
1235 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
1236 {
1237         if (unlikely(ctrl->kato == 0))
1238                 return;
1239
1240         cancel_delayed_work_sync(&ctrl->ka_work);
1241 }
1242 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
1243
1244 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
1245                                    struct nvme_command *cmd)
1246 {
1247         unsigned int new_kato =
1248                 DIV_ROUND_UP(le32_to_cpu(cmd->common.cdw11), 1000);
1249
1250         dev_info(ctrl->device,
1251                  "keep alive interval updated from %u ms to %u ms\n",
1252                  ctrl->kato * 1000 / 2, new_kato * 1000 / 2);
1253
1254         nvme_stop_keep_alive(ctrl);
1255         ctrl->kato = new_kato;
1256         nvme_start_keep_alive(ctrl);
1257 }
1258
1259 /*
1260  * In NVMe 1.0 the CNS field was just a binary controller or namespace
1261  * flag, thus sending any new CNS opcodes has a big chance of not working.
1262  * Qemu unfortunately had that bug after reporting a 1.1 version compliance
1263  * (but not for any later version).
1264  */
1265 static bool nvme_ctrl_limited_cns(struct nvme_ctrl *ctrl)
1266 {
1267         if (ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)
1268                 return ctrl->vs < NVME_VS(1, 2, 0);
1269         return ctrl->vs < NVME_VS(1, 1, 0);
1270 }
1271
1272 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
1273 {
1274         struct nvme_command c = { };
1275         int error;
1276
1277         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1278         c.identify.opcode = nvme_admin_identify;
1279         c.identify.cns = NVME_ID_CNS_CTRL;
1280
1281         *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
1282         if (!*id)
1283                 return -ENOMEM;
1284
1285         error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1286                         sizeof(struct nvme_id_ctrl));
1287         if (error)
1288                 kfree(*id);
1289         return error;
1290 }
1291
1292 static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,
1293                 struct nvme_ns_id_desc *cur, bool *csi_seen)
1294 {
1295         const char *warn_str = "ctrl returned bogus length:";
1296         void *data = cur;
1297
1298         switch (cur->nidt) {
1299         case NVME_NIDT_EUI64:
1300                 if (cur->nidl != NVME_NIDT_EUI64_LEN) {
1301                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_EUI64\n",
1302                                  warn_str, cur->nidl);
1303                         return -1;
1304                 }
1305                 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1306                         return NVME_NIDT_EUI64_LEN;
1307                 memcpy(ids->eui64, data + sizeof(*cur), NVME_NIDT_EUI64_LEN);
1308                 return NVME_NIDT_EUI64_LEN;
1309         case NVME_NIDT_NGUID:
1310                 if (cur->nidl != NVME_NIDT_NGUID_LEN) {
1311                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_NGUID\n",
1312                                  warn_str, cur->nidl);
1313                         return -1;
1314                 }
1315                 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1316                         return NVME_NIDT_NGUID_LEN;
1317                 memcpy(ids->nguid, data + sizeof(*cur), NVME_NIDT_NGUID_LEN);
1318                 return NVME_NIDT_NGUID_LEN;
1319         case NVME_NIDT_UUID:
1320                 if (cur->nidl != NVME_NIDT_UUID_LEN) {
1321                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_UUID\n",
1322                                  warn_str, cur->nidl);
1323                         return -1;
1324                 }
1325                 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1326                         return NVME_NIDT_UUID_LEN;
1327                 uuid_copy(&ids->uuid, data + sizeof(*cur));
1328                 return NVME_NIDT_UUID_LEN;
1329         case NVME_NIDT_CSI:
1330                 if (cur->nidl != NVME_NIDT_CSI_LEN) {
1331                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_CSI\n",
1332                                  warn_str, cur->nidl);
1333                         return -1;
1334                 }
1335                 memcpy(&ids->csi, data + sizeof(*cur), NVME_NIDT_CSI_LEN);
1336                 *csi_seen = true;
1337                 return NVME_NIDT_CSI_LEN;
1338         default:
1339                 /* Skip unknown types */
1340                 return cur->nidl;
1341         }
1342 }
1343
1344 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl,
1345                 struct nvme_ns_info *info)
1346 {
1347         struct nvme_command c = { };
1348         bool csi_seen = false;
1349         int status, pos, len;
1350         void *data;
1351
1352         if (ctrl->vs < NVME_VS(1, 3, 0) && !nvme_multi_css(ctrl))
1353                 return 0;
1354         if (ctrl->quirks & NVME_QUIRK_NO_NS_DESC_LIST)
1355                 return 0;
1356
1357         c.identify.opcode = nvme_admin_identify;
1358         c.identify.nsid = cpu_to_le32(info->nsid);
1359         c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
1360
1361         data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
1362         if (!data)
1363                 return -ENOMEM;
1364
1365         status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
1366                                       NVME_IDENTIFY_DATA_SIZE);
1367         if (status) {
1368                 dev_warn(ctrl->device,
1369                         "Identify Descriptors failed (nsid=%u, status=0x%x)\n",
1370                         info->nsid, status);
1371                 goto free_data;
1372         }
1373
1374         for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
1375                 struct nvme_ns_id_desc *cur = data + pos;
1376
1377                 if (cur->nidl == 0)
1378                         break;
1379
1380                 len = nvme_process_ns_desc(ctrl, &info->ids, cur, &csi_seen);
1381                 if (len < 0)
1382                         break;
1383
1384                 len += sizeof(*cur);
1385         }
1386
1387         if (nvme_multi_css(ctrl) && !csi_seen) {
1388                 dev_warn(ctrl->device, "Command set not reported for nsid:%d\n",
1389                          info->nsid);
1390                 status = -EINVAL;
1391         }
1392
1393 free_data:
1394         kfree(data);
1395         return status;
1396 }
1397
1398 static int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid,
1399                         struct nvme_id_ns **id)
1400 {
1401         struct nvme_command c = { };
1402         int error;
1403
1404         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1405         c.identify.opcode = nvme_admin_identify;
1406         c.identify.nsid = cpu_to_le32(nsid);
1407         c.identify.cns = NVME_ID_CNS_NS;
1408
1409         *id = kmalloc(sizeof(**id), GFP_KERNEL);
1410         if (!*id)
1411                 return -ENOMEM;
1412
1413         error = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id));
1414         if (error) {
1415                 dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error);
1416                 kfree(*id);
1417         }
1418         return error;
1419 }
1420
1421 static int nvme_ns_info_from_identify(struct nvme_ctrl *ctrl,
1422                 struct nvme_ns_info *info)
1423 {
1424         struct nvme_ns_ids *ids = &info->ids;
1425         struct nvme_id_ns *id;
1426         int ret;
1427
1428         ret = nvme_identify_ns(ctrl, info->nsid, &id);
1429         if (ret)
1430                 return ret;
1431
1432         if (id->ncap == 0) {
1433                 /* namespace not allocated or attached */
1434                 info->is_removed = true;
1435                 return -ENODEV;
1436         }
1437
1438         info->anagrpid = id->anagrpid;
1439         info->is_shared = id->nmic & NVME_NS_NMIC_SHARED;
1440         info->is_readonly = id->nsattr & NVME_NS_ATTR_RO;
1441         info->is_ready = true;
1442         if (ctrl->quirks & NVME_QUIRK_BOGUS_NID) {
1443                 dev_info(ctrl->device,
1444                          "Ignoring bogus Namespace Identifiers\n");
1445         } else {
1446                 if (ctrl->vs >= NVME_VS(1, 1, 0) &&
1447                     !memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
1448                         memcpy(ids->eui64, id->eui64, sizeof(ids->eui64));
1449                 if (ctrl->vs >= NVME_VS(1, 2, 0) &&
1450                     !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
1451                         memcpy(ids->nguid, id->nguid, sizeof(ids->nguid));
1452         }
1453         kfree(id);
1454         return 0;
1455 }
1456
1457 static int nvme_ns_info_from_id_cs_indep(struct nvme_ctrl *ctrl,
1458                 struct nvme_ns_info *info)
1459 {
1460         struct nvme_id_ns_cs_indep *id;
1461         struct nvme_command c = {
1462                 .identify.opcode        = nvme_admin_identify,
1463                 .identify.nsid          = cpu_to_le32(info->nsid),
1464                 .identify.cns           = NVME_ID_CNS_NS_CS_INDEP,
1465         };
1466         int ret;
1467
1468         id = kmalloc(sizeof(*id), GFP_KERNEL);
1469         if (!id)
1470                 return -ENOMEM;
1471
1472         ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
1473         if (!ret) {
1474                 info->anagrpid = id->anagrpid;
1475                 info->is_shared = id->nmic & NVME_NS_NMIC_SHARED;
1476                 info->is_readonly = id->nsattr & NVME_NS_ATTR_RO;
1477                 info->is_ready = id->nstat & NVME_NSTAT_NRDY;
1478         }
1479         kfree(id);
1480         return ret;
1481 }
1482
1483 static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
1484                 unsigned int dword11, void *buffer, size_t buflen, u32 *result)
1485 {
1486         union nvme_result res = { 0 };
1487         struct nvme_command c = { };
1488         int ret;
1489
1490         c.features.opcode = op;
1491         c.features.fid = cpu_to_le32(fid);
1492         c.features.dword11 = cpu_to_le32(dword11);
1493
1494         ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
1495                         buffer, buflen, NVME_QID_ANY, 0, 0);
1496         if (ret >= 0 && result)
1497                 *result = le32_to_cpu(res.u32);
1498         return ret;
1499 }
1500
1501 int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
1502                       unsigned int dword11, void *buffer, size_t buflen,
1503                       u32 *result)
1504 {
1505         return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
1506                              buflen, result);
1507 }
1508 EXPORT_SYMBOL_GPL(nvme_set_features);
1509
1510 int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
1511                       unsigned int dword11, void *buffer, size_t buflen,
1512                       u32 *result)
1513 {
1514         return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
1515                              buflen, result);
1516 }
1517 EXPORT_SYMBOL_GPL(nvme_get_features);
1518
1519 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
1520 {
1521         u32 q_count = (*count - 1) | ((*count - 1) << 16);
1522         u32 result;
1523         int status, nr_io_queues;
1524
1525         status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
1526                         &result);
1527         if (status < 0)
1528                 return status;
1529
1530         /*
1531          * Degraded controllers might return an error when setting the queue
1532          * count.  We still want to be able to bring them online and offer
1533          * access to the admin queue, as that might be only way to fix them up.
1534          */
1535         if (status > 0) {
1536                 dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
1537                 *count = 0;
1538         } else {
1539                 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
1540                 *count = min(*count, nr_io_queues);
1541         }
1542
1543         return 0;
1544 }
1545 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
1546
1547 #define NVME_AEN_SUPPORTED \
1548         (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \
1549          NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE)
1550
1551 static void nvme_enable_aen(struct nvme_ctrl *ctrl)
1552 {
1553         u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
1554         int status;
1555
1556         if (!supported_aens)
1557                 return;
1558
1559         status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens,
1560                         NULL, 0, &result);
1561         if (status)
1562                 dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
1563                          supported_aens);
1564
1565         queue_work(nvme_wq, &ctrl->async_event_work);
1566 }
1567
1568 static int nvme_ns_open(struct nvme_ns *ns)
1569 {
1570
1571         /* should never be called due to GENHD_FL_HIDDEN */
1572         if (WARN_ON_ONCE(nvme_ns_head_multipath(ns->head)))
1573                 goto fail;
1574         if (!nvme_get_ns(ns))
1575                 goto fail;
1576         if (!try_module_get(ns->ctrl->ops->module))
1577                 goto fail_put_ns;
1578
1579         return 0;
1580
1581 fail_put_ns:
1582         nvme_put_ns(ns);
1583 fail:
1584         return -ENXIO;
1585 }
1586
1587 static void nvme_ns_release(struct nvme_ns *ns)
1588 {
1589
1590         module_put(ns->ctrl->ops->module);
1591         nvme_put_ns(ns);
1592 }
1593
1594 static int nvme_open(struct block_device *bdev, fmode_t mode)
1595 {
1596         return nvme_ns_open(bdev->bd_disk->private_data);
1597 }
1598
1599 static void nvme_release(struct gendisk *disk, fmode_t mode)
1600 {
1601         nvme_ns_release(disk->private_data);
1602 }
1603
1604 int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1605 {
1606         /* some standard values */
1607         geo->heads = 1 << 6;
1608         geo->sectors = 1 << 5;
1609         geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1610         return 0;
1611 }
1612
1613 #ifdef CONFIG_BLK_DEV_INTEGRITY
1614 static void nvme_init_integrity(struct gendisk *disk, struct nvme_ns *ns,
1615                                 u32 max_integrity_segments)
1616 {
1617         struct blk_integrity integrity = { };
1618
1619         switch (ns->pi_type) {
1620         case NVME_NS_DPS_PI_TYPE3:
1621                 switch (ns->guard_type) {
1622                 case NVME_NVM_NS_16B_GUARD:
1623                         integrity.profile = &t10_pi_type3_crc;
1624                         integrity.tag_size = sizeof(u16) + sizeof(u32);
1625                         integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1626                         break;
1627                 case NVME_NVM_NS_64B_GUARD:
1628                         integrity.profile = &ext_pi_type3_crc64;
1629                         integrity.tag_size = sizeof(u16) + 6;
1630                         integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1631                         break;
1632                 default:
1633                         integrity.profile = NULL;
1634                         break;
1635                 }
1636                 break;
1637         case NVME_NS_DPS_PI_TYPE1:
1638         case NVME_NS_DPS_PI_TYPE2:
1639                 switch (ns->guard_type) {
1640                 case NVME_NVM_NS_16B_GUARD:
1641                         integrity.profile = &t10_pi_type1_crc;
1642                         integrity.tag_size = sizeof(u16);
1643                         integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1644                         break;
1645                 case NVME_NVM_NS_64B_GUARD:
1646                         integrity.profile = &ext_pi_type1_crc64;
1647                         integrity.tag_size = sizeof(u16);
1648                         integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1649                         break;
1650                 default:
1651                         integrity.profile = NULL;
1652                         break;
1653                 }
1654                 break;
1655         default:
1656                 integrity.profile = NULL;
1657                 break;
1658         }
1659
1660         integrity.tuple_size = ns->ms;
1661         blk_integrity_register(disk, &integrity);
1662         blk_queue_max_integrity_segments(disk->queue, max_integrity_segments);
1663 }
1664 #else
1665 static void nvme_init_integrity(struct gendisk *disk, struct nvme_ns *ns,
1666                                 u32 max_integrity_segments)
1667 {
1668 }
1669 #endif /* CONFIG_BLK_DEV_INTEGRITY */
1670
1671 static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns)
1672 {
1673         struct nvme_ctrl *ctrl = ns->ctrl;
1674         struct request_queue *queue = disk->queue;
1675         u32 size = queue_logical_block_size(queue);
1676
1677         if (ctrl->dmrsl && ctrl->dmrsl <= nvme_sect_to_lba(ns, UINT_MAX))
1678                 ctrl->max_discard_sectors = nvme_lba_to_sect(ns, ctrl->dmrsl);
1679
1680         if (ctrl->max_discard_sectors == 0) {
1681                 blk_queue_max_discard_sectors(queue, 0);
1682                 return;
1683         }
1684
1685         BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
1686                         NVME_DSM_MAX_RANGES);
1687
1688         queue->limits.discard_granularity = size;
1689
1690         /* If discard is already enabled, don't reset queue limits */
1691         if (queue->limits.max_discard_sectors)
1692                 return;
1693
1694         blk_queue_max_discard_sectors(queue, ctrl->max_discard_sectors);
1695         blk_queue_max_discard_segments(queue, ctrl->max_discard_segments);
1696
1697         if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
1698                 blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
1699 }
1700
1701 static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
1702 {
1703         return uuid_equal(&a->uuid, &b->uuid) &&
1704                 memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
1705                 memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0 &&
1706                 a->csi == b->csi;
1707 }
1708
1709 static int nvme_init_ms(struct nvme_ns *ns, struct nvme_id_ns *id)
1710 {
1711         bool first = id->dps & NVME_NS_DPS_PI_FIRST;
1712         unsigned lbaf = nvme_lbaf_index(id->flbas);
1713         struct nvme_ctrl *ctrl = ns->ctrl;
1714         struct nvme_command c = { };
1715         struct nvme_id_ns_nvm *nvm;
1716         int ret = 0;
1717         u32 elbaf;
1718
1719         ns->pi_size = 0;
1720         ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
1721         if (!(ctrl->ctratt & NVME_CTRL_ATTR_ELBAS)) {
1722                 ns->pi_size = sizeof(struct t10_pi_tuple);
1723                 ns->guard_type = NVME_NVM_NS_16B_GUARD;
1724                 goto set_pi;
1725         }
1726
1727         nvm = kzalloc(sizeof(*nvm), GFP_KERNEL);
1728         if (!nvm)
1729                 return -ENOMEM;
1730
1731         c.identify.opcode = nvme_admin_identify;
1732         c.identify.nsid = cpu_to_le32(ns->head->ns_id);
1733         c.identify.cns = NVME_ID_CNS_CS_NS;
1734         c.identify.csi = NVME_CSI_NVM;
1735
1736         ret = nvme_submit_sync_cmd(ns->ctrl->admin_q, &c, nvm, sizeof(*nvm));
1737         if (ret)
1738                 goto free_data;
1739
1740         elbaf = le32_to_cpu(nvm->elbaf[lbaf]);
1741
1742         /* no support for storage tag formats right now */
1743         if (nvme_elbaf_sts(elbaf))
1744                 goto free_data;
1745
1746         ns->guard_type = nvme_elbaf_guard_type(elbaf);
1747         switch (ns->guard_type) {
1748         case NVME_NVM_NS_64B_GUARD:
1749                 ns->pi_size = sizeof(struct crc64_pi_tuple);
1750                 break;
1751         case NVME_NVM_NS_16B_GUARD:
1752                 ns->pi_size = sizeof(struct t10_pi_tuple);
1753                 break;
1754         default:
1755                 break;
1756         }
1757
1758 free_data:
1759         kfree(nvm);
1760 set_pi:
1761         if (ns->pi_size && (first || ns->ms == ns->pi_size))
1762                 ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
1763         else
1764                 ns->pi_type = 0;
1765
1766         return ret;
1767 }
1768
1769 static void nvme_configure_metadata(struct nvme_ns *ns, struct nvme_id_ns *id)
1770 {
1771         struct nvme_ctrl *ctrl = ns->ctrl;
1772
1773         if (nvme_init_ms(ns, id))
1774                 return;
1775
1776         ns->features &= ~(NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS);
1777         if (!ns->ms || !(ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
1778                 return;
1779
1780         if (ctrl->ops->flags & NVME_F_FABRICS) {
1781                 /*
1782                  * The NVMe over Fabrics specification only supports metadata as
1783                  * part of the extended data LBA.  We rely on HCA/HBA support to
1784                  * remap the separate metadata buffer from the block layer.
1785                  */
1786                 if (WARN_ON_ONCE(!(id->flbas & NVME_NS_FLBAS_META_EXT)))
1787                         return;
1788
1789                 ns->features |= NVME_NS_EXT_LBAS;
1790
1791                 /*
1792                  * The current fabrics transport drivers support namespace
1793                  * metadata formats only if nvme_ns_has_pi() returns true.
1794                  * Suppress support for all other formats so the namespace will
1795                  * have a 0 capacity and not be usable through the block stack.
1796                  *
1797                  * Note, this check will need to be modified if any drivers
1798                  * gain the ability to use other metadata formats.
1799                  */
1800                 if (ctrl->max_integrity_segments && nvme_ns_has_pi(ns))
1801                         ns->features |= NVME_NS_METADATA_SUPPORTED;
1802         } else {
1803                 /*
1804                  * For PCIe controllers, we can't easily remap the separate
1805                  * metadata buffer from the block layer and thus require a
1806                  * separate metadata buffer for block layer metadata/PI support.
1807                  * We allow extended LBAs for the passthrough interface, though.
1808                  */
1809                 if (id->flbas & NVME_NS_FLBAS_META_EXT)
1810                         ns->features |= NVME_NS_EXT_LBAS;
1811                 else
1812                         ns->features |= NVME_NS_METADATA_SUPPORTED;
1813         }
1814 }
1815
1816 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
1817                 struct request_queue *q)
1818 {
1819         bool vwc = ctrl->vwc & NVME_CTRL_VWC_PRESENT;
1820
1821         if (ctrl->max_hw_sectors) {
1822                 u32 max_segments =
1823                         (ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> 9)) + 1;
1824
1825                 max_segments = min_not_zero(max_segments, ctrl->max_segments);
1826                 blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
1827                 blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
1828         }
1829         blk_queue_virt_boundary(q, NVME_CTRL_PAGE_SIZE - 1);
1830         blk_queue_dma_alignment(q, 3);
1831         blk_queue_write_cache(q, vwc, vwc);
1832 }
1833
1834 static void nvme_update_disk_info(struct gendisk *disk,
1835                 struct nvme_ns *ns, struct nvme_id_ns *id)
1836 {
1837         sector_t capacity = nvme_lba_to_sect(ns, le64_to_cpu(id->nsze));
1838         unsigned short bs = 1 << ns->lba_shift;
1839         u32 atomic_bs, phys_bs, io_opt = 0;
1840
1841         /*
1842          * The block layer can't support LBA sizes larger than the page size
1843          * yet, so catch this early and don't allow block I/O.
1844          */
1845         if (ns->lba_shift > PAGE_SHIFT) {
1846                 capacity = 0;
1847                 bs = (1 << 9);
1848         }
1849
1850         blk_integrity_unregister(disk);
1851
1852         atomic_bs = phys_bs = bs;
1853         if (id->nabo == 0) {
1854                 /*
1855                  * Bit 1 indicates whether NAWUPF is defined for this namespace
1856                  * and whether it should be used instead of AWUPF. If NAWUPF ==
1857                  * 0 then AWUPF must be used instead.
1858                  */
1859                 if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf)
1860                         atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
1861                 else
1862                         atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
1863         }
1864
1865         if (id->nsfeat & NVME_NS_FEAT_IO_OPT) {
1866                 /* NPWG = Namespace Preferred Write Granularity */
1867                 phys_bs = bs * (1 + le16_to_cpu(id->npwg));
1868                 /* NOWS = Namespace Optimal Write Size */
1869                 io_opt = bs * (1 + le16_to_cpu(id->nows));
1870         }
1871
1872         blk_queue_logical_block_size(disk->queue, bs);
1873         /*
1874          * Linux filesystems assume writing a single physical block is
1875          * an atomic operation. Hence limit the physical block size to the
1876          * value of the Atomic Write Unit Power Fail parameter.
1877          */
1878         blk_queue_physical_block_size(disk->queue, min(phys_bs, atomic_bs));
1879         blk_queue_io_min(disk->queue, phys_bs);
1880         blk_queue_io_opt(disk->queue, io_opt);
1881
1882         /*
1883          * Register a metadata profile for PI, or the plain non-integrity NVMe
1884          * metadata masquerading as Type 0 if supported, otherwise reject block
1885          * I/O to namespaces with metadata except when the namespace supports
1886          * PI, as it can strip/insert in that case.
1887          */
1888         if (ns->ms) {
1889                 if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
1890                     (ns->features & NVME_NS_METADATA_SUPPORTED))
1891                         nvme_init_integrity(disk, ns,
1892                                             ns->ctrl->max_integrity_segments);
1893                 else if (!nvme_ns_has_pi(ns))
1894                         capacity = 0;
1895         }
1896
1897         set_capacity_and_notify(disk, capacity);
1898
1899         nvme_config_discard(disk, ns);
1900         blk_queue_max_write_zeroes_sectors(disk->queue,
1901                                            ns->ctrl->max_zeroes_sectors);
1902 }
1903
1904 static bool nvme_ns_is_readonly(struct nvme_ns *ns, struct nvme_ns_info *info)
1905 {
1906         return info->is_readonly || test_bit(NVME_NS_FORCE_RO, &ns->flags);
1907 }
1908
1909 static inline bool nvme_first_scan(struct gendisk *disk)
1910 {
1911         /* nvme_alloc_ns() scans the disk prior to adding it */
1912         return !disk_live(disk);
1913 }
1914
1915 static void nvme_set_chunk_sectors(struct nvme_ns *ns, struct nvme_id_ns *id)
1916 {
1917         struct nvme_ctrl *ctrl = ns->ctrl;
1918         u32 iob;
1919
1920         if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
1921             is_power_of_2(ctrl->max_hw_sectors))
1922                 iob = ctrl->max_hw_sectors;
1923         else
1924                 iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob));
1925
1926         if (!iob)
1927                 return;
1928
1929         if (!is_power_of_2(iob)) {
1930                 if (nvme_first_scan(ns->disk))
1931                         pr_warn("%s: ignoring unaligned IO boundary:%u\n",
1932                                 ns->disk->disk_name, iob);
1933                 return;
1934         }
1935
1936         if (blk_queue_is_zoned(ns->disk->queue)) {
1937                 if (nvme_first_scan(ns->disk))
1938                         pr_warn("%s: ignoring zoned namespace IO boundary\n",
1939                                 ns->disk->disk_name);
1940                 return;
1941         }
1942
1943         blk_queue_chunk_sectors(ns->queue, iob);
1944 }
1945
1946 static int nvme_update_ns_info_generic(struct nvme_ns *ns,
1947                 struct nvme_ns_info *info)
1948 {
1949         blk_mq_freeze_queue(ns->disk->queue);
1950         nvme_set_queue_limits(ns->ctrl, ns->queue);
1951         set_disk_ro(ns->disk, nvme_ns_is_readonly(ns, info));
1952         blk_mq_unfreeze_queue(ns->disk->queue);
1953
1954         if (nvme_ns_head_multipath(ns->head)) {
1955                 blk_mq_freeze_queue(ns->head->disk->queue);
1956                 set_disk_ro(ns->head->disk, nvme_ns_is_readonly(ns, info));
1957                 nvme_mpath_revalidate_paths(ns);
1958                 blk_stack_limits(&ns->head->disk->queue->limits,
1959                                  &ns->queue->limits, 0);
1960                 ns->head->disk->flags |= GENHD_FL_HIDDEN;
1961                 blk_mq_unfreeze_queue(ns->head->disk->queue);
1962         }
1963
1964         /* Hide the block-interface for these devices */
1965         ns->disk->flags |= GENHD_FL_HIDDEN;
1966         set_bit(NVME_NS_READY, &ns->flags);
1967
1968         return 0;
1969 }
1970
1971 static int nvme_update_ns_info_block(struct nvme_ns *ns,
1972                 struct nvme_ns_info *info)
1973 {
1974         struct nvme_id_ns *id;
1975         unsigned lbaf;
1976         int ret;
1977
1978         ret = nvme_identify_ns(ns->ctrl, info->nsid, &id);
1979         if (ret)
1980                 return ret;
1981
1982         blk_mq_freeze_queue(ns->disk->queue);
1983         lbaf = nvme_lbaf_index(id->flbas);
1984         ns->lba_shift = id->lbaf[lbaf].ds;
1985         nvme_set_queue_limits(ns->ctrl, ns->queue);
1986
1987         nvme_configure_metadata(ns, id);
1988         nvme_set_chunk_sectors(ns, id);
1989         nvme_update_disk_info(ns->disk, ns, id);
1990
1991         if (ns->head->ids.csi == NVME_CSI_ZNS) {
1992                 ret = nvme_update_zone_info(ns, lbaf);
1993                 if (ret) {
1994                         blk_mq_unfreeze_queue(ns->disk->queue);
1995                         goto out;
1996                 }
1997         }
1998
1999         /*
2000          * Only set the DEAC bit if the device guarantees that reads from
2001          * deallocated data return zeroes.  While the DEAC bit does not
2002          * require that, it must be a no-op if reads from deallocated data
2003          * do not return zeroes.
2004          */
2005         if ((id->dlfeat & 0x7) == 0x1 && (id->dlfeat & (1 << 3)))
2006                 ns->features |= NVME_NS_DEAC;
2007         set_disk_ro(ns->disk, nvme_ns_is_readonly(ns, info));
2008         set_bit(NVME_NS_READY, &ns->flags);
2009         blk_mq_unfreeze_queue(ns->disk->queue);
2010
2011         if (blk_queue_is_zoned(ns->queue)) {
2012                 ret = nvme_revalidate_zones(ns);
2013                 if (ret && !nvme_first_scan(ns->disk))
2014                         goto out;
2015         }
2016
2017         if (nvme_ns_head_multipath(ns->head)) {
2018                 blk_mq_freeze_queue(ns->head->disk->queue);
2019                 nvme_update_disk_info(ns->head->disk, ns, id);
2020                 set_disk_ro(ns->head->disk, nvme_ns_is_readonly(ns, info));
2021                 nvme_mpath_revalidate_paths(ns);
2022                 blk_stack_limits(&ns->head->disk->queue->limits,
2023                                  &ns->queue->limits, 0);
2024                 disk_update_readahead(ns->head->disk);
2025                 blk_mq_unfreeze_queue(ns->head->disk->queue);
2026         }
2027
2028         ret = 0;
2029 out:
2030         /*
2031          * If probing fails due an unsupported feature, hide the block device,
2032          * but still allow other access.
2033          */
2034         if (ret == -ENODEV) {
2035                 ns->disk->flags |= GENHD_FL_HIDDEN;
2036                 set_bit(NVME_NS_READY, &ns->flags);
2037                 ret = 0;
2038         }
2039         kfree(id);
2040         return ret;
2041 }
2042
2043 static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_ns_info *info)
2044 {
2045         switch (info->ids.csi) {
2046         case NVME_CSI_ZNS:
2047                 if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
2048                         dev_info(ns->ctrl->device,
2049         "block device for nsid %u not supported without CONFIG_BLK_DEV_ZONED\n",
2050                                 info->nsid);
2051                         return nvme_update_ns_info_generic(ns, info);
2052                 }
2053                 return nvme_update_ns_info_block(ns, info);
2054         case NVME_CSI_NVM:
2055                 return nvme_update_ns_info_block(ns, info);
2056         default:
2057                 dev_info(ns->ctrl->device,
2058                         "block device for nsid %u not supported (csi %u)\n",
2059                         info->nsid, info->ids.csi);
2060                 return nvme_update_ns_info_generic(ns, info);
2061         }
2062 }
2063
2064 #ifdef CONFIG_BLK_SED_OPAL
2065 static int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
2066                 bool send)
2067 {
2068         struct nvme_ctrl *ctrl = data;
2069         struct nvme_command cmd = { };
2070
2071         if (send)
2072                 cmd.common.opcode = nvme_admin_security_send;
2073         else
2074                 cmd.common.opcode = nvme_admin_security_recv;
2075         cmd.common.nsid = 0;
2076         cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
2077         cmd.common.cdw11 = cpu_to_le32(len);
2078
2079         return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
2080                         NVME_QID_ANY, 1, 0);
2081 }
2082
2083 static void nvme_configure_opal(struct nvme_ctrl *ctrl, bool was_suspended)
2084 {
2085         if (ctrl->oacs & NVME_CTRL_OACS_SEC_SUPP) {
2086                 if (!ctrl->opal_dev)
2087                         ctrl->opal_dev = init_opal_dev(ctrl, &nvme_sec_submit);
2088                 else if (was_suspended)
2089                         opal_unlock_from_suspend(ctrl->opal_dev);
2090         } else {
2091                 free_opal_dev(ctrl->opal_dev);
2092                 ctrl->opal_dev = NULL;
2093         }
2094 }
2095 #else
2096 static void nvme_configure_opal(struct nvme_ctrl *ctrl, bool was_suspended)
2097 {
2098 }
2099 #endif /* CONFIG_BLK_SED_OPAL */
2100
2101 #ifdef CONFIG_BLK_DEV_ZONED
2102 static int nvme_report_zones(struct gendisk *disk, sector_t sector,
2103                 unsigned int nr_zones, report_zones_cb cb, void *data)
2104 {
2105         return nvme_ns_report_zones(disk->private_data, sector, nr_zones, cb,
2106                         data);
2107 }
2108 #else
2109 #define nvme_report_zones       NULL
2110 #endif /* CONFIG_BLK_DEV_ZONED */
2111
2112 static const struct block_device_operations nvme_bdev_ops = {
2113         .owner          = THIS_MODULE,
2114         .ioctl          = nvme_ioctl,
2115         .compat_ioctl   = blkdev_compat_ptr_ioctl,
2116         .open           = nvme_open,
2117         .release        = nvme_release,
2118         .getgeo         = nvme_getgeo,
2119         .report_zones   = nvme_report_zones,
2120         .pr_ops         = &nvme_pr_ops,
2121 };
2122
2123 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u32 mask, u32 val,
2124                 u32 timeout, const char *op)
2125 {
2126         unsigned long timeout_jiffies = jiffies + timeout * HZ;
2127         u32 csts;
2128         int ret;
2129
2130         while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2131                 if (csts == ~0)
2132                         return -ENODEV;
2133                 if ((csts & mask) == val)
2134                         break;
2135
2136                 usleep_range(1000, 2000);
2137                 if (fatal_signal_pending(current))
2138                         return -EINTR;
2139                 if (time_after(jiffies, timeout_jiffies)) {
2140                         dev_err(ctrl->device,
2141                                 "Device not ready; aborting %s, CSTS=0x%x\n",
2142                                 op, csts);
2143                         return -ENODEV;
2144                 }
2145         }
2146
2147         return ret;
2148 }
2149
2150 int nvme_disable_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
2151 {
2152         int ret;
2153
2154         ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2155         if (shutdown)
2156                 ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
2157         else
2158                 ctrl->ctrl_config &= ~NVME_CC_ENABLE;
2159
2160         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2161         if (ret)
2162                 return ret;
2163
2164         if (shutdown) {
2165                 return nvme_wait_ready(ctrl, NVME_CSTS_SHST_MASK,
2166                                        NVME_CSTS_SHST_CMPLT,
2167                                        ctrl->shutdown_timeout, "shutdown");
2168         }
2169         if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
2170                 msleep(NVME_QUIRK_DELAY_AMOUNT);
2171         return nvme_wait_ready(ctrl, NVME_CSTS_RDY, 0,
2172                                (NVME_CAP_TIMEOUT(ctrl->cap) + 1) / 2, "reset");
2173 }
2174 EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
2175
2176 int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
2177 {
2178         unsigned dev_page_min;
2179         u32 timeout;
2180         int ret;
2181
2182         ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2183         if (ret) {
2184                 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2185                 return ret;
2186         }
2187         dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2188
2189         if (NVME_CTRL_PAGE_SHIFT < dev_page_min) {
2190                 dev_err(ctrl->device,
2191                         "Minimum device page size %u too large for host (%u)\n",
2192                         1 << dev_page_min, 1 << NVME_CTRL_PAGE_SHIFT);
2193                 return -ENODEV;
2194         }
2195
2196         if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI)
2197                 ctrl->ctrl_config = NVME_CC_CSS_CSI;
2198         else
2199                 ctrl->ctrl_config = NVME_CC_CSS_NVM;
2200
2201         if (ctrl->cap & NVME_CAP_CRMS_CRWMS) {
2202                 u32 crto;
2203
2204                 ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CRTO, &crto);
2205                 if (ret) {
2206                         dev_err(ctrl->device, "Reading CRTO failed (%d)\n",
2207                                 ret);
2208                         return ret;
2209                 }
2210
2211                 if (ctrl->cap & NVME_CAP_CRMS_CRIMS) {
2212                         ctrl->ctrl_config |= NVME_CC_CRIME;
2213                         timeout = NVME_CRTO_CRIMT(crto);
2214                 } else {
2215                         timeout = NVME_CRTO_CRWMT(crto);
2216                 }
2217         } else {
2218                 timeout = NVME_CAP_TIMEOUT(ctrl->cap);
2219         }
2220
2221         ctrl->ctrl_config |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
2222         ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
2223         ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
2224         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2225         if (ret)
2226                 return ret;
2227
2228         /* Flush write to device (required if transport is PCI) */
2229         ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CC, &ctrl->ctrl_config);
2230         if (ret)
2231                 return ret;
2232
2233         ctrl->ctrl_config |= NVME_CC_ENABLE;
2234         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2235         if (ret)
2236                 return ret;
2237         return nvme_wait_ready(ctrl, NVME_CSTS_RDY, NVME_CSTS_RDY,
2238                                (timeout + 1) / 2, "initialisation");
2239 }
2240 EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
2241
2242 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
2243 {
2244         __le64 ts;
2245         int ret;
2246
2247         if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
2248                 return 0;
2249
2250         ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
2251         ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
2252                         NULL);
2253         if (ret)
2254                 dev_warn_once(ctrl->device,
2255                         "could not set timestamp (%d)\n", ret);
2256         return ret;
2257 }
2258
2259 static int nvme_configure_host_options(struct nvme_ctrl *ctrl)
2260 {
2261         struct nvme_feat_host_behavior *host;
2262         u8 acre = 0, lbafee = 0;
2263         int ret;
2264
2265         /* Don't bother enabling the feature if retry delay is not reported */
2266         if (ctrl->crdt[0])
2267                 acre = NVME_ENABLE_ACRE;
2268         if (ctrl->ctratt & NVME_CTRL_ATTR_ELBAS)
2269                 lbafee = NVME_ENABLE_LBAFEE;
2270
2271         if (!acre && !lbafee)
2272                 return 0;
2273
2274         host = kzalloc(sizeof(*host), GFP_KERNEL);
2275         if (!host)
2276                 return 0;
2277
2278         host->acre = acre;
2279         host->lbafee = lbafee;
2280         ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
2281                                 host, sizeof(*host), NULL);
2282         kfree(host);
2283         return ret;
2284 }
2285
2286 /*
2287  * The function checks whether the given total (exlat + enlat) latency of
2288  * a power state allows the latter to be used as an APST transition target.
2289  * It does so by comparing the latency to the primary and secondary latency
2290  * tolerances defined by module params. If there's a match, the corresponding
2291  * timeout value is returned and the matching tolerance index (1 or 2) is
2292  * reported.
2293  */
2294 static bool nvme_apst_get_transition_time(u64 total_latency,
2295                 u64 *transition_time, unsigned *last_index)
2296 {
2297         if (total_latency <= apst_primary_latency_tol_us) {
2298                 if (*last_index == 1)
2299                         return false;
2300                 *last_index = 1;
2301                 *transition_time = apst_primary_timeout_ms;
2302                 return true;
2303         }
2304         if (apst_secondary_timeout_ms &&
2305                 total_latency <= apst_secondary_latency_tol_us) {
2306                 if (*last_index <= 2)
2307                         return false;
2308                 *last_index = 2;
2309                 *transition_time = apst_secondary_timeout_ms;
2310                 return true;
2311         }
2312         return false;
2313 }
2314
2315 /*
2316  * APST (Autonomous Power State Transition) lets us program a table of power
2317  * state transitions that the controller will perform automatically.
2318  *
2319  * Depending on module params, one of the two supported techniques will be used:
2320  *
2321  * - If the parameters provide explicit timeouts and tolerances, they will be
2322  *   used to build a table with up to 2 non-operational states to transition to.
2323  *   The default parameter values were selected based on the values used by
2324  *   Microsoft's and Intel's NVMe drivers. Yet, since we don't implement dynamic
2325  *   regeneration of the APST table in the event of switching between external
2326  *   and battery power, the timeouts and tolerances reflect a compromise
2327  *   between values used by Microsoft for AC and battery scenarios.
2328  * - If not, we'll configure the table with a simple heuristic: we are willing
2329  *   to spend at most 2% of the time transitioning between power states.
2330  *   Therefore, when running in any given state, we will enter the next
2331  *   lower-power non-operational state after waiting 50 * (enlat + exlat)
2332  *   microseconds, as long as that state's exit latency is under the requested
2333  *   maximum latency.
2334  *
2335  * We will not autonomously enter any non-operational state for which the total
2336  * latency exceeds ps_max_latency_us.
2337  *
2338  * Users can set ps_max_latency_us to zero to turn off APST.
2339  */
2340 static int nvme_configure_apst(struct nvme_ctrl *ctrl)
2341 {
2342         struct nvme_feat_auto_pst *table;
2343         unsigned apste = 0;
2344         u64 max_lat_us = 0;
2345         __le64 target = 0;
2346         int max_ps = -1;
2347         int state;
2348         int ret;
2349         unsigned last_lt_index = UINT_MAX;
2350
2351         /*
2352          * If APST isn't supported or if we haven't been initialized yet,
2353          * then don't do anything.
2354          */
2355         if (!ctrl->apsta)
2356                 return 0;
2357
2358         if (ctrl->npss > 31) {
2359                 dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
2360                 return 0;
2361         }
2362
2363         table = kzalloc(sizeof(*table), GFP_KERNEL);
2364         if (!table)
2365                 return 0;
2366
2367         if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
2368                 /* Turn off APST. */
2369                 dev_dbg(ctrl->device, "APST disabled\n");
2370                 goto done;
2371         }
2372
2373         /*
2374          * Walk through all states from lowest- to highest-power.
2375          * According to the spec, lower-numbered states use more power.  NPSS,
2376          * despite the name, is the index of the lowest-power state, not the
2377          * number of states.
2378          */
2379         for (state = (int)ctrl->npss; state >= 0; state--) {
2380                 u64 total_latency_us, exit_latency_us, transition_ms;
2381
2382                 if (target)
2383                         table->entries[state] = target;
2384
2385                 /*
2386                  * Don't allow transitions to the deepest state if it's quirked
2387                  * off.
2388                  */
2389                 if (state == ctrl->npss &&
2390                     (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
2391                         continue;
2392
2393                 /*
2394                  * Is this state a useful non-operational state for higher-power
2395                  * states to autonomously transition to?
2396                  */
2397                 if (!(ctrl->psd[state].flags & NVME_PS_FLAGS_NON_OP_STATE))
2398                         continue;
2399
2400                 exit_latency_us = (u64)le32_to_cpu(ctrl->psd[state].exit_lat);
2401                 if (exit_latency_us > ctrl->ps_max_latency_us)
2402                         continue;
2403
2404                 total_latency_us = exit_latency_us +
2405                         le32_to_cpu(ctrl->psd[state].entry_lat);
2406
2407                 /*
2408                  * This state is good. It can be used as the APST idle target
2409                  * for higher power states.
2410                  */
2411                 if (apst_primary_timeout_ms && apst_primary_latency_tol_us) {
2412                         if (!nvme_apst_get_transition_time(total_latency_us,
2413                                         &transition_ms, &last_lt_index))
2414                                 continue;
2415                 } else {
2416                         transition_ms = total_latency_us + 19;
2417                         do_div(transition_ms, 20);
2418                         if (transition_ms > (1 << 24) - 1)
2419                                 transition_ms = (1 << 24) - 1;
2420                 }
2421
2422                 target = cpu_to_le64((state << 3) | (transition_ms << 8));
2423                 if (max_ps == -1)
2424                         max_ps = state;
2425                 if (total_latency_us > max_lat_us)
2426                         max_lat_us = total_latency_us;
2427         }
2428
2429         if (max_ps == -1)
2430                 dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
2431         else
2432                 dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
2433                         max_ps, max_lat_us, (int)sizeof(*table), table);
2434         apste = 1;
2435
2436 done:
2437         ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
2438                                 table, sizeof(*table), NULL);
2439         if (ret)
2440                 dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
2441         kfree(table);
2442         return ret;
2443 }
2444
2445 static void nvme_set_latency_tolerance(struct device *dev, s32 val)
2446 {
2447         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2448         u64 latency;
2449
2450         switch (val) {
2451         case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
2452         case PM_QOS_LATENCY_ANY:
2453                 latency = U64_MAX;
2454                 break;
2455
2456         default:
2457                 latency = val;
2458         }
2459
2460         if (ctrl->ps_max_latency_us != latency) {
2461                 ctrl->ps_max_latency_us = latency;
2462                 if (ctrl->state == NVME_CTRL_LIVE)
2463                         nvme_configure_apst(ctrl);
2464         }
2465 }
2466
2467 struct nvme_core_quirk_entry {
2468         /*
2469          * NVMe model and firmware strings are padded with spaces.  For
2470          * simplicity, strings in the quirk table are padded with NULLs
2471          * instead.
2472          */
2473         u16 vid;
2474         const char *mn;
2475         const char *fr;
2476         unsigned long quirks;
2477 };
2478
2479 static const struct nvme_core_quirk_entry core_quirks[] = {
2480         {
2481                 /*
2482                  * This Toshiba device seems to die using any APST states.  See:
2483                  * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
2484                  */
2485                 .vid = 0x1179,
2486                 .mn = "THNSF5256GPUK TOSHIBA",
2487                 .quirks = NVME_QUIRK_NO_APST,
2488         },
2489         {
2490                 /*
2491                  * This LiteON CL1-3D*-Q11 firmware version has a race
2492                  * condition associated with actions related to suspend to idle
2493                  * LiteON has resolved the problem in future firmware
2494                  */
2495                 .vid = 0x14a4,
2496                 .fr = "22301111",
2497                 .quirks = NVME_QUIRK_SIMPLE_SUSPEND,
2498         },
2499         {
2500                 /*
2501                  * This Kioxia CD6-V Series / HPE PE8030 device times out and
2502                  * aborts I/O during any load, but more easily reproducible
2503                  * with discards (fstrim).
2504                  *
2505                  * The device is left in a state where it is also not possible
2506                  * to use "nvme set-feature" to disable APST, but booting with
2507                  * nvme_core.default_ps_max_latency=0 works.
2508                  */
2509                 .vid = 0x1e0f,
2510                 .mn = "KCD6XVUL6T40",
2511                 .quirks = NVME_QUIRK_NO_APST,
2512         },
2513         {
2514                 /*
2515                  * The external Samsung X5 SSD fails initialization without a
2516                  * delay before checking if it is ready and has a whole set of
2517                  * other problems.  To make this even more interesting, it
2518                  * shares the PCI ID with internal Samsung 970 Evo Plus that
2519                  * does not need or want these quirks.
2520                  */
2521                 .vid = 0x144d,
2522                 .mn = "Samsung Portable SSD X5",
2523                 .quirks = NVME_QUIRK_DELAY_BEFORE_CHK_RDY |
2524                           NVME_QUIRK_NO_DEEPEST_PS |
2525                           NVME_QUIRK_IGNORE_DEV_SUBNQN,
2526         }
2527 };
2528
2529 /* match is null-terminated but idstr is space-padded. */
2530 static bool string_matches(const char *idstr, const char *match, size_t len)
2531 {
2532         size_t matchlen;
2533
2534         if (!match)
2535                 return true;
2536
2537         matchlen = strlen(match);
2538         WARN_ON_ONCE(matchlen > len);
2539
2540         if (memcmp(idstr, match, matchlen))
2541                 return false;
2542
2543         for (; matchlen < len; matchlen++)
2544                 if (idstr[matchlen] != ' ')
2545                         return false;
2546
2547         return true;
2548 }
2549
2550 static bool quirk_matches(const struct nvme_id_ctrl *id,
2551                           const struct nvme_core_quirk_entry *q)
2552 {
2553         return q->vid == le16_to_cpu(id->vid) &&
2554                 string_matches(id->mn, q->mn, sizeof(id->mn)) &&
2555                 string_matches(id->fr, q->fr, sizeof(id->fr));
2556 }
2557
2558 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
2559                 struct nvme_id_ctrl *id)
2560 {
2561         size_t nqnlen;
2562         int off;
2563
2564         if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) {
2565                 nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
2566                 if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
2567                         strscpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2568                         return;
2569                 }
2570
2571                 if (ctrl->vs >= NVME_VS(1, 2, 1))
2572                         dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
2573         }
2574
2575         /*
2576          * Generate a "fake" NQN similar to the one in Section 4.5 of the NVMe
2577          * Base Specification 2.0.  It is slightly different from the format
2578          * specified there due to historic reasons, and we can't change it now.
2579          */
2580         off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
2581                         "nqn.2014.08.org.nvmexpress:%04x%04x",
2582                         le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
2583         memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
2584         off += sizeof(id->sn);
2585         memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
2586         off += sizeof(id->mn);
2587         memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
2588 }
2589
2590 static void nvme_release_subsystem(struct device *dev)
2591 {
2592         struct nvme_subsystem *subsys =
2593                 container_of(dev, struct nvme_subsystem, dev);
2594
2595         if (subsys->instance >= 0)
2596                 ida_free(&nvme_instance_ida, subsys->instance);
2597         kfree(subsys);
2598 }
2599
2600 static void nvme_destroy_subsystem(struct kref *ref)
2601 {
2602         struct nvme_subsystem *subsys =
2603                         container_of(ref, struct nvme_subsystem, ref);
2604
2605         mutex_lock(&nvme_subsystems_lock);
2606         list_del(&subsys->entry);
2607         mutex_unlock(&nvme_subsystems_lock);
2608
2609         ida_destroy(&subsys->ns_ida);
2610         device_del(&subsys->dev);
2611         put_device(&subsys->dev);
2612 }
2613
2614 static void nvme_put_subsystem(struct nvme_subsystem *subsys)
2615 {
2616         kref_put(&subsys->ref, nvme_destroy_subsystem);
2617 }
2618
2619 static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
2620 {
2621         struct nvme_subsystem *subsys;
2622
2623         lockdep_assert_held(&nvme_subsystems_lock);
2624
2625         /*
2626          * Fail matches for discovery subsystems. This results
2627          * in each discovery controller bound to a unique subsystem.
2628          * This avoids issues with validating controller values
2629          * that can only be true when there is a single unique subsystem.
2630          * There may be multiple and completely independent entities
2631          * that provide discovery controllers.
2632          */
2633         if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME))
2634                 return NULL;
2635
2636         list_for_each_entry(subsys, &nvme_subsystems, entry) {
2637                 if (strcmp(subsys->subnqn, subsysnqn))
2638                         continue;
2639                 if (!kref_get_unless_zero(&subsys->ref))
2640                         continue;
2641                 return subsys;
2642         }
2643
2644         return NULL;
2645 }
2646
2647 #define SUBSYS_ATTR_RO(_name, _mode, _show)                     \
2648         struct device_attribute subsys_attr_##_name = \
2649                 __ATTR(_name, _mode, _show, NULL)
2650
2651 static ssize_t nvme_subsys_show_nqn(struct device *dev,
2652                                     struct device_attribute *attr,
2653                                     char *buf)
2654 {
2655         struct nvme_subsystem *subsys =
2656                 container_of(dev, struct nvme_subsystem, dev);
2657
2658         return sysfs_emit(buf, "%s\n", subsys->subnqn);
2659 }
2660 static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn);
2661
2662 static ssize_t nvme_subsys_show_type(struct device *dev,
2663                                     struct device_attribute *attr,
2664                                     char *buf)
2665 {
2666         struct nvme_subsystem *subsys =
2667                 container_of(dev, struct nvme_subsystem, dev);
2668
2669         switch (subsys->subtype) {
2670         case NVME_NQN_DISC:
2671                 return sysfs_emit(buf, "discovery\n");
2672         case NVME_NQN_NVME:
2673                 return sysfs_emit(buf, "nvm\n");
2674         default:
2675                 return sysfs_emit(buf, "reserved\n");
2676         }
2677 }
2678 static SUBSYS_ATTR_RO(subsystype, S_IRUGO, nvme_subsys_show_type);
2679
2680 #define nvme_subsys_show_str_function(field)                            \
2681 static ssize_t subsys_##field##_show(struct device *dev,                \
2682                             struct device_attribute *attr, char *buf)   \
2683 {                                                                       \
2684         struct nvme_subsystem *subsys =                                 \
2685                 container_of(dev, struct nvme_subsystem, dev);          \
2686         return sysfs_emit(buf, "%.*s\n",                                \
2687                            (int)sizeof(subsys->field), subsys->field);  \
2688 }                                                                       \
2689 static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show);
2690
2691 nvme_subsys_show_str_function(model);
2692 nvme_subsys_show_str_function(serial);
2693 nvme_subsys_show_str_function(firmware_rev);
2694
2695 static struct attribute *nvme_subsys_attrs[] = {
2696         &subsys_attr_model.attr,
2697         &subsys_attr_serial.attr,
2698         &subsys_attr_firmware_rev.attr,
2699         &subsys_attr_subsysnqn.attr,
2700         &subsys_attr_subsystype.attr,
2701 #ifdef CONFIG_NVME_MULTIPATH
2702         &subsys_attr_iopolicy.attr,
2703 #endif
2704         NULL,
2705 };
2706
2707 static const struct attribute_group nvme_subsys_attrs_group = {
2708         .attrs = nvme_subsys_attrs,
2709 };
2710
2711 static const struct attribute_group *nvme_subsys_attrs_groups[] = {
2712         &nvme_subsys_attrs_group,
2713         NULL,
2714 };
2715
2716 static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)
2717 {
2718         return ctrl->opts && ctrl->opts->discovery_nqn;
2719 }
2720
2721 static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
2722                 struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2723 {
2724         struct nvme_ctrl *tmp;
2725
2726         lockdep_assert_held(&nvme_subsystems_lock);
2727
2728         list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) {
2729                 if (nvme_state_terminal(tmp))
2730                         continue;
2731
2732                 if (tmp->cntlid == ctrl->cntlid) {
2733                         dev_err(ctrl->device,
2734                                 "Duplicate cntlid %u with %s, subsys %s, rejecting\n",
2735                                 ctrl->cntlid, dev_name(tmp->device),
2736                                 subsys->subnqn);
2737                         return false;
2738                 }
2739
2740                 if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
2741                     nvme_discovery_ctrl(ctrl))
2742                         continue;
2743
2744                 dev_err(ctrl->device,
2745                         "Subsystem does not support multiple controllers\n");
2746                 return false;
2747         }
2748
2749         return true;
2750 }
2751
2752 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2753 {
2754         struct nvme_subsystem *subsys, *found;
2755         int ret;
2756
2757         subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
2758         if (!subsys)
2759                 return -ENOMEM;
2760
2761         subsys->instance = -1;
2762         mutex_init(&subsys->lock);
2763         kref_init(&subsys->ref);
2764         INIT_LIST_HEAD(&subsys->ctrls);
2765         INIT_LIST_HEAD(&subsys->nsheads);
2766         nvme_init_subnqn(subsys, ctrl, id);
2767         memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
2768         memcpy(subsys->model, id->mn, sizeof(subsys->model));
2769         subsys->vendor_id = le16_to_cpu(id->vid);
2770         subsys->cmic = id->cmic;
2771
2772         /* Versions prior to 1.4 don't necessarily report a valid type */
2773         if (id->cntrltype == NVME_CTRL_DISC ||
2774             !strcmp(subsys->subnqn, NVME_DISC_SUBSYS_NAME))
2775                 subsys->subtype = NVME_NQN_DISC;
2776         else
2777                 subsys->subtype = NVME_NQN_NVME;
2778
2779         if (nvme_discovery_ctrl(ctrl) && subsys->subtype != NVME_NQN_DISC) {
2780                 dev_err(ctrl->device,
2781                         "Subsystem %s is not a discovery controller",
2782                         subsys->subnqn);
2783                 kfree(subsys);
2784                 return -EINVAL;
2785         }
2786         subsys->awupf = le16_to_cpu(id->awupf);
2787         nvme_mpath_default_iopolicy(subsys);
2788
2789         subsys->dev.class = nvme_subsys_class;
2790         subsys->dev.release = nvme_release_subsystem;
2791         subsys->dev.groups = nvme_subsys_attrs_groups;
2792         dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance);
2793         device_initialize(&subsys->dev);
2794
2795         mutex_lock(&nvme_subsystems_lock);
2796         found = __nvme_find_get_subsystem(subsys->subnqn);
2797         if (found) {
2798                 put_device(&subsys->dev);
2799                 subsys = found;
2800
2801                 if (!nvme_validate_cntlid(subsys, ctrl, id)) {
2802                         ret = -EINVAL;
2803                         goto out_put_subsystem;
2804                 }
2805         } else {
2806                 ret = device_add(&subsys->dev);
2807                 if (ret) {
2808                         dev_err(ctrl->device,
2809                                 "failed to register subsystem device.\n");
2810                         put_device(&subsys->dev);
2811                         goto out_unlock;
2812                 }
2813                 ida_init(&subsys->ns_ida);
2814                 list_add_tail(&subsys->entry, &nvme_subsystems);
2815         }
2816
2817         ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
2818                                 dev_name(ctrl->device));
2819         if (ret) {
2820                 dev_err(ctrl->device,
2821                         "failed to create sysfs link from subsystem.\n");
2822                 goto out_put_subsystem;
2823         }
2824
2825         if (!found)
2826                 subsys->instance = ctrl->instance;
2827         ctrl->subsys = subsys;
2828         list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
2829         mutex_unlock(&nvme_subsystems_lock);
2830         return 0;
2831
2832 out_put_subsystem:
2833         nvme_put_subsystem(subsys);
2834 out_unlock:
2835         mutex_unlock(&nvme_subsystems_lock);
2836         return ret;
2837 }
2838
2839 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
2840                 void *log, size_t size, u64 offset)
2841 {
2842         struct nvme_command c = { };
2843         u32 dwlen = nvme_bytes_to_numd(size);
2844
2845         c.get_log_page.opcode = nvme_admin_get_log_page;
2846         c.get_log_page.nsid = cpu_to_le32(nsid);
2847         c.get_log_page.lid = log_page;
2848         c.get_log_page.lsp = lsp;
2849         c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
2850         c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
2851         c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
2852         c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
2853         c.get_log_page.csi = csi;
2854
2855         return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
2856 }
2857
2858 static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
2859                                 struct nvme_effects_log **log)
2860 {
2861         struct nvme_effects_log *cel = xa_load(&ctrl->cels, csi);
2862         int ret;
2863
2864         if (cel)
2865                 goto out;
2866
2867         cel = kzalloc(sizeof(*cel), GFP_KERNEL);
2868         if (!cel)
2869                 return -ENOMEM;
2870
2871         ret = nvme_get_log(ctrl, 0x00, NVME_LOG_CMD_EFFECTS, 0, csi,
2872                         cel, sizeof(*cel), 0);
2873         if (ret) {
2874                 kfree(cel);
2875                 return ret;
2876         }
2877
2878         xa_store(&ctrl->cels, csi, cel, GFP_KERNEL);
2879 out:
2880         *log = cel;
2881         return 0;
2882 }
2883
2884 static inline u32 nvme_mps_to_sectors(struct nvme_ctrl *ctrl, u32 units)
2885 {
2886         u32 page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12, val;
2887
2888         if (check_shl_overflow(1U, units + page_shift - 9, &val))
2889                 return UINT_MAX;
2890         return val;
2891 }
2892
2893 static int nvme_init_non_mdts_limits(struct nvme_ctrl *ctrl)
2894 {
2895         struct nvme_command c = { };
2896         struct nvme_id_ctrl_nvm *id;
2897         int ret;
2898
2899         if (ctrl->oncs & NVME_CTRL_ONCS_DSM) {
2900                 ctrl->max_discard_sectors = UINT_MAX;
2901                 ctrl->max_discard_segments = NVME_DSM_MAX_RANGES;
2902         } else {
2903                 ctrl->max_discard_sectors = 0;
2904                 ctrl->max_discard_segments = 0;
2905         }
2906
2907         /*
2908          * Even though NVMe spec explicitly states that MDTS is not applicable
2909          * to the write-zeroes, we are cautious and limit the size to the
2910          * controllers max_hw_sectors value, which is based on the MDTS field
2911          * and possibly other limiting factors.
2912          */
2913         if ((ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) &&
2914             !(ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
2915                 ctrl->max_zeroes_sectors = ctrl->max_hw_sectors;
2916         else
2917                 ctrl->max_zeroes_sectors = 0;
2918
2919         if (ctrl->subsys->subtype != NVME_NQN_NVME ||
2920             nvme_ctrl_limited_cns(ctrl))
2921                 return 0;
2922
2923         id = kzalloc(sizeof(*id), GFP_KERNEL);
2924         if (!id)
2925                 return -ENOMEM;
2926
2927         c.identify.opcode = nvme_admin_identify;
2928         c.identify.cns = NVME_ID_CNS_CS_CTRL;
2929         c.identify.csi = NVME_CSI_NVM;
2930
2931         ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
2932         if (ret)
2933                 goto free_data;
2934
2935         if (id->dmrl)
2936                 ctrl->max_discard_segments = id->dmrl;
2937         ctrl->dmrsl = le32_to_cpu(id->dmrsl);
2938         if (id->wzsl)
2939                 ctrl->max_zeroes_sectors = nvme_mps_to_sectors(ctrl, id->wzsl);
2940
2941 free_data:
2942         kfree(id);
2943         return ret;
2944 }
2945
2946 static void nvme_init_known_nvm_effects(struct nvme_ctrl *ctrl)
2947 {
2948         struct nvme_effects_log *log = ctrl->effects;
2949
2950         log->acs[nvme_admin_format_nvm] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC |
2951                                                 NVME_CMD_EFFECTS_NCC |
2952                                                 NVME_CMD_EFFECTS_CSE_MASK);
2953         log->acs[nvme_admin_sanitize_nvm] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC |
2954                                                 NVME_CMD_EFFECTS_CSE_MASK);
2955
2956         /*
2957          * The spec says the result of a security receive command depends on
2958          * the previous security send command. As such, many vendors log this
2959          * command as one to submitted only when no other commands to the same
2960          * namespace are outstanding. The intention is to tell the host to
2961          * prevent mixing security send and receive.
2962          *
2963          * This driver can only enforce such exclusive access against IO
2964          * queues, though. We are not readily able to enforce such a rule for
2965          * two commands to the admin queue, which is the only queue that
2966          * matters for this command.
2967          *
2968          * Rather than blindly freezing the IO queues for this effect that
2969          * doesn't even apply to IO, mask it off.
2970          */
2971         log->acs[nvme_admin_security_recv] &= cpu_to_le32(~NVME_CMD_EFFECTS_CSE_MASK);
2972
2973         log->iocs[nvme_cmd_write] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC);
2974         log->iocs[nvme_cmd_write_zeroes] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC);
2975         log->iocs[nvme_cmd_write_uncor] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC);
2976 }
2977
2978 static int nvme_init_effects(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2979 {
2980         int ret = 0;
2981
2982         if (ctrl->effects)
2983                 return 0;
2984
2985         if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
2986                 ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects);
2987                 if (ret < 0)
2988                         return ret;
2989         }
2990
2991         if (!ctrl->effects) {
2992                 ctrl->effects = kzalloc(sizeof(*ctrl->effects), GFP_KERNEL);
2993                 if (!ctrl->effects)
2994                         return -ENOMEM;
2995                 xa_store(&ctrl->cels, NVME_CSI_NVM, ctrl->effects, GFP_KERNEL);
2996         }
2997
2998         nvme_init_known_nvm_effects(ctrl);
2999         return 0;
3000 }
3001
3002 static int nvme_init_identify(struct nvme_ctrl *ctrl)
3003 {
3004         struct nvme_id_ctrl *id;
3005         u32 max_hw_sectors;
3006         bool prev_apst_enabled;
3007         int ret;
3008
3009         ret = nvme_identify_ctrl(ctrl, &id);
3010         if (ret) {
3011                 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
3012                 return -EIO;
3013         }
3014
3015         if (!(ctrl->ops->flags & NVME_F_FABRICS))
3016                 ctrl->cntlid = le16_to_cpu(id->cntlid);
3017
3018         if (!ctrl->identified) {
3019                 unsigned int i;
3020
3021                 /*
3022                  * Check for quirks.  Quirk can depend on firmware version,
3023                  * so, in principle, the set of quirks present can change
3024                  * across a reset.  As a possible future enhancement, we
3025                  * could re-scan for quirks every time we reinitialize
3026                  * the device, but we'd have to make sure that the driver
3027                  * behaves intelligently if the quirks change.
3028                  */
3029                 for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
3030                         if (quirk_matches(id, &core_quirks[i]))
3031                                 ctrl->quirks |= core_quirks[i].quirks;
3032                 }
3033
3034                 ret = nvme_init_subsystem(ctrl, id);
3035                 if (ret)
3036                         goto out_free;
3037
3038                 ret = nvme_init_effects(ctrl, id);
3039                 if (ret)
3040                         goto out_free;
3041         }
3042         memcpy(ctrl->subsys->firmware_rev, id->fr,
3043                sizeof(ctrl->subsys->firmware_rev));
3044
3045         if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
3046                 dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
3047                 ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
3048         }
3049
3050         ctrl->crdt[0] = le16_to_cpu(id->crdt1);
3051         ctrl->crdt[1] = le16_to_cpu(id->crdt2);
3052         ctrl->crdt[2] = le16_to_cpu(id->crdt3);
3053
3054         ctrl->oacs = le16_to_cpu(id->oacs);
3055         ctrl->oncs = le16_to_cpu(id->oncs);
3056         ctrl->mtfa = le16_to_cpu(id->mtfa);
3057         ctrl->oaes = le32_to_cpu(id->oaes);
3058         ctrl->wctemp = le16_to_cpu(id->wctemp);
3059         ctrl->cctemp = le16_to_cpu(id->cctemp);
3060
3061         atomic_set(&ctrl->abort_limit, id->acl + 1);
3062         ctrl->vwc = id->vwc;
3063         if (id->mdts)
3064                 max_hw_sectors = nvme_mps_to_sectors(ctrl, id->mdts);
3065         else
3066                 max_hw_sectors = UINT_MAX;
3067         ctrl->max_hw_sectors =
3068                 min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
3069
3070         nvme_set_queue_limits(ctrl, ctrl->admin_q);
3071         ctrl->sgls = le32_to_cpu(id->sgls);
3072         ctrl->kas = le16_to_cpu(id->kas);
3073         ctrl->max_namespaces = le32_to_cpu(id->mnan);
3074         ctrl->ctratt = le32_to_cpu(id->ctratt);
3075
3076         ctrl->cntrltype = id->cntrltype;
3077         ctrl->dctype = id->dctype;
3078
3079         if (id->rtd3e) {
3080                 /* us -> s */
3081                 u32 transition_time = le32_to_cpu(id->rtd3e) / USEC_PER_SEC;
3082
3083                 ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
3084                                                  shutdown_timeout, 60);
3085
3086                 if (ctrl->shutdown_timeout != shutdown_timeout)
3087                         dev_info(ctrl->device,
3088                                  "Shutdown timeout set to %u seconds\n",
3089                                  ctrl->shutdown_timeout);
3090         } else
3091                 ctrl->shutdown_timeout = shutdown_timeout;
3092
3093         ctrl->npss = id->npss;
3094         ctrl->apsta = id->apsta;
3095         prev_apst_enabled = ctrl->apst_enabled;
3096         if (ctrl->quirks & NVME_QUIRK_NO_APST) {
3097                 if (force_apst && id->apsta) {
3098                         dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
3099                         ctrl->apst_enabled = true;
3100                 } else {
3101                         ctrl->apst_enabled = false;
3102                 }
3103         } else {
3104                 ctrl->apst_enabled = id->apsta;
3105         }
3106         memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
3107
3108         if (ctrl->ops->flags & NVME_F_FABRICS) {
3109                 ctrl->icdoff = le16_to_cpu(id->icdoff);
3110                 ctrl->ioccsz = le32_to_cpu(id->ioccsz);
3111                 ctrl->iorcsz = le32_to_cpu(id->iorcsz);
3112                 ctrl->maxcmd = le16_to_cpu(id->maxcmd);
3113
3114                 /*
3115                  * In fabrics we need to verify the cntlid matches the
3116                  * admin connect
3117                  */
3118                 if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
3119                         dev_err(ctrl->device,
3120                                 "Mismatching cntlid: Connect %u vs Identify "
3121                                 "%u, rejecting\n",
3122                                 ctrl->cntlid, le16_to_cpu(id->cntlid));
3123                         ret = -EINVAL;
3124                         goto out_free;
3125                 }
3126
3127                 if (!nvme_discovery_ctrl(ctrl) && !ctrl->kas) {
3128                         dev_err(ctrl->device,
3129                                 "keep-alive support is mandatory for fabrics\n");
3130                         ret = -EINVAL;
3131                         goto out_free;
3132                 }
3133         } else {
3134                 ctrl->hmpre = le32_to_cpu(id->hmpre);
3135                 ctrl->hmmin = le32_to_cpu(id->hmmin);
3136                 ctrl->hmminds = le32_to_cpu(id->hmminds);
3137                 ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
3138         }
3139
3140         ret = nvme_mpath_init_identify(ctrl, id);
3141         if (ret < 0)
3142                 goto out_free;
3143
3144         if (ctrl->apst_enabled && !prev_apst_enabled)
3145                 dev_pm_qos_expose_latency_tolerance(ctrl->device);
3146         else if (!ctrl->apst_enabled && prev_apst_enabled)
3147                 dev_pm_qos_hide_latency_tolerance(ctrl->device);
3148
3149 out_free:
3150         kfree(id);
3151         return ret;
3152 }
3153
3154 /*
3155  * Initialize the cached copies of the Identify data and various controller
3156  * register in our nvme_ctrl structure.  This should be called as soon as
3157  * the admin queue is fully up and running.
3158  */
3159 int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl, bool was_suspended)
3160 {
3161         int ret;
3162
3163         ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
3164         if (ret) {
3165                 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
3166                 return ret;
3167         }
3168
3169         ctrl->sqsize = min_t(u16, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
3170
3171         if (ctrl->vs >= NVME_VS(1, 1, 0))
3172                 ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap);
3173
3174         ret = nvme_init_identify(ctrl);
3175         if (ret)
3176                 return ret;
3177
3178         ret = nvme_configure_apst(ctrl);
3179         if (ret < 0)
3180                 return ret;
3181
3182         ret = nvme_configure_timestamp(ctrl);
3183         if (ret < 0)
3184                 return ret;
3185
3186         ret = nvme_configure_host_options(ctrl);
3187         if (ret < 0)
3188                 return ret;
3189
3190         nvme_configure_opal(ctrl, was_suspended);
3191
3192         if (!ctrl->identified && !nvme_discovery_ctrl(ctrl)) {
3193                 /*
3194                  * Do not return errors unless we are in a controller reset,
3195                  * the controller works perfectly fine without hwmon.
3196                  */
3197                 ret = nvme_hwmon_init(ctrl);
3198                 if (ret == -EINTR)
3199                         return ret;
3200         }
3201
3202         ctrl->identified = true;
3203
3204         return 0;
3205 }
3206 EXPORT_SYMBOL_GPL(nvme_init_ctrl_finish);
3207
3208 static int nvme_dev_open(struct inode *inode, struct file *file)
3209 {
3210         struct nvme_ctrl *ctrl =
3211                 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3212
3213         switch (ctrl->state) {
3214         case NVME_CTRL_LIVE:
3215                 break;
3216         default:
3217                 return -EWOULDBLOCK;
3218         }
3219
3220         nvme_get_ctrl(ctrl);
3221         if (!try_module_get(ctrl->ops->module)) {
3222                 nvme_put_ctrl(ctrl);
3223                 return -EINVAL;
3224         }
3225
3226         file->private_data = ctrl;
3227         return 0;
3228 }
3229
3230 static int nvme_dev_release(struct inode *inode, struct file *file)
3231 {
3232         struct nvme_ctrl *ctrl =
3233                 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3234
3235         module_put(ctrl->ops->module);
3236         nvme_put_ctrl(ctrl);
3237         return 0;
3238 }
3239
3240 static const struct file_operations nvme_dev_fops = {
3241         .owner          = THIS_MODULE,
3242         .open           = nvme_dev_open,
3243         .release        = nvme_dev_release,
3244         .unlocked_ioctl = nvme_dev_ioctl,
3245         .compat_ioctl   = compat_ptr_ioctl,
3246         .uring_cmd      = nvme_dev_uring_cmd,
3247 };
3248
3249 static ssize_t nvme_sysfs_reset(struct device *dev,
3250                                 struct device_attribute *attr, const char *buf,
3251                                 size_t count)
3252 {
3253         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3254         int ret;
3255
3256         ret = nvme_reset_ctrl_sync(ctrl);
3257         if (ret < 0)
3258                 return ret;
3259         return count;
3260 }
3261 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
3262
3263 static ssize_t nvme_sysfs_rescan(struct device *dev,
3264                                 struct device_attribute *attr, const char *buf,
3265                                 size_t count)
3266 {
3267         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3268
3269         nvme_queue_scan(ctrl);
3270         return count;
3271 }
3272 static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
3273
3274 static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
3275 {
3276         struct gendisk *disk = dev_to_disk(dev);
3277
3278         if (disk->fops == &nvme_bdev_ops)
3279                 return nvme_get_ns_from_dev(dev)->head;
3280         else
3281                 return disk->private_data;
3282 }
3283
3284 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
3285                 char *buf)
3286 {
3287         struct nvme_ns_head *head = dev_to_ns_head(dev);
3288         struct nvme_ns_ids *ids = &head->ids;
3289         struct nvme_subsystem *subsys = head->subsys;
3290         int serial_len = sizeof(subsys->serial);
3291         int model_len = sizeof(subsys->model);
3292
3293         if (!uuid_is_null(&ids->uuid))
3294                 return sysfs_emit(buf, "uuid.%pU\n", &ids->uuid);
3295
3296         if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3297                 return sysfs_emit(buf, "eui.%16phN\n", ids->nguid);
3298
3299         if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3300                 return sysfs_emit(buf, "eui.%8phN\n", ids->eui64);
3301
3302         while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' ||
3303                                   subsys->serial[serial_len - 1] == '\0'))
3304                 serial_len--;
3305         while (model_len > 0 && (subsys->model[model_len - 1] == ' ' ||
3306                                  subsys->model[model_len - 1] == '\0'))
3307                 model_len--;
3308
3309         return sysfs_emit(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id,
3310                 serial_len, subsys->serial, model_len, subsys->model,
3311                 head->ns_id);
3312 }
3313 static DEVICE_ATTR_RO(wwid);
3314
3315 static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
3316                 char *buf)
3317 {
3318         return sysfs_emit(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
3319 }
3320 static DEVICE_ATTR_RO(nguid);
3321
3322 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
3323                 char *buf)
3324 {
3325         struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3326
3327         /* For backward compatibility expose the NGUID to userspace if
3328          * we have no UUID set
3329          */
3330         if (uuid_is_null(&ids->uuid)) {
3331                 dev_warn_ratelimited(dev,
3332                         "No UUID available providing old NGUID\n");
3333                 return sysfs_emit(buf, "%pU\n", ids->nguid);
3334         }
3335         return sysfs_emit(buf, "%pU\n", &ids->uuid);
3336 }
3337 static DEVICE_ATTR_RO(uuid);
3338
3339 static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
3340                 char *buf)
3341 {
3342         return sysfs_emit(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
3343 }
3344 static DEVICE_ATTR_RO(eui);
3345
3346 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
3347                 char *buf)
3348 {
3349         return sysfs_emit(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
3350 }
3351 static DEVICE_ATTR_RO(nsid);
3352
3353 static struct attribute *nvme_ns_id_attrs[] = {
3354         &dev_attr_wwid.attr,
3355         &dev_attr_uuid.attr,
3356         &dev_attr_nguid.attr,
3357         &dev_attr_eui.attr,
3358         &dev_attr_nsid.attr,
3359 #ifdef CONFIG_NVME_MULTIPATH
3360         &dev_attr_ana_grpid.attr,
3361         &dev_attr_ana_state.attr,
3362 #endif
3363         NULL,
3364 };
3365
3366 static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
3367                 struct attribute *a, int n)
3368 {
3369         struct device *dev = container_of(kobj, struct device, kobj);
3370         struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3371
3372         if (a == &dev_attr_uuid.attr) {
3373                 if (uuid_is_null(&ids->uuid) &&
3374                     !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3375                         return 0;
3376         }
3377         if (a == &dev_attr_nguid.attr) {
3378                 if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3379                         return 0;
3380         }
3381         if (a == &dev_attr_eui.attr) {
3382                 if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3383                         return 0;
3384         }
3385 #ifdef CONFIG_NVME_MULTIPATH
3386         if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
3387                 if (dev_to_disk(dev)->fops != &nvme_bdev_ops) /* per-path attr */
3388                         return 0;
3389                 if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl))
3390                         return 0;
3391         }
3392 #endif
3393         return a->mode;
3394 }
3395
3396 static const struct attribute_group nvme_ns_id_attr_group = {
3397         .attrs          = nvme_ns_id_attrs,
3398         .is_visible     = nvme_ns_id_attrs_are_visible,
3399 };
3400
3401 const struct attribute_group *nvme_ns_id_attr_groups[] = {
3402         &nvme_ns_id_attr_group,
3403         NULL,
3404 };
3405
3406 #define nvme_show_str_function(field)                                           \
3407 static ssize_t  field##_show(struct device *dev,                                \
3408                             struct device_attribute *attr, char *buf)           \
3409 {                                                                               \
3410         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
3411         return sysfs_emit(buf, "%.*s\n",                                        \
3412                 (int)sizeof(ctrl->subsys->field), ctrl->subsys->field);         \
3413 }                                                                               \
3414 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3415
3416 nvme_show_str_function(model);
3417 nvme_show_str_function(serial);
3418 nvme_show_str_function(firmware_rev);
3419
3420 #define nvme_show_int_function(field)                                           \
3421 static ssize_t  field##_show(struct device *dev,                                \
3422                             struct device_attribute *attr, char *buf)           \
3423 {                                                                               \
3424         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
3425         return sysfs_emit(buf, "%d\n", ctrl->field);                            \
3426 }                                                                               \
3427 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3428
3429 nvme_show_int_function(cntlid);
3430 nvme_show_int_function(numa_node);
3431 nvme_show_int_function(queue_count);
3432 nvme_show_int_function(sqsize);
3433 nvme_show_int_function(kato);
3434
3435 static ssize_t nvme_sysfs_delete(struct device *dev,
3436                                 struct device_attribute *attr, const char *buf,
3437                                 size_t count)
3438 {
3439         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3440
3441         if (device_remove_file_self(dev, attr))
3442                 nvme_delete_ctrl_sync(ctrl);
3443         return count;
3444 }
3445 static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
3446
3447 static ssize_t nvme_sysfs_show_transport(struct device *dev,
3448                                          struct device_attribute *attr,
3449                                          char *buf)
3450 {
3451         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3452
3453         return sysfs_emit(buf, "%s\n", ctrl->ops->name);
3454 }
3455 static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
3456
3457 static ssize_t nvme_sysfs_show_state(struct device *dev,
3458                                      struct device_attribute *attr,
3459                                      char *buf)
3460 {
3461         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3462         static const char *const state_name[] = {
3463                 [NVME_CTRL_NEW]         = "new",
3464                 [NVME_CTRL_LIVE]        = "live",
3465                 [NVME_CTRL_RESETTING]   = "resetting",
3466                 [NVME_CTRL_CONNECTING]  = "connecting",
3467                 [NVME_CTRL_DELETING]    = "deleting",
3468                 [NVME_CTRL_DELETING_NOIO]= "deleting (no IO)",
3469                 [NVME_CTRL_DEAD]        = "dead",
3470         };
3471
3472         if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
3473             state_name[ctrl->state])
3474                 return sysfs_emit(buf, "%s\n", state_name[ctrl->state]);
3475
3476         return sysfs_emit(buf, "unknown state\n");
3477 }
3478
3479 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
3480
3481 static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
3482                                          struct device_attribute *attr,
3483                                          char *buf)
3484 {
3485         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3486
3487         return sysfs_emit(buf, "%s\n", ctrl->subsys->subnqn);
3488 }
3489 static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
3490
3491 static ssize_t nvme_sysfs_show_hostnqn(struct device *dev,
3492                                         struct device_attribute *attr,
3493                                         char *buf)
3494 {
3495         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3496
3497         return sysfs_emit(buf, "%s\n", ctrl->opts->host->nqn);
3498 }
3499 static DEVICE_ATTR(hostnqn, S_IRUGO, nvme_sysfs_show_hostnqn, NULL);
3500
3501 static ssize_t nvme_sysfs_show_hostid(struct device *dev,
3502                                         struct device_attribute *attr,
3503                                         char *buf)
3504 {
3505         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3506
3507         return sysfs_emit(buf, "%pU\n", &ctrl->opts->host->id);
3508 }
3509 static DEVICE_ATTR(hostid, S_IRUGO, nvme_sysfs_show_hostid, NULL);
3510
3511 static ssize_t nvme_sysfs_show_address(struct device *dev,
3512                                          struct device_attribute *attr,
3513                                          char *buf)
3514 {
3515         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3516
3517         return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
3518 }
3519 static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
3520
3521 static ssize_t nvme_ctrl_loss_tmo_show(struct device *dev,
3522                 struct device_attribute *attr, char *buf)
3523 {
3524         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3525         struct nvmf_ctrl_options *opts = ctrl->opts;
3526
3527         if (ctrl->opts->max_reconnects == -1)
3528                 return sysfs_emit(buf, "off\n");
3529         return sysfs_emit(buf, "%d\n",
3530                           opts->max_reconnects * opts->reconnect_delay);
3531 }
3532
3533 static ssize_t nvme_ctrl_loss_tmo_store(struct device *dev,
3534                 struct device_attribute *attr, const char *buf, size_t count)
3535 {
3536         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3537         struct nvmf_ctrl_options *opts = ctrl->opts;
3538         int ctrl_loss_tmo, err;
3539
3540         err = kstrtoint(buf, 10, &ctrl_loss_tmo);
3541         if (err)
3542                 return -EINVAL;
3543
3544         if (ctrl_loss_tmo < 0)
3545                 opts->max_reconnects = -1;
3546         else
3547                 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3548                                                 opts->reconnect_delay);
3549         return count;
3550 }
3551 static DEVICE_ATTR(ctrl_loss_tmo, S_IRUGO | S_IWUSR,
3552         nvme_ctrl_loss_tmo_show, nvme_ctrl_loss_tmo_store);
3553
3554 static ssize_t nvme_ctrl_reconnect_delay_show(struct device *dev,
3555                 struct device_attribute *attr, char *buf)
3556 {
3557         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3558
3559         if (ctrl->opts->reconnect_delay == -1)
3560                 return sysfs_emit(buf, "off\n");
3561         return sysfs_emit(buf, "%d\n", ctrl->opts->reconnect_delay);
3562 }
3563
3564 static ssize_t nvme_ctrl_reconnect_delay_store(struct device *dev,
3565                 struct device_attribute *attr, const char *buf, size_t count)
3566 {
3567         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3568         unsigned int v;
3569         int err;
3570
3571         err = kstrtou32(buf, 10, &v);
3572         if (err)
3573                 return err;
3574
3575         ctrl->opts->reconnect_delay = v;
3576         return count;
3577 }
3578 static DEVICE_ATTR(reconnect_delay, S_IRUGO | S_IWUSR,
3579         nvme_ctrl_reconnect_delay_show, nvme_ctrl_reconnect_delay_store);
3580
3581 static ssize_t nvme_ctrl_fast_io_fail_tmo_show(struct device *dev,
3582                 struct device_attribute *attr, char *buf)
3583 {
3584         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3585
3586         if (ctrl->opts->fast_io_fail_tmo == -1)
3587                 return sysfs_emit(buf, "off\n");
3588         return sysfs_emit(buf, "%d\n", ctrl->opts->fast_io_fail_tmo);
3589 }
3590
3591 static ssize_t nvme_ctrl_fast_io_fail_tmo_store(struct device *dev,
3592                 struct device_attribute *attr, const char *buf, size_t count)
3593 {
3594         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3595         struct nvmf_ctrl_options *opts = ctrl->opts;
3596         int fast_io_fail_tmo, err;
3597
3598         err = kstrtoint(buf, 10, &fast_io_fail_tmo);
3599         if (err)
3600                 return -EINVAL;
3601
3602         if (fast_io_fail_tmo < 0)
3603                 opts->fast_io_fail_tmo = -1;
3604         else
3605                 opts->fast_io_fail_tmo = fast_io_fail_tmo;
3606         return count;
3607 }
3608 static DEVICE_ATTR(fast_io_fail_tmo, S_IRUGO | S_IWUSR,
3609         nvme_ctrl_fast_io_fail_tmo_show, nvme_ctrl_fast_io_fail_tmo_store);
3610
3611 static ssize_t cntrltype_show(struct device *dev,
3612                               struct device_attribute *attr, char *buf)
3613 {
3614         static const char * const type[] = {
3615                 [NVME_CTRL_IO] = "io\n",
3616                 [NVME_CTRL_DISC] = "discovery\n",
3617                 [NVME_CTRL_ADMIN] = "admin\n",
3618         };
3619         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3620
3621         if (ctrl->cntrltype > NVME_CTRL_ADMIN || !type[ctrl->cntrltype])
3622                 return sysfs_emit(buf, "reserved\n");
3623
3624         return sysfs_emit(buf, type[ctrl->cntrltype]);
3625 }
3626 static DEVICE_ATTR_RO(cntrltype);
3627
3628 static ssize_t dctype_show(struct device *dev,
3629                            struct device_attribute *attr, char *buf)
3630 {
3631         static const char * const type[] = {
3632                 [NVME_DCTYPE_NOT_REPORTED] = "none\n",
3633                 [NVME_DCTYPE_DDC] = "ddc\n",
3634                 [NVME_DCTYPE_CDC] = "cdc\n",
3635         };
3636         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3637
3638         if (ctrl->dctype > NVME_DCTYPE_CDC || !type[ctrl->dctype])
3639                 return sysfs_emit(buf, "reserved\n");
3640
3641         return sysfs_emit(buf, type[ctrl->dctype]);
3642 }
3643 static DEVICE_ATTR_RO(dctype);
3644
3645 #ifdef CONFIG_NVME_AUTH
3646 static ssize_t nvme_ctrl_dhchap_secret_show(struct device *dev,
3647                 struct device_attribute *attr, char *buf)
3648 {
3649         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3650         struct nvmf_ctrl_options *opts = ctrl->opts;
3651
3652         if (!opts->dhchap_secret)
3653                 return sysfs_emit(buf, "none\n");
3654         return sysfs_emit(buf, "%s\n", opts->dhchap_secret);
3655 }
3656
3657 static ssize_t nvme_ctrl_dhchap_secret_store(struct device *dev,
3658                 struct device_attribute *attr, const char *buf, size_t count)
3659 {
3660         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3661         struct nvmf_ctrl_options *opts = ctrl->opts;
3662         char *dhchap_secret;
3663
3664         if (!ctrl->opts->dhchap_secret)
3665                 return -EINVAL;
3666         if (count < 7)
3667                 return -EINVAL;
3668         if (memcmp(buf, "DHHC-1:", 7))
3669                 return -EINVAL;
3670
3671         dhchap_secret = kzalloc(count + 1, GFP_KERNEL);
3672         if (!dhchap_secret)
3673                 return -ENOMEM;
3674         memcpy(dhchap_secret, buf, count);
3675         nvme_auth_stop(ctrl);
3676         if (strcmp(dhchap_secret, opts->dhchap_secret)) {
3677                 struct nvme_dhchap_key *key, *host_key;
3678                 int ret;
3679
3680                 ret = nvme_auth_generate_key(dhchap_secret, &key);
3681                 if (ret)
3682                         return ret;
3683                 kfree(opts->dhchap_secret);
3684                 opts->dhchap_secret = dhchap_secret;
3685                 host_key = ctrl->host_key;
3686                 mutex_lock(&ctrl->dhchap_auth_mutex);
3687                 ctrl->host_key = key;
3688                 mutex_unlock(&ctrl->dhchap_auth_mutex);
3689                 nvme_auth_free_key(host_key);
3690         }
3691         /* Start re-authentication */
3692         dev_info(ctrl->device, "re-authenticating controller\n");
3693         queue_work(nvme_wq, &ctrl->dhchap_auth_work);
3694
3695         return count;
3696 }
3697 static DEVICE_ATTR(dhchap_secret, S_IRUGO | S_IWUSR,
3698         nvme_ctrl_dhchap_secret_show, nvme_ctrl_dhchap_secret_store);
3699
3700 static ssize_t nvme_ctrl_dhchap_ctrl_secret_show(struct device *dev,
3701                 struct device_attribute *attr, char *buf)
3702 {
3703         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3704         struct nvmf_ctrl_options *opts = ctrl->opts;
3705
3706         if (!opts->dhchap_ctrl_secret)
3707                 return sysfs_emit(buf, "none\n");
3708         return sysfs_emit(buf, "%s\n", opts->dhchap_ctrl_secret);
3709 }
3710
3711 static ssize_t nvme_ctrl_dhchap_ctrl_secret_store(struct device *dev,
3712                 struct device_attribute *attr, const char *buf, size_t count)
3713 {
3714         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3715         struct nvmf_ctrl_options *opts = ctrl->opts;
3716         char *dhchap_secret;
3717
3718         if (!ctrl->opts->dhchap_ctrl_secret)
3719                 return -EINVAL;
3720         if (count < 7)
3721                 return -EINVAL;
3722         if (memcmp(buf, "DHHC-1:", 7))
3723                 return -EINVAL;
3724
3725         dhchap_secret = kzalloc(count + 1, GFP_KERNEL);
3726         if (!dhchap_secret)
3727                 return -ENOMEM;
3728         memcpy(dhchap_secret, buf, count);
3729         nvme_auth_stop(ctrl);
3730         if (strcmp(dhchap_secret, opts->dhchap_ctrl_secret)) {
3731                 struct nvme_dhchap_key *key, *ctrl_key;
3732                 int ret;
3733
3734                 ret = nvme_auth_generate_key(dhchap_secret, &key);
3735                 if (ret)
3736                         return ret;
3737                 kfree(opts->dhchap_ctrl_secret);
3738                 opts->dhchap_ctrl_secret = dhchap_secret;
3739                 ctrl_key = ctrl->ctrl_key;
3740                 mutex_lock(&ctrl->dhchap_auth_mutex);
3741                 ctrl->ctrl_key = key;
3742                 mutex_unlock(&ctrl->dhchap_auth_mutex);
3743                 nvme_auth_free_key(ctrl_key);
3744         }
3745         /* Start re-authentication */
3746         dev_info(ctrl->device, "re-authenticating controller\n");
3747         queue_work(nvme_wq, &ctrl->dhchap_auth_work);
3748
3749         return count;
3750 }
3751 static DEVICE_ATTR(dhchap_ctrl_secret, S_IRUGO | S_IWUSR,
3752         nvme_ctrl_dhchap_ctrl_secret_show, nvme_ctrl_dhchap_ctrl_secret_store);
3753 #endif
3754
3755 static struct attribute *nvme_dev_attrs[] = {
3756         &dev_attr_reset_controller.attr,
3757         &dev_attr_rescan_controller.attr,
3758         &dev_attr_model.attr,
3759         &dev_attr_serial.attr,
3760         &dev_attr_firmware_rev.attr,
3761         &dev_attr_cntlid.attr,
3762         &dev_attr_delete_controller.attr,
3763         &dev_attr_transport.attr,
3764         &dev_attr_subsysnqn.attr,
3765         &dev_attr_address.attr,
3766         &dev_attr_state.attr,
3767         &dev_attr_numa_node.attr,
3768         &dev_attr_queue_count.attr,
3769         &dev_attr_sqsize.attr,
3770         &dev_attr_hostnqn.attr,
3771         &dev_attr_hostid.attr,
3772         &dev_attr_ctrl_loss_tmo.attr,
3773         &dev_attr_reconnect_delay.attr,
3774         &dev_attr_fast_io_fail_tmo.attr,
3775         &dev_attr_kato.attr,
3776         &dev_attr_cntrltype.attr,
3777         &dev_attr_dctype.attr,
3778 #ifdef CONFIG_NVME_AUTH
3779         &dev_attr_dhchap_secret.attr,
3780         &dev_attr_dhchap_ctrl_secret.attr,
3781 #endif
3782         NULL
3783 };
3784
3785 static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
3786                 struct attribute *a, int n)
3787 {
3788         struct device *dev = container_of(kobj, struct device, kobj);
3789         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3790
3791         if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
3792                 return 0;
3793         if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
3794                 return 0;
3795         if (a == &dev_attr_hostnqn.attr && !ctrl->opts)
3796                 return 0;
3797         if (a == &dev_attr_hostid.attr && !ctrl->opts)
3798                 return 0;
3799         if (a == &dev_attr_ctrl_loss_tmo.attr && !ctrl->opts)
3800                 return 0;
3801         if (a == &dev_attr_reconnect_delay.attr && !ctrl->opts)
3802                 return 0;
3803         if (a == &dev_attr_fast_io_fail_tmo.attr && !ctrl->opts)
3804                 return 0;
3805 #ifdef CONFIG_NVME_AUTH
3806         if (a == &dev_attr_dhchap_secret.attr && !ctrl->opts)
3807                 return 0;
3808         if (a == &dev_attr_dhchap_ctrl_secret.attr && !ctrl->opts)
3809                 return 0;
3810 #endif
3811
3812         return a->mode;
3813 }
3814
3815 const struct attribute_group nvme_dev_attrs_group = {
3816         .attrs          = nvme_dev_attrs,
3817         .is_visible     = nvme_dev_attrs_are_visible,
3818 };
3819 EXPORT_SYMBOL_GPL(nvme_dev_attrs_group);
3820
3821 static const struct attribute_group *nvme_dev_attr_groups[] = {
3822         &nvme_dev_attrs_group,
3823         NULL,
3824 };
3825
3826 static struct nvme_ns_head *nvme_find_ns_head(struct nvme_ctrl *ctrl,
3827                 unsigned nsid)
3828 {
3829         struct nvme_ns_head *h;
3830
3831         lockdep_assert_held(&ctrl->subsys->lock);
3832
3833         list_for_each_entry(h, &ctrl->subsys->nsheads, entry) {
3834                 /*
3835                  * Private namespaces can share NSIDs under some conditions.
3836                  * In that case we can't use the same ns_head for namespaces
3837                  * with the same NSID.
3838                  */
3839                 if (h->ns_id != nsid || !nvme_is_unique_nsid(ctrl, h))
3840                         continue;
3841                 if (!list_empty(&h->list) && nvme_tryget_ns_head(h))
3842                         return h;
3843         }
3844
3845         return NULL;
3846 }
3847
3848 static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys,
3849                 struct nvme_ns_ids *ids)
3850 {
3851         bool has_uuid = !uuid_is_null(&ids->uuid);
3852         bool has_nguid = memchr_inv(ids->nguid, 0, sizeof(ids->nguid));
3853         bool has_eui64 = memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
3854         struct nvme_ns_head *h;
3855
3856         lockdep_assert_held(&subsys->lock);
3857
3858         list_for_each_entry(h, &subsys->nsheads, entry) {
3859                 if (has_uuid && uuid_equal(&ids->uuid, &h->ids.uuid))
3860                         return -EINVAL;
3861                 if (has_nguid &&
3862                     memcmp(&ids->nguid, &h->ids.nguid, sizeof(ids->nguid)) == 0)
3863                         return -EINVAL;
3864                 if (has_eui64 &&
3865                     memcmp(&ids->eui64, &h->ids.eui64, sizeof(ids->eui64)) == 0)
3866                         return -EINVAL;
3867         }
3868
3869         return 0;
3870 }
3871
3872 static void nvme_cdev_rel(struct device *dev)
3873 {
3874         ida_free(&nvme_ns_chr_minor_ida, MINOR(dev->devt));
3875 }
3876
3877 void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device)
3878 {
3879         cdev_device_del(cdev, cdev_device);
3880         put_device(cdev_device);
3881 }
3882
3883 int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device,
3884                 const struct file_operations *fops, struct module *owner)
3885 {
3886         int minor, ret;
3887
3888         minor = ida_alloc(&nvme_ns_chr_minor_ida, GFP_KERNEL);
3889         if (minor < 0)
3890                 return minor;
3891         cdev_device->devt = MKDEV(MAJOR(nvme_ns_chr_devt), minor);
3892         cdev_device->class = nvme_ns_chr_class;
3893         cdev_device->release = nvme_cdev_rel;
3894         device_initialize(cdev_device);
3895         cdev_init(cdev, fops);
3896         cdev->owner = owner;
3897         ret = cdev_device_add(cdev, cdev_device);
3898         if (ret)
3899                 put_device(cdev_device);
3900
3901         return ret;
3902 }
3903
3904 static int nvme_ns_chr_open(struct inode *inode, struct file *file)
3905 {
3906         return nvme_ns_open(container_of(inode->i_cdev, struct nvme_ns, cdev));
3907 }
3908
3909 static int nvme_ns_chr_release(struct inode *inode, struct file *file)
3910 {
3911         nvme_ns_release(container_of(inode->i_cdev, struct nvme_ns, cdev));
3912         return 0;
3913 }
3914
3915 static const struct file_operations nvme_ns_chr_fops = {
3916         .owner          = THIS_MODULE,
3917         .open           = nvme_ns_chr_open,
3918         .release        = nvme_ns_chr_release,
3919         .unlocked_ioctl = nvme_ns_chr_ioctl,
3920         .compat_ioctl   = compat_ptr_ioctl,
3921         .uring_cmd      = nvme_ns_chr_uring_cmd,
3922         .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll,
3923 };
3924
3925 static int nvme_add_ns_cdev(struct nvme_ns *ns)
3926 {
3927         int ret;
3928
3929         ns->cdev_device.parent = ns->ctrl->device;
3930         ret = dev_set_name(&ns->cdev_device, "ng%dn%d",
3931                            ns->ctrl->instance, ns->head->instance);
3932         if (ret)
3933                 return ret;
3934
3935         return nvme_cdev_add(&ns->cdev, &ns->cdev_device, &nvme_ns_chr_fops,
3936                              ns->ctrl->ops->module);
3937 }
3938
3939 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
3940                 struct nvme_ns_info *info)
3941 {
3942         struct nvme_ns_head *head;
3943         size_t size = sizeof(*head);
3944         int ret = -ENOMEM;
3945
3946 #ifdef CONFIG_NVME_MULTIPATH
3947         size += num_possible_nodes() * sizeof(struct nvme_ns *);
3948 #endif
3949
3950         head = kzalloc(size, GFP_KERNEL);
3951         if (!head)
3952                 goto out;
3953         ret = ida_alloc_min(&ctrl->subsys->ns_ida, 1, GFP_KERNEL);
3954         if (ret < 0)
3955                 goto out_free_head;
3956         head->instance = ret;
3957         INIT_LIST_HEAD(&head->list);
3958         ret = init_srcu_struct(&head->srcu);
3959         if (ret)
3960                 goto out_ida_remove;
3961         head->subsys = ctrl->subsys;
3962         head->ns_id = info->nsid;
3963         head->ids = info->ids;
3964         head->shared = info->is_shared;
3965         kref_init(&head->ref);
3966
3967         if (head->ids.csi) {
3968                 ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects);
3969                 if (ret)
3970                         goto out_cleanup_srcu;
3971         } else
3972                 head->effects = ctrl->effects;
3973
3974         ret = nvme_mpath_alloc_disk(ctrl, head);
3975         if (ret)
3976                 goto out_cleanup_srcu;
3977
3978         list_add_tail(&head->entry, &ctrl->subsys->nsheads);
3979
3980         kref_get(&ctrl->subsys->ref);
3981
3982         return head;
3983 out_cleanup_srcu:
3984         cleanup_srcu_struct(&head->srcu);
3985 out_ida_remove:
3986         ida_free(&ctrl->subsys->ns_ida, head->instance);
3987 out_free_head:
3988         kfree(head);
3989 out:
3990         if (ret > 0)
3991                 ret = blk_status_to_errno(nvme_error_status(ret));
3992         return ERR_PTR(ret);
3993 }
3994
3995 static int nvme_global_check_duplicate_ids(struct nvme_subsystem *this,
3996                 struct nvme_ns_ids *ids)
3997 {
3998         struct nvme_subsystem *s;
3999         int ret = 0;
4000
4001         /*
4002          * Note that this check is racy as we try to avoid holding the global
4003          * lock over the whole ns_head creation.  But it is only intended as
4004          * a sanity check anyway.
4005          */
4006         mutex_lock(&nvme_subsystems_lock);
4007         list_for_each_entry(s, &nvme_subsystems, entry) {
4008                 if (s == this)
4009                         continue;
4010                 mutex_lock(&s->lock);
4011                 ret = nvme_subsys_check_duplicate_ids(s, ids);
4012                 mutex_unlock(&s->lock);
4013                 if (ret)
4014                         break;
4015         }
4016         mutex_unlock(&nvme_subsystems_lock);
4017
4018         return ret;
4019 }
4020
4021 static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
4022 {
4023         struct nvme_ctrl *ctrl = ns->ctrl;
4024         struct nvme_ns_head *head = NULL;
4025         int ret;
4026
4027         ret = nvme_global_check_duplicate_ids(ctrl->subsys, &info->ids);
4028         if (ret) {
4029                 dev_err(ctrl->device,
4030                         "globally duplicate IDs for nsid %d\n", info->nsid);
4031                 nvme_print_device_info(ctrl);
4032                 return ret;
4033         }
4034
4035         mutex_lock(&ctrl->subsys->lock);
4036         head = nvme_find_ns_head(ctrl, info->nsid);
4037         if (!head) {
4038                 ret = nvme_subsys_check_duplicate_ids(ctrl->subsys, &info->ids);
4039                 if (ret) {
4040                         dev_err(ctrl->device,
4041                                 "duplicate IDs in subsystem for nsid %d\n",
4042                                 info->nsid);
4043                         goto out_unlock;
4044                 }
4045                 head = nvme_alloc_ns_head(ctrl, info);
4046                 if (IS_ERR(head)) {
4047                         ret = PTR_ERR(head);
4048                         goto out_unlock;
4049                 }
4050         } else {
4051                 ret = -EINVAL;
4052                 if (!info->is_shared || !head->shared) {
4053                         dev_err(ctrl->device,
4054                                 "Duplicate unshared namespace %d\n",
4055                                 info->nsid);
4056                         goto out_put_ns_head;
4057                 }
4058                 if (!nvme_ns_ids_equal(&head->ids, &info->ids)) {
4059                         dev_err(ctrl->device,
4060                                 "IDs don't match for shared namespace %d\n",
4061                                         info->nsid);
4062                         goto out_put_ns_head;
4063                 }
4064
4065                 if (!multipath && !list_empty(&head->list)) {
4066                         dev_warn(ctrl->device,
4067                                 "Found shared namespace %d, but multipathing not supported.\n",
4068                                 info->nsid);
4069                         dev_warn_once(ctrl->device,
4070                                 "Support for shared namespaces without CONFIG_NVME_MULTIPATH is deprecated and will be removed in Linux 6.0\n.");
4071                 }
4072         }
4073
4074         list_add_tail_rcu(&ns->siblings, &head->list);
4075         ns->head = head;
4076         mutex_unlock(&ctrl->subsys->lock);
4077         return 0;
4078
4079 out_put_ns_head:
4080         nvme_put_ns_head(head);
4081 out_unlock:
4082         mutex_unlock(&ctrl->subsys->lock);
4083         return ret;
4084 }
4085
4086 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
4087 {
4088         struct nvme_ns *ns, *ret = NULL;
4089
4090         down_read(&ctrl->namespaces_rwsem);
4091         list_for_each_entry(ns, &ctrl->namespaces, list) {
4092                 if (ns->head->ns_id == nsid) {
4093                         if (!nvme_get_ns(ns))
4094                                 continue;
4095                         ret = ns;
4096                         break;
4097                 }
4098                 if (ns->head->ns_id > nsid)
4099                         break;
4100         }
4101         up_read(&ctrl->namespaces_rwsem);
4102         return ret;
4103 }
4104 EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
4105
4106 /*
4107  * Add the namespace to the controller list while keeping the list ordered.
4108  */
4109 static void nvme_ns_add_to_ctrl_list(struct nvme_ns *ns)
4110 {
4111         struct nvme_ns *tmp;
4112
4113         list_for_each_entry_reverse(tmp, &ns->ctrl->namespaces, list) {
4114                 if (tmp->head->ns_id < ns->head->ns_id) {
4115                         list_add(&ns->list, &tmp->list);
4116                         return;
4117                 }
4118         }
4119         list_add(&ns->list, &ns->ctrl->namespaces);
4120 }
4121
4122 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info)
4123 {
4124         struct nvme_ns *ns;
4125         struct gendisk *disk;
4126         int node = ctrl->numa_node;
4127
4128         ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
4129         if (!ns)
4130                 return;
4131
4132         disk = blk_mq_alloc_disk(ctrl->tagset, ns);
4133         if (IS_ERR(disk))
4134                 goto out_free_ns;
4135         disk->fops = &nvme_bdev_ops;
4136         disk->private_data = ns;
4137
4138         ns->disk = disk;
4139         ns->queue = disk->queue;
4140
4141         if (ctrl->opts && ctrl->opts->data_digest)
4142                 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, ns->queue);
4143
4144         blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
4145         if (ctrl->ops->supports_pci_p2pdma &&
4146             ctrl->ops->supports_pci_p2pdma(ctrl))
4147                 blk_queue_flag_set(QUEUE_FLAG_PCI_P2PDMA, ns->queue);
4148
4149         ns->ctrl = ctrl;
4150         kref_init(&ns->kref);
4151
4152         if (nvme_init_ns_head(ns, info))
4153                 goto out_cleanup_disk;
4154
4155         /*
4156          * If multipathing is enabled, the device name for all disks and not
4157          * just those that represent shared namespaces needs to be based on the
4158          * subsystem instance.  Using the controller instance for private
4159          * namespaces could lead to naming collisions between shared and private
4160          * namespaces if they don't use a common numbering scheme.
4161          *
4162          * If multipathing is not enabled, disk names must use the controller
4163          * instance as shared namespaces will show up as multiple block
4164          * devices.
4165          */
4166         if (ns->head->disk) {
4167                 sprintf(disk->disk_name, "nvme%dc%dn%d", ctrl->subsys->instance,
4168                         ctrl->instance, ns->head->instance);
4169                 disk->flags |= GENHD_FL_HIDDEN;
4170         } else if (multipath) {
4171                 sprintf(disk->disk_name, "nvme%dn%d", ctrl->subsys->instance,
4172                         ns->head->instance);
4173         } else {
4174                 sprintf(disk->disk_name, "nvme%dn%d", ctrl->instance,
4175                         ns->head->instance);
4176         }
4177
4178         if (nvme_update_ns_info(ns, info))
4179                 goto out_unlink_ns;
4180
4181         down_write(&ctrl->namespaces_rwsem);
4182         nvme_ns_add_to_ctrl_list(ns);
4183         up_write(&ctrl->namespaces_rwsem);
4184         nvme_get_ctrl(ctrl);
4185
4186         if (device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups))
4187                 goto out_cleanup_ns_from_list;
4188
4189         if (!nvme_ns_head_multipath(ns->head))
4190                 nvme_add_ns_cdev(ns);
4191
4192         nvme_mpath_add_disk(ns, info->anagrpid);
4193         nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
4194
4195         return;
4196
4197  out_cleanup_ns_from_list:
4198         nvme_put_ctrl(ctrl);
4199         down_write(&ctrl->namespaces_rwsem);
4200         list_del_init(&ns->list);
4201         up_write(&ctrl->namespaces_rwsem);
4202  out_unlink_ns:
4203         mutex_lock(&ctrl->subsys->lock);
4204         list_del_rcu(&ns->siblings);
4205         if (list_empty(&ns->head->list))
4206                 list_del_init(&ns->head->entry);
4207         mutex_unlock(&ctrl->subsys->lock);
4208         nvme_put_ns_head(ns->head);
4209  out_cleanup_disk:
4210         put_disk(disk);
4211  out_free_ns:
4212         kfree(ns);
4213 }
4214
4215 static void nvme_ns_remove(struct nvme_ns *ns)
4216 {
4217         bool last_path = false;
4218
4219         if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
4220                 return;
4221
4222         clear_bit(NVME_NS_READY, &ns->flags);
4223         set_capacity(ns->disk, 0);
4224         nvme_fault_inject_fini(&ns->fault_inject);
4225
4226         /*
4227          * Ensure that !NVME_NS_READY is seen by other threads to prevent
4228          * this ns going back into current_path.
4229          */
4230         synchronize_srcu(&ns->head->srcu);
4231
4232         /* wait for concurrent submissions */
4233         if (nvme_mpath_clear_current_path(ns))
4234                 synchronize_srcu(&ns->head->srcu);
4235
4236         mutex_lock(&ns->ctrl->subsys->lock);
4237         list_del_rcu(&ns->siblings);
4238         if (list_empty(&ns->head->list)) {
4239                 list_del_init(&ns->head->entry);
4240                 last_path = true;
4241         }
4242         mutex_unlock(&ns->ctrl->subsys->lock);
4243
4244         /* guarantee not available in head->list */
4245         synchronize_srcu(&ns->head->srcu);
4246
4247         if (!nvme_ns_head_multipath(ns->head))
4248                 nvme_cdev_del(&ns->cdev, &ns->cdev_device);
4249         del_gendisk(ns->disk);
4250
4251         down_write(&ns->ctrl->namespaces_rwsem);
4252         list_del_init(&ns->list);
4253         up_write(&ns->ctrl->namespaces_rwsem);
4254
4255         if (last_path)
4256                 nvme_mpath_shutdown_disk(ns->head);
4257         nvme_put_ns(ns);
4258 }
4259
4260 static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid)
4261 {
4262         struct nvme_ns *ns = nvme_find_get_ns(ctrl, nsid);
4263
4264         if (ns) {
4265                 nvme_ns_remove(ns);
4266                 nvme_put_ns(ns);
4267         }
4268 }
4269
4270 static void nvme_validate_ns(struct nvme_ns *ns, struct nvme_ns_info *info)
4271 {
4272         int ret = NVME_SC_INVALID_NS | NVME_SC_DNR;
4273
4274         if (!nvme_ns_ids_equal(&ns->head->ids, &info->ids)) {
4275                 dev_err(ns->ctrl->device,
4276                         "identifiers changed for nsid %d\n", ns->head->ns_id);
4277                 goto out;
4278         }
4279
4280         ret = nvme_update_ns_info(ns, info);
4281 out:
4282         /*
4283          * Only remove the namespace if we got a fatal error back from the
4284          * device, otherwise ignore the error and just move on.
4285          *
4286          * TODO: we should probably schedule a delayed retry here.
4287          */
4288         if (ret > 0 && (ret & NVME_SC_DNR))
4289                 nvme_ns_remove(ns);
4290 }
4291
4292 static void nvme_scan_ns(struct nvme_ctrl *ctrl, unsigned nsid)
4293 {
4294         struct nvme_ns_info info = { .nsid = nsid };
4295         struct nvme_ns *ns;
4296         int ret;
4297
4298         if (nvme_identify_ns_descs(ctrl, &info))
4299                 return;
4300
4301         if (info.ids.csi != NVME_CSI_NVM && !nvme_multi_css(ctrl)) {
4302                 dev_warn(ctrl->device,
4303                         "command set not reported for nsid: %d\n", nsid);
4304                 return;
4305         }
4306
4307         /*
4308          * If available try to use the Command Set Idependent Identify Namespace
4309          * data structure to find all the generic information that is needed to
4310          * set up a namespace.  If not fall back to the legacy version.
4311          */
4312         if ((ctrl->cap & NVME_CAP_CRMS_CRIMS) ||
4313             (info.ids.csi != NVME_CSI_NVM && info.ids.csi != NVME_CSI_ZNS))
4314                 ret = nvme_ns_info_from_id_cs_indep(ctrl, &info);
4315         else
4316                 ret = nvme_ns_info_from_identify(ctrl, &info);
4317
4318         if (info.is_removed)
4319                 nvme_ns_remove_by_nsid(ctrl, nsid);
4320
4321         /*
4322          * Ignore the namespace if it is not ready. We will get an AEN once it
4323          * becomes ready and restart the scan.
4324          */
4325         if (ret || !info.is_ready)
4326                 return;
4327
4328         ns = nvme_find_get_ns(ctrl, nsid);
4329         if (ns) {
4330                 nvme_validate_ns(ns, &info);
4331                 nvme_put_ns(ns);
4332         } else {
4333                 nvme_alloc_ns(ctrl, &info);
4334         }
4335 }
4336
4337 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
4338                                         unsigned nsid)
4339 {
4340         struct nvme_ns *ns, *next;
4341         LIST_HEAD(rm_list);
4342
4343         down_write(&ctrl->namespaces_rwsem);
4344         list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
4345                 if (ns->head->ns_id > nsid)
4346                         list_move_tail(&ns->list, &rm_list);
4347         }
4348         up_write(&ctrl->namespaces_rwsem);
4349
4350         list_for_each_entry_safe(ns, next, &rm_list, list)
4351                 nvme_ns_remove(ns);
4352
4353 }
4354
4355 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl)
4356 {
4357         const int nr_entries = NVME_IDENTIFY_DATA_SIZE / sizeof(__le32);
4358         __le32 *ns_list;
4359         u32 prev = 0;
4360         int ret = 0, i;
4361
4362         ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
4363         if (!ns_list)
4364                 return -ENOMEM;
4365
4366         for (;;) {
4367                 struct nvme_command cmd = {
4368                         .identify.opcode        = nvme_admin_identify,
4369                         .identify.cns           = NVME_ID_CNS_NS_ACTIVE_LIST,
4370                         .identify.nsid          = cpu_to_le32(prev),
4371                 };
4372
4373                 ret = nvme_submit_sync_cmd(ctrl->admin_q, &cmd, ns_list,
4374                                             NVME_IDENTIFY_DATA_SIZE);
4375                 if (ret) {
4376                         dev_warn(ctrl->device,
4377                                 "Identify NS List failed (status=0x%x)\n", ret);
4378                         goto free;
4379                 }
4380
4381                 for (i = 0; i < nr_entries; i++) {
4382                         u32 nsid = le32_to_cpu(ns_list[i]);
4383
4384                         if (!nsid)      /* end of the list? */
4385                                 goto out;
4386                         nvme_scan_ns(ctrl, nsid);
4387                         while (++prev < nsid)
4388                                 nvme_ns_remove_by_nsid(ctrl, prev);
4389                 }
4390         }
4391  out:
4392         nvme_remove_invalid_namespaces(ctrl, prev);
4393  free:
4394         kfree(ns_list);
4395         return ret;
4396 }
4397
4398 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl)
4399 {
4400         struct nvme_id_ctrl *id;
4401         u32 nn, i;
4402
4403         if (nvme_identify_ctrl(ctrl, &id))
4404                 return;
4405         nn = le32_to_cpu(id->nn);
4406         kfree(id);
4407
4408         for (i = 1; i <= nn; i++)
4409                 nvme_scan_ns(ctrl, i);
4410
4411         nvme_remove_invalid_namespaces(ctrl, nn);
4412 }
4413
4414 static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
4415 {
4416         size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32);
4417         __le32 *log;
4418         int error;
4419
4420         log = kzalloc(log_size, GFP_KERNEL);
4421         if (!log)
4422                 return;
4423
4424         /*
4425          * We need to read the log to clear the AEN, but we don't want to rely
4426          * on it for the changed namespace information as userspace could have
4427          * raced with us in reading the log page, which could cause us to miss
4428          * updates.
4429          */
4430         error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0,
4431                         NVME_CSI_NVM, log, log_size, 0);
4432         if (error)
4433                 dev_warn(ctrl->device,
4434                         "reading changed ns log failed: %d\n", error);
4435
4436         kfree(log);
4437 }
4438
4439 static void nvme_scan_work(struct work_struct *work)
4440 {
4441         struct nvme_ctrl *ctrl =
4442                 container_of(work, struct nvme_ctrl, scan_work);
4443         int ret;
4444
4445         /* No tagset on a live ctrl means IO queues could not created */
4446         if (ctrl->state != NVME_CTRL_LIVE || !ctrl->tagset)
4447                 return;
4448
4449         /*
4450          * Identify controller limits can change at controller reset due to
4451          * new firmware download, even though it is not common we cannot ignore
4452          * such scenario. Controller's non-mdts limits are reported in the unit
4453          * of logical blocks that is dependent on the format of attached
4454          * namespace. Hence re-read the limits at the time of ns allocation.
4455          */
4456         ret = nvme_init_non_mdts_limits(ctrl);
4457         if (ret < 0) {
4458                 dev_warn(ctrl->device,
4459                         "reading non-mdts-limits failed: %d\n", ret);
4460                 return;
4461         }
4462
4463         if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
4464                 dev_info(ctrl->device, "rescanning namespaces.\n");
4465                 nvme_clear_changed_ns_log(ctrl);
4466         }
4467
4468         mutex_lock(&ctrl->scan_lock);
4469         if (nvme_ctrl_limited_cns(ctrl)) {
4470                 nvme_scan_ns_sequential(ctrl);
4471         } else {
4472                 /*
4473                  * Fall back to sequential scan if DNR is set to handle broken
4474                  * devices which should support Identify NS List (as per the VS
4475                  * they report) but don't actually support it.
4476                  */
4477                 ret = nvme_scan_ns_list(ctrl);
4478                 if (ret > 0 && ret & NVME_SC_DNR)
4479                         nvme_scan_ns_sequential(ctrl);
4480         }
4481         mutex_unlock(&ctrl->scan_lock);
4482 }
4483
4484 /*
4485  * This function iterates the namespace list unlocked to allow recovery from
4486  * controller failure. It is up to the caller to ensure the namespace list is
4487  * not modified by scan work while this function is executing.
4488  */
4489 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
4490 {
4491         struct nvme_ns *ns, *next;
4492         LIST_HEAD(ns_list);
4493
4494         /*
4495          * make sure to requeue I/O to all namespaces as these
4496          * might result from the scan itself and must complete
4497          * for the scan_work to make progress
4498          */
4499         nvme_mpath_clear_ctrl_paths(ctrl);
4500
4501         /* prevent racing with ns scanning */
4502         flush_work(&ctrl->scan_work);
4503
4504         /*
4505          * The dead states indicates the controller was not gracefully
4506          * disconnected. In that case, we won't be able to flush any data while
4507          * removing the namespaces' disks; fail all the queues now to avoid
4508          * potentially having to clean up the failed sync later.
4509          */
4510         if (ctrl->state == NVME_CTRL_DEAD) {
4511                 nvme_mark_namespaces_dead(ctrl);
4512                 nvme_unquiesce_io_queues(ctrl);
4513         }
4514
4515         /* this is a no-op when called from the controller reset handler */
4516         nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO);
4517
4518         down_write(&ctrl->namespaces_rwsem);
4519         list_splice_init(&ctrl->namespaces, &ns_list);
4520         up_write(&ctrl->namespaces_rwsem);
4521
4522         list_for_each_entry_safe(ns, next, &ns_list, list)
4523                 nvme_ns_remove(ns);
4524 }
4525 EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
4526
4527 static int nvme_class_uevent(const struct device *dev, struct kobj_uevent_env *env)
4528 {
4529         const struct nvme_ctrl *ctrl =
4530                 container_of(dev, struct nvme_ctrl, ctrl_device);
4531         struct nvmf_ctrl_options *opts = ctrl->opts;
4532         int ret;
4533
4534         ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);
4535         if (ret)
4536                 return ret;
4537
4538         if (opts) {
4539                 ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);
4540                 if (ret)
4541                         return ret;
4542
4543                 ret = add_uevent_var(env, "NVME_TRSVCID=%s",
4544                                 opts->trsvcid ?: "none");
4545                 if (ret)
4546                         return ret;
4547
4548                 ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",
4549                                 opts->host_traddr ?: "none");
4550                 if (ret)
4551                         return ret;
4552
4553                 ret = add_uevent_var(env, "NVME_HOST_IFACE=%s",
4554                                 opts->host_iface ?: "none");
4555         }
4556         return ret;
4557 }
4558
4559 static void nvme_change_uevent(struct nvme_ctrl *ctrl, char *envdata)
4560 {
4561         char *envp[2] = { envdata, NULL };
4562
4563         kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4564 }
4565
4566 static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
4567 {
4568         char *envp[2] = { NULL, NULL };
4569         u32 aen_result = ctrl->aen_result;
4570
4571         ctrl->aen_result = 0;
4572         if (!aen_result)
4573                 return;
4574
4575         envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
4576         if (!envp[0])
4577                 return;
4578         kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4579         kfree(envp[0]);
4580 }
4581
4582 static void nvme_async_event_work(struct work_struct *work)
4583 {
4584         struct nvme_ctrl *ctrl =
4585                 container_of(work, struct nvme_ctrl, async_event_work);
4586
4587         nvme_aen_uevent(ctrl);
4588
4589         /*
4590          * The transport drivers must guarantee AER submission here is safe by
4591          * flushing ctrl async_event_work after changing the controller state
4592          * from LIVE and before freeing the admin queue.
4593         */
4594         if (ctrl->state == NVME_CTRL_LIVE)
4595                 ctrl->ops->submit_async_event(ctrl);
4596 }
4597
4598 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
4599 {
4600
4601         u32 csts;
4602
4603         if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
4604                 return false;
4605
4606         if (csts == ~0)
4607                 return false;
4608
4609         return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
4610 }
4611
4612 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
4613 {
4614         struct nvme_fw_slot_info_log *log;
4615
4616         log = kmalloc(sizeof(*log), GFP_KERNEL);
4617         if (!log)
4618                 return;
4619
4620         if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, NVME_CSI_NVM,
4621                         log, sizeof(*log), 0))
4622                 dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
4623         kfree(log);
4624 }
4625
4626 static void nvme_fw_act_work(struct work_struct *work)
4627 {
4628         struct nvme_ctrl *ctrl = container_of(work,
4629                                 struct nvme_ctrl, fw_act_work);
4630         unsigned long fw_act_timeout;
4631
4632         if (ctrl->mtfa)
4633                 fw_act_timeout = jiffies +
4634                                 msecs_to_jiffies(ctrl->mtfa * 100);
4635         else
4636                 fw_act_timeout = jiffies +
4637                                 msecs_to_jiffies(admin_timeout * 1000);
4638
4639         nvme_quiesce_io_queues(ctrl);
4640         while (nvme_ctrl_pp_status(ctrl)) {
4641                 if (time_after(jiffies, fw_act_timeout)) {
4642                         dev_warn(ctrl->device,
4643                                 "Fw activation timeout, reset controller\n");
4644                         nvme_try_sched_reset(ctrl);
4645                         return;
4646                 }
4647                 msleep(100);
4648         }
4649
4650         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
4651                 return;
4652
4653         nvme_unquiesce_io_queues(ctrl);
4654         /* read FW slot information to clear the AER */
4655         nvme_get_fw_slot_info(ctrl);
4656
4657         queue_work(nvme_wq, &ctrl->async_event_work);
4658 }
4659
4660 static u32 nvme_aer_type(u32 result)
4661 {
4662         return result & 0x7;
4663 }
4664
4665 static u32 nvme_aer_subtype(u32 result)
4666 {
4667         return (result & 0xff00) >> 8;
4668 }
4669
4670 static bool nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
4671 {
4672         u32 aer_notice_type = nvme_aer_subtype(result);
4673         bool requeue = true;
4674
4675         switch (aer_notice_type) {
4676         case NVME_AER_NOTICE_NS_CHANGED:
4677                 set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
4678                 nvme_queue_scan(ctrl);
4679                 break;
4680         case NVME_AER_NOTICE_FW_ACT_STARTING:
4681                 /*
4682                  * We are (ab)using the RESETTING state to prevent subsequent
4683                  * recovery actions from interfering with the controller's
4684                  * firmware activation.
4685                  */
4686                 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) {
4687                         nvme_auth_stop(ctrl);
4688                         requeue = false;
4689                         queue_work(nvme_wq, &ctrl->fw_act_work);
4690                 }
4691                 break;
4692 #ifdef CONFIG_NVME_MULTIPATH
4693         case NVME_AER_NOTICE_ANA:
4694                 if (!ctrl->ana_log_buf)
4695                         break;
4696                 queue_work(nvme_wq, &ctrl->ana_work);
4697                 break;
4698 #endif
4699         case NVME_AER_NOTICE_DISC_CHANGED:
4700                 ctrl->aen_result = result;
4701                 break;
4702         default:
4703                 dev_warn(ctrl->device, "async event result %08x\n", result);
4704         }
4705         return requeue;
4706 }
4707
4708 static void nvme_handle_aer_persistent_error(struct nvme_ctrl *ctrl)
4709 {
4710         dev_warn(ctrl->device, "resetting controller due to AER\n");
4711         nvme_reset_ctrl(ctrl);
4712 }
4713
4714 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
4715                 volatile union nvme_result *res)
4716 {
4717         u32 result = le32_to_cpu(res->u32);
4718         u32 aer_type = nvme_aer_type(result);
4719         u32 aer_subtype = nvme_aer_subtype(result);
4720         bool requeue = true;
4721
4722         if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
4723                 return;
4724
4725         trace_nvme_async_event(ctrl, result);
4726         switch (aer_type) {
4727         case NVME_AER_NOTICE:
4728                 requeue = nvme_handle_aen_notice(ctrl, result);
4729                 break;
4730         case NVME_AER_ERROR:
4731                 /*
4732                  * For a persistent internal error, don't run async_event_work
4733                  * to submit a new AER. The controller reset will do it.
4734                  */
4735                 if (aer_subtype == NVME_AER_ERROR_PERSIST_INT_ERR) {
4736                         nvme_handle_aer_persistent_error(ctrl);
4737                         return;
4738                 }
4739                 fallthrough;
4740         case NVME_AER_SMART:
4741         case NVME_AER_CSS:
4742         case NVME_AER_VS:
4743                 ctrl->aen_result = result;
4744                 break;
4745         default:
4746                 break;
4747         }
4748
4749         if (requeue)
4750                 queue_work(nvme_wq, &ctrl->async_event_work);
4751 }
4752 EXPORT_SYMBOL_GPL(nvme_complete_async_event);
4753
4754 int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
4755                 const struct blk_mq_ops *ops, unsigned int cmd_size)
4756 {
4757         int ret;
4758
4759         memset(set, 0, sizeof(*set));
4760         set->ops = ops;
4761         set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
4762         if (ctrl->ops->flags & NVME_F_FABRICS)
4763                 set->reserved_tags = NVMF_RESERVED_TAGS;
4764         set->numa_node = ctrl->numa_node;
4765         set->flags = BLK_MQ_F_NO_SCHED;
4766         if (ctrl->ops->flags & NVME_F_BLOCKING)
4767                 set->flags |= BLK_MQ_F_BLOCKING;
4768         set->cmd_size = cmd_size;
4769         set->driver_data = ctrl;
4770         set->nr_hw_queues = 1;
4771         set->timeout = NVME_ADMIN_TIMEOUT;
4772         ret = blk_mq_alloc_tag_set(set);
4773         if (ret)
4774                 return ret;
4775
4776         ctrl->admin_q = blk_mq_init_queue(set);
4777         if (IS_ERR(ctrl->admin_q)) {
4778                 ret = PTR_ERR(ctrl->admin_q);
4779                 goto out_free_tagset;
4780         }
4781
4782         if (ctrl->ops->flags & NVME_F_FABRICS) {
4783                 ctrl->fabrics_q = blk_mq_init_queue(set);
4784                 if (IS_ERR(ctrl->fabrics_q)) {
4785                         ret = PTR_ERR(ctrl->fabrics_q);
4786                         goto out_cleanup_admin_q;
4787                 }
4788         }
4789
4790         ctrl->admin_tagset = set;
4791         return 0;
4792
4793 out_cleanup_admin_q:
4794         blk_mq_destroy_queue(ctrl->admin_q);
4795         blk_put_queue(ctrl->admin_q);
4796 out_free_tagset:
4797         blk_mq_free_tag_set(set);
4798         ctrl->admin_q = NULL;
4799         ctrl->fabrics_q = NULL;
4800         return ret;
4801 }
4802 EXPORT_SYMBOL_GPL(nvme_alloc_admin_tag_set);
4803
4804 void nvme_remove_admin_tag_set(struct nvme_ctrl *ctrl)
4805 {
4806         blk_mq_destroy_queue(ctrl->admin_q);
4807         blk_put_queue(ctrl->admin_q);
4808         if (ctrl->ops->flags & NVME_F_FABRICS) {
4809                 blk_mq_destroy_queue(ctrl->fabrics_q);
4810                 blk_put_queue(ctrl->fabrics_q);
4811         }
4812         blk_mq_free_tag_set(ctrl->admin_tagset);
4813 }
4814 EXPORT_SYMBOL_GPL(nvme_remove_admin_tag_set);
4815
4816 int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
4817                 const struct blk_mq_ops *ops, unsigned int nr_maps,
4818                 unsigned int cmd_size)
4819 {
4820         int ret;
4821
4822         memset(set, 0, sizeof(*set));
4823         set->ops = ops;
4824         set->queue_depth = min_t(unsigned, ctrl->sqsize, BLK_MQ_MAX_DEPTH - 1);
4825         /*
4826          * Some Apple controllers requires tags to be unique across admin and
4827          * the (only) I/O queue, so reserve the first 32 tags of the I/O queue.
4828          */
4829         if (ctrl->quirks & NVME_QUIRK_SHARED_TAGS)
4830                 set->reserved_tags = NVME_AQ_DEPTH;
4831         else if (ctrl->ops->flags & NVME_F_FABRICS)
4832                 set->reserved_tags = NVMF_RESERVED_TAGS;
4833         set->numa_node = ctrl->numa_node;
4834         set->flags = BLK_MQ_F_SHOULD_MERGE;
4835         if (ctrl->ops->flags & NVME_F_BLOCKING)
4836                 set->flags |= BLK_MQ_F_BLOCKING;
4837         set->cmd_size = cmd_size,
4838         set->driver_data = ctrl;
4839         set->nr_hw_queues = ctrl->queue_count - 1;
4840         set->timeout = NVME_IO_TIMEOUT;
4841         set->nr_maps = nr_maps;
4842         ret = blk_mq_alloc_tag_set(set);
4843         if (ret)
4844                 return ret;
4845
4846         if (ctrl->ops->flags & NVME_F_FABRICS) {
4847                 ctrl->connect_q = blk_mq_init_queue(set);
4848                 if (IS_ERR(ctrl->connect_q)) {
4849                         ret = PTR_ERR(ctrl->connect_q);
4850                         goto out_free_tag_set;
4851                 }
4852                 blk_queue_flag_set(QUEUE_FLAG_SKIP_TAGSET_QUIESCE,
4853                                    ctrl->connect_q);
4854         }
4855
4856         ctrl->tagset = set;
4857         return 0;
4858
4859 out_free_tag_set:
4860         blk_mq_free_tag_set(set);
4861         ctrl->connect_q = NULL;
4862         return ret;
4863 }
4864 EXPORT_SYMBOL_GPL(nvme_alloc_io_tag_set);
4865
4866 void nvme_remove_io_tag_set(struct nvme_ctrl *ctrl)
4867 {
4868         if (ctrl->ops->flags & NVME_F_FABRICS) {
4869                 blk_mq_destroy_queue(ctrl->connect_q);
4870                 blk_put_queue(ctrl->connect_q);
4871         }
4872         blk_mq_free_tag_set(ctrl->tagset);
4873 }
4874 EXPORT_SYMBOL_GPL(nvme_remove_io_tag_set);
4875
4876 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
4877 {
4878         nvme_mpath_stop(ctrl);
4879         nvme_auth_stop(ctrl);
4880         nvme_stop_keep_alive(ctrl);
4881         nvme_stop_failfast_work(ctrl);
4882         flush_work(&ctrl->async_event_work);
4883         cancel_work_sync(&ctrl->fw_act_work);
4884         if (ctrl->ops->stop_ctrl)
4885                 ctrl->ops->stop_ctrl(ctrl);
4886 }
4887 EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
4888
4889 void nvme_start_ctrl(struct nvme_ctrl *ctrl)
4890 {
4891         nvme_start_keep_alive(ctrl);
4892
4893         nvme_enable_aen(ctrl);
4894
4895         /*
4896          * persistent discovery controllers need to send indication to userspace
4897          * to re-read the discovery log page to learn about possible changes
4898          * that were missed. We identify persistent discovery controllers by
4899          * checking that they started once before, hence are reconnecting back.
4900          */
4901         if (test_and_set_bit(NVME_CTRL_STARTED_ONCE, &ctrl->flags) &&
4902             nvme_discovery_ctrl(ctrl))
4903                 nvme_change_uevent(ctrl, "NVME_EVENT=rediscover");
4904
4905         if (ctrl->queue_count > 1) {
4906                 nvme_queue_scan(ctrl);
4907                 nvme_unquiesce_io_queues(ctrl);
4908                 nvme_mpath_update(ctrl);
4909         }
4910
4911         nvme_change_uevent(ctrl, "NVME_EVENT=connected");
4912 }
4913 EXPORT_SYMBOL_GPL(nvme_start_ctrl);
4914
4915 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
4916 {
4917         nvme_hwmon_exit(ctrl);
4918         nvme_fault_inject_fini(&ctrl->fault_inject);
4919         dev_pm_qos_hide_latency_tolerance(ctrl->device);
4920         cdev_device_del(&ctrl->cdev, ctrl->device);
4921         nvme_put_ctrl(ctrl);
4922 }
4923 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
4924
4925 static void nvme_free_cels(struct nvme_ctrl *ctrl)
4926 {
4927         struct nvme_effects_log *cel;
4928         unsigned long i;
4929
4930         xa_for_each(&ctrl->cels, i, cel) {
4931                 xa_erase(&ctrl->cels, i);
4932                 kfree(cel);
4933         }
4934
4935         xa_destroy(&ctrl->cels);
4936 }
4937
4938 static void nvme_free_ctrl(struct device *dev)
4939 {
4940         struct nvme_ctrl *ctrl =
4941                 container_of(dev, struct nvme_ctrl, ctrl_device);
4942         struct nvme_subsystem *subsys = ctrl->subsys;
4943
4944         if (!subsys || ctrl->instance != subsys->instance)
4945                 ida_free(&nvme_instance_ida, ctrl->instance);
4946
4947         nvme_free_cels(ctrl);
4948         nvme_mpath_uninit(ctrl);
4949         nvme_auth_stop(ctrl);
4950         nvme_auth_free(ctrl);
4951         __free_page(ctrl->discard_page);
4952         free_opal_dev(ctrl->opal_dev);
4953
4954         if (subsys) {
4955                 mutex_lock(&nvme_subsystems_lock);
4956                 list_del(&ctrl->subsys_entry);
4957                 sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
4958                 mutex_unlock(&nvme_subsystems_lock);
4959         }
4960
4961         ctrl->ops->free_ctrl(ctrl);
4962
4963         if (subsys)
4964                 nvme_put_subsystem(subsys);
4965 }
4966
4967 /*
4968  * Initialize a NVMe controller structures.  This needs to be called during
4969  * earliest initialization so that we have the initialized structured around
4970  * during probing.
4971  */
4972 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
4973                 const struct nvme_ctrl_ops *ops, unsigned long quirks)
4974 {
4975         int ret;
4976
4977         ctrl->state = NVME_CTRL_NEW;
4978         clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
4979         spin_lock_init(&ctrl->lock);
4980         mutex_init(&ctrl->scan_lock);
4981         INIT_LIST_HEAD(&ctrl->namespaces);
4982         xa_init(&ctrl->cels);
4983         init_rwsem(&ctrl->namespaces_rwsem);
4984         ctrl->dev = dev;
4985         ctrl->ops = ops;
4986         ctrl->quirks = quirks;
4987         ctrl->numa_node = NUMA_NO_NODE;
4988         INIT_WORK(&ctrl->scan_work, nvme_scan_work);
4989         INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
4990         INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
4991         INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
4992         init_waitqueue_head(&ctrl->state_wq);
4993
4994         INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
4995         INIT_DELAYED_WORK(&ctrl->failfast_work, nvme_failfast_work);
4996         memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
4997         ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
4998
4999         BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) >
5000                         PAGE_SIZE);
5001         ctrl->discard_page = alloc_page(GFP_KERNEL);
5002         if (!ctrl->discard_page) {
5003                 ret = -ENOMEM;
5004                 goto out;
5005         }
5006
5007         ret = ida_alloc(&nvme_instance_ida, GFP_KERNEL);
5008         if (ret < 0)
5009                 goto out;
5010         ctrl->instance = ret;
5011
5012         device_initialize(&ctrl->ctrl_device);
5013         ctrl->device = &ctrl->ctrl_device;
5014         ctrl->device->devt = MKDEV(MAJOR(nvme_ctrl_base_chr_devt),
5015                         ctrl->instance);
5016         ctrl->device->class = nvme_class;
5017         ctrl->device->parent = ctrl->dev;
5018         if (ops->dev_attr_groups)
5019                 ctrl->device->groups = ops->dev_attr_groups;
5020         else
5021                 ctrl->device->groups = nvme_dev_attr_groups;
5022         ctrl->device->release = nvme_free_ctrl;
5023         dev_set_drvdata(ctrl->device, ctrl);
5024         ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
5025         if (ret)
5026                 goto out_release_instance;
5027
5028         nvme_get_ctrl(ctrl);
5029         cdev_init(&ctrl->cdev, &nvme_dev_fops);
5030         ctrl->cdev.owner = ops->module;
5031         ret = cdev_device_add(&ctrl->cdev, ctrl->device);
5032         if (ret)
5033                 goto out_free_name;
5034
5035         /*
5036          * Initialize latency tolerance controls.  The sysfs files won't
5037          * be visible to userspace unless the device actually supports APST.
5038          */
5039         ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
5040         dev_pm_qos_update_user_latency_tolerance(ctrl->device,
5041                 min(default_ps_max_latency_us, (unsigned long)S32_MAX));
5042
5043         nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
5044         nvme_mpath_init_ctrl(ctrl);
5045         ret = nvme_auth_init_ctrl(ctrl);
5046         if (ret)
5047                 goto out_free_cdev;
5048
5049         return 0;
5050 out_free_cdev:
5051         cdev_device_del(&ctrl->cdev, ctrl->device);
5052 out_free_name:
5053         nvme_put_ctrl(ctrl);
5054         kfree_const(ctrl->device->kobj.name);
5055 out_release_instance:
5056         ida_free(&nvme_instance_ida, ctrl->instance);
5057 out:
5058         if (ctrl->discard_page)
5059                 __free_page(ctrl->discard_page);
5060         return ret;
5061 }
5062 EXPORT_SYMBOL_GPL(nvme_init_ctrl);
5063
5064 /* let I/O to all namespaces fail in preparation for surprise removal */
5065 void nvme_mark_namespaces_dead(struct nvme_ctrl *ctrl)
5066 {
5067         struct nvme_ns *ns;
5068
5069         down_read(&ctrl->namespaces_rwsem);
5070         list_for_each_entry(ns, &ctrl->namespaces, list)
5071                 blk_mark_disk_dead(ns->disk);
5072         up_read(&ctrl->namespaces_rwsem);
5073 }
5074 EXPORT_SYMBOL_GPL(nvme_mark_namespaces_dead);
5075
5076 void nvme_unfreeze(struct nvme_ctrl *ctrl)
5077 {
5078         struct nvme_ns *ns;
5079
5080         down_read(&ctrl->namespaces_rwsem);
5081         list_for_each_entry(ns, &ctrl->namespaces, list)
5082                 blk_mq_unfreeze_queue(ns->queue);
5083         up_read(&ctrl->namespaces_rwsem);
5084 }
5085 EXPORT_SYMBOL_GPL(nvme_unfreeze);
5086
5087 int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
5088 {
5089         struct nvme_ns *ns;
5090
5091         down_read(&ctrl->namespaces_rwsem);
5092         list_for_each_entry(ns, &ctrl->namespaces, list) {
5093                 timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
5094                 if (timeout <= 0)
5095                         break;
5096         }
5097         up_read(&ctrl->namespaces_rwsem);
5098         return timeout;
5099 }
5100 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
5101
5102 void nvme_wait_freeze(struct nvme_ctrl *ctrl)
5103 {
5104         struct nvme_ns *ns;
5105
5106         down_read(&ctrl->namespaces_rwsem);
5107         list_for_each_entry(ns, &ctrl->namespaces, list)
5108                 blk_mq_freeze_queue_wait(ns->queue);
5109         up_read(&ctrl->namespaces_rwsem);
5110 }
5111 EXPORT_SYMBOL_GPL(nvme_wait_freeze);
5112
5113 void nvme_start_freeze(struct nvme_ctrl *ctrl)
5114 {
5115         struct nvme_ns *ns;
5116
5117         down_read(&ctrl->namespaces_rwsem);
5118         list_for_each_entry(ns, &ctrl->namespaces, list)
5119                 blk_freeze_queue_start(ns->queue);
5120         up_read(&ctrl->namespaces_rwsem);
5121 }
5122 EXPORT_SYMBOL_GPL(nvme_start_freeze);
5123
5124 void nvme_quiesce_io_queues(struct nvme_ctrl *ctrl)
5125 {
5126         if (!ctrl->tagset)
5127                 return;
5128         if (!test_and_set_bit(NVME_CTRL_STOPPED, &ctrl->flags))
5129                 blk_mq_quiesce_tagset(ctrl->tagset);
5130         else
5131                 blk_mq_wait_quiesce_done(ctrl->tagset);
5132 }
5133 EXPORT_SYMBOL_GPL(nvme_quiesce_io_queues);
5134
5135 void nvme_unquiesce_io_queues(struct nvme_ctrl *ctrl)
5136 {
5137         if (!ctrl->tagset)
5138                 return;
5139         if (test_and_clear_bit(NVME_CTRL_STOPPED, &ctrl->flags))
5140                 blk_mq_unquiesce_tagset(ctrl->tagset);
5141 }
5142 EXPORT_SYMBOL_GPL(nvme_unquiesce_io_queues);
5143
5144 void nvme_quiesce_admin_queue(struct nvme_ctrl *ctrl)
5145 {
5146         if (!test_and_set_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))
5147                 blk_mq_quiesce_queue(ctrl->admin_q);
5148         else
5149                 blk_mq_wait_quiesce_done(ctrl->admin_q->tag_set);
5150 }
5151 EXPORT_SYMBOL_GPL(nvme_quiesce_admin_queue);
5152
5153 void nvme_unquiesce_admin_queue(struct nvme_ctrl *ctrl)
5154 {
5155         if (test_and_clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))
5156                 blk_mq_unquiesce_queue(ctrl->admin_q);
5157 }
5158 EXPORT_SYMBOL_GPL(nvme_unquiesce_admin_queue);
5159
5160 void nvme_sync_io_queues(struct nvme_ctrl *ctrl)
5161 {
5162         struct nvme_ns *ns;
5163
5164         down_read(&ctrl->namespaces_rwsem);
5165         list_for_each_entry(ns, &ctrl->namespaces, list)
5166                 blk_sync_queue(ns->queue);
5167         up_read(&ctrl->namespaces_rwsem);
5168 }
5169 EXPORT_SYMBOL_GPL(nvme_sync_io_queues);
5170
5171 void nvme_sync_queues(struct nvme_ctrl *ctrl)
5172 {
5173         nvme_sync_io_queues(ctrl);
5174         if (ctrl->admin_q)
5175                 blk_sync_queue(ctrl->admin_q);
5176 }
5177 EXPORT_SYMBOL_GPL(nvme_sync_queues);
5178
5179 struct nvme_ctrl *nvme_ctrl_from_file(struct file *file)
5180 {
5181         if (file->f_op != &nvme_dev_fops)
5182                 return NULL;
5183         return file->private_data;
5184 }
5185 EXPORT_SYMBOL_NS_GPL(nvme_ctrl_from_file, NVME_TARGET_PASSTHRU);
5186
5187 /*
5188  * Check we didn't inadvertently grow the command structure sizes:
5189  */
5190 static inline void _nvme_check_size(void)
5191 {
5192         BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64);
5193         BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
5194         BUILD_BUG_ON(sizeof(struct nvme_identify) != 64);
5195         BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
5196         BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64);
5197         BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
5198         BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
5199         BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
5200         BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
5201         BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
5202         BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
5203         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
5204         BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
5205         BUILD_BUG_ON(sizeof(struct nvme_id_ns_cs_indep) !=
5206                         NVME_IDENTIFY_DATA_SIZE);
5207         BUILD_BUG_ON(sizeof(struct nvme_id_ns_zns) != NVME_IDENTIFY_DATA_SIZE);
5208         BUILD_BUG_ON(sizeof(struct nvme_id_ns_nvm) != NVME_IDENTIFY_DATA_SIZE);
5209         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_zns) != NVME_IDENTIFY_DATA_SIZE);
5210         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_nvm) != NVME_IDENTIFY_DATA_SIZE);
5211         BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
5212         BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
5213         BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
5214         BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64);
5215         BUILD_BUG_ON(sizeof(struct nvme_feat_host_behavior) != 512);
5216 }
5217
5218
5219 static int __init nvme_core_init(void)
5220 {
5221         int result = -ENOMEM;
5222
5223         _nvme_check_size();
5224
5225         nvme_wq = alloc_workqueue("nvme-wq",
5226                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
5227         if (!nvme_wq)
5228                 goto out;
5229
5230         nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
5231                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
5232         if (!nvme_reset_wq)
5233                 goto destroy_wq;
5234
5235         nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
5236                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
5237         if (!nvme_delete_wq)
5238                 goto destroy_reset_wq;
5239
5240         result = alloc_chrdev_region(&nvme_ctrl_base_chr_devt, 0,
5241                         NVME_MINORS, "nvme");
5242         if (result < 0)
5243                 goto destroy_delete_wq;
5244
5245         nvme_class = class_create("nvme");
5246         if (IS_ERR(nvme_class)) {
5247                 result = PTR_ERR(nvme_class);
5248                 goto unregister_chrdev;
5249         }
5250         nvme_class->dev_uevent = nvme_class_uevent;
5251
5252         nvme_subsys_class = class_create("nvme-subsystem");
5253         if (IS_ERR(nvme_subsys_class)) {
5254                 result = PTR_ERR(nvme_subsys_class);
5255                 goto destroy_class;
5256         }
5257
5258         result = alloc_chrdev_region(&nvme_ns_chr_devt, 0, NVME_MINORS,
5259                                      "nvme-generic");
5260         if (result < 0)
5261                 goto destroy_subsys_class;
5262
5263         nvme_ns_chr_class = class_create("nvme-generic");
5264         if (IS_ERR(nvme_ns_chr_class)) {
5265                 result = PTR_ERR(nvme_ns_chr_class);
5266                 goto unregister_generic_ns;
5267         }
5268
5269         result = nvme_init_auth();
5270         if (result)
5271                 goto destroy_ns_chr;
5272         return 0;
5273
5274 destroy_ns_chr:
5275         class_destroy(nvme_ns_chr_class);
5276 unregister_generic_ns:
5277         unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);
5278 destroy_subsys_class:
5279         class_destroy(nvme_subsys_class);
5280 destroy_class:
5281         class_destroy(nvme_class);
5282 unregister_chrdev:
5283         unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
5284 destroy_delete_wq:
5285         destroy_workqueue(nvme_delete_wq);
5286 destroy_reset_wq:
5287         destroy_workqueue(nvme_reset_wq);
5288 destroy_wq:
5289         destroy_workqueue(nvme_wq);
5290 out:
5291         return result;
5292 }
5293
5294 static void __exit nvme_core_exit(void)
5295 {
5296         nvme_exit_auth();
5297         class_destroy(nvme_ns_chr_class);
5298         class_destroy(nvme_subsys_class);
5299         class_destroy(nvme_class);
5300         unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);
5301         unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
5302         destroy_workqueue(nvme_delete_wq);
5303         destroy_workqueue(nvme_reset_wq);
5304         destroy_workqueue(nvme_wq);
5305         ida_destroy(&nvme_ns_chr_minor_ida);
5306         ida_destroy(&nvme_instance_ida);
5307 }
5308
5309 MODULE_LICENSE("GPL");
5310 MODULE_VERSION("1.0");
5311 module_init(nvme_core_init);
5312 module_exit(nvme_core_exit);