drivers/nvme/host/pci.c

   1 /*
   2  * NVM Express device driver
   3  * Copyright (c) 2011-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  */
  14
  15 #include <linux/aer.h>
  16 #include <linux/bitops.h>
  17 #include <linux/blkdev.h>
  18 #include <linux/blk-mq.h>
  19 #include <linux/blk-mq-pci.h>
  20 #include <linux/dmi.h>
  21 #include <linux/init.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/io.h>
  24 #include <linux/mm.h>
  25 #include <linux/module.h>
  26 #include <linux/mutex.h>
  27 #include <linux/pci.h>
  28 #include <linux/poison.h>
  29 #include <linux/t10-pi.h>
  30 #include <linux/timer.h>
  31 #include <linux/types.h>
  32 #include <linux/io-64-nonatomic-lo-hi.h>
  33 #include <asm/unaligned.h>
  34 #include <linux/sed-opal.h>
  35
  36 #include "nvme.h"
  37
  38 #define SQ_SIZE(depth)          (depth * sizeof(struct nvme_command))
  39 #define CQ_SIZE(depth)          (depth * sizeof(struct nvme_completion))
  40
  41 /*
  42  * We handle AEN commands ourselves and don't even let the
  43  * block layer know about them.
  44  */
  45 #define NVME_AQ_BLKMQ_DEPTH     (NVME_AQ_DEPTH - NVME_NR_AERS)
  46
  47 static int use_threaded_interrupts;
  48 module_param(use_threaded_interrupts, int, 0);
  49
  50 static bool use_cmb_sqes = true;
  51 module_param(use_cmb_sqes, bool, 0644);
  52 MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");
  53
  54 static unsigned int max_host_mem_size_mb = 128;
  55 module_param(max_host_mem_size_mb, uint, 0444);
  56 MODULE_PARM_DESC(max_host_mem_size_mb,
  57         "Maximum Host Memory Buffer (HMB) size per controller (in MiB)");
  58
  59 static int io_queue_depth_set(const char *val, const struct kernel_param *kp);
  60 static const struct kernel_param_ops io_queue_depth_ops = {
  61         .set = io_queue_depth_set,
  62         .get = param_get_int,
  63 };
  64
  65 static int io_queue_depth = 1024;
  66 module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644);
  67 MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2");
  68
  69 struct nvme_dev;
  70 struct nvme_queue;
  71
  72 static void nvme_process_cq(struct nvme_queue *nvmeq);
  73 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
  74
  75 /*
  76  * Represents an NVM Express device.  Each nvme_dev is a PCI function.
  77  */
  78 struct nvme_dev {
  79         struct nvme_queue **queues;
  80         struct blk_mq_tag_set tagset;
  81         struct blk_mq_tag_set admin_tagset;
  82         u32 __iomem *dbs;
  83         struct device *dev;
  84         struct dma_pool *prp_page_pool;
  85         struct dma_pool *prp_small_pool;
  86         unsigned online_queues;
  87         unsigned max_qid;
  88         int q_depth;
  89         u32 db_stride;
  90         void __iomem *bar;
  91         unsigned long bar_mapped_size;
  92         struct work_struct remove_work;
  93         struct mutex shutdown_lock;
  94         bool subsystem;
  95         void __iomem *cmb;
  96         dma_addr_t cmb_dma_addr;
  97         u64 cmb_size;
  98         u32 cmbsz;
  99         u32 cmbloc;
 100         struct nvme_ctrl ctrl;
 101         struct completion ioq_wait;
 102
 103         /* shadow doorbell buffer support: */
 104         u32 *dbbuf_dbs;
 105         dma_addr_t dbbuf_dbs_dma_addr;
 106         u32 *dbbuf_eis;
 107         dma_addr_t dbbuf_eis_dma_addr;
 108
 109         /* host memory buffer support: */
 110         u64 host_mem_size;
 111         u32 nr_host_mem_descs;
 112         dma_addr_t host_mem_descs_dma;
 113         struct nvme_host_mem_buf_desc *host_mem_descs;
 114         void **host_mem_desc_bufs;
 115 };
 116
 117 static int io_queue_depth_set(const char *val, const struct kernel_param *kp)
 118 {
 119         int n = 0, ret;
 120
 121         ret = kstrtoint(val, 10, &n);
 122         if (ret != 0 || n < 2)
 123                 return -EINVAL;
 124
 125         return param_set_int(val, kp);
 126 }
 127
 128 static inline unsigned int sq_idx(unsigned int qid, u32 stride)
 129 {
 130         return qid * 2 * stride;
 131 }
 132
 133 static inline unsigned int cq_idx(unsigned int qid, u32 stride)
 134 {
 135         return (qid * 2 + 1) * stride;
 136 }
 137
 138 static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
 139 {
 140         return container_of(ctrl, struct nvme_dev, ctrl);
 141 }
 142
 143 /*
 144  * An NVM Express queue.  Each device has at least two (one for admin
 145  * commands and one for I/O commands).
 146  */
 147 struct nvme_queue {
 148         struct device *q_dmadev;
 149         struct nvme_dev *dev;
 150         spinlock_t q_lock;
 151         struct nvme_command *sq_cmds;
 152         struct nvme_command __iomem *sq_cmds_io;
 153         volatile struct nvme_completion *cqes;
 154         struct blk_mq_tags **tags;
 155         dma_addr_t sq_dma_addr;
 156         dma_addr_t cq_dma_addr;
 157         u32 __iomem *q_db;
 158         u16 q_depth;
 159         s16 cq_vector;
 160         u16 sq_tail;
 161         u16 cq_head;
 162         u16 qid;
 163         u8 cq_phase;
 164         u8 cqe_seen;
 165         u32 *dbbuf_sq_db;
 166         u32 *dbbuf_cq_db;
 167         u32 *dbbuf_sq_ei;
 168         u32 *dbbuf_cq_ei;
 169 };
 170
 171 /*
 172  * The nvme_iod describes the data in an I/O, including the list of PRP
 173  * entries.  You can't see it in this data structure because C doesn't let
 174  * me express that.  Use nvme_init_iod to ensure there's enough space
 175  * allocated to store the PRP list.
 176  */
 177 struct nvme_iod {
 178         struct nvme_request req;
 179         struct nvme_queue *nvmeq;
 180         int aborted;
 181         int npages;             /* In the PRP list. 0 means small pool in use */
 182         int nents;              /* Used in scatterlist */
 183         int length;             /* Of data, in bytes */
 184         dma_addr_t first_dma;
 185         struct scatterlist meta_sg; /* metadata requires single contiguous buffer */
 186         struct scatterlist *sg;
 187         struct scatterlist inline_sg[0];
 188 };
 189
 190 /*
 191  * Check we didin't inadvertently grow the command struct
 192  */
 193 static inline void _nvme_check_size(void)
 194 {
 195         BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
 196         BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
 197         BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
 198         BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
 199         BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
 200         BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
 201         BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
 202         BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
 203         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
 204         BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
 205         BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
 206         BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
 207         BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
 208 }
 209
 210 static inline unsigned int nvme_dbbuf_size(u32 stride)
 211 {
 212         return ((num_possible_cpus() + 1) * 8 * stride);
 213 }
 214
 215 static int nvme_dbbuf_dma_alloc(struct nvme_dev *dev)
 216 {
 217         unsigned int mem_size = nvme_dbbuf_size(dev->db_stride);
 218
 219         if (dev->dbbuf_dbs)
 220                 return 0;
 221
 222         dev->dbbuf_dbs = dma_alloc_coherent(dev->dev, mem_size,
 223                                             &dev->dbbuf_dbs_dma_addr,
 224                                             GFP_KERNEL);
 225         if (!dev->dbbuf_dbs)
 226                 return -ENOMEM;
 227         dev->dbbuf_eis = dma_alloc_coherent(dev->dev, mem_size,
 228                                             &dev->dbbuf_eis_dma_addr,
 229                                             GFP_KERNEL);
 230         if (!dev->dbbuf_eis) {
 231                 dma_free_coherent(dev->dev, mem_size,
 232                                   dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
 233                 dev->dbbuf_dbs = NULL;
 234                 return -ENOMEM;
 235         }
 236
 237         return 0;
 238 }
 239
 240 static void nvme_dbbuf_dma_free(struct nvme_dev *dev)
 241 {
 242         unsigned int mem_size = nvme_dbbuf_size(dev->db_stride);
 243
 244         if (dev->dbbuf_dbs) {
 245                 dma_free_coherent(dev->dev, mem_size,
 246                                   dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
 247                 dev->dbbuf_dbs = NULL;
 248         }
 249         if (dev->dbbuf_eis) {
 250                 dma_free_coherent(dev->dev, mem_size,
 251                                   dev->dbbuf_eis, dev->dbbuf_eis_dma_addr);
 252                 dev->dbbuf_eis = NULL;
 253         }
 254 }
 255
 256 static void nvme_dbbuf_init(struct nvme_dev *dev,
 257                             struct nvme_queue *nvmeq, int qid)
 258 {
 259         if (!dev->dbbuf_dbs || !qid)
 260                 return;
 261
 262         nvmeq->dbbuf_sq_db = &dev->dbbuf_dbs[sq_idx(qid, dev->db_stride)];
 263         nvmeq->dbbuf_cq_db = &dev->dbbuf_dbs[cq_idx(qid, dev->db_stride)];
 264         nvmeq->dbbuf_sq_ei = &dev->dbbuf_eis[sq_idx(qid, dev->db_stride)];
 265         nvmeq->dbbuf_cq_ei = &dev->dbbuf_eis[cq_idx(qid, dev->db_stride)];
 266 }
 267
 268 static void nvme_dbbuf_set(struct nvme_dev *dev)
 269 {
 270         struct nvme_command c;
 271
 272         if (!dev->dbbuf_dbs)
 273                 return;
 274
 275         memset(&c, 0, sizeof(c));
 276         c.dbbuf.opcode = nvme_admin_dbbuf;
 277         c.dbbuf.prp1 = cpu_to_le64(dev->dbbuf_dbs_dma_addr);
 278         c.dbbuf.prp2 = cpu_to_le64(dev->dbbuf_eis_dma_addr);
 279
 280         if (nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0)) {
 281                 dev_warn(dev->ctrl.device, "unable to set dbbuf\n");
 282                 /* Free memory and continue on */
 283                 nvme_dbbuf_dma_free(dev);
 284         }
 285 }
 286
 287 static inline int nvme_dbbuf_need_event(u16 event_idx, u16 new_idx, u16 old)
 288 {
 289         return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old);
 290 }
 291
 292 /* Update dbbuf and return true if an MMIO is required */
 293 static bool nvme_dbbuf_update_and_check_event(u16 value, u32 *dbbuf_db,
 294                                               volatile u32 *dbbuf_ei)
 295 {
 296         if (dbbuf_db) {
 297                 u16 old_value;
 298
 299                 /*
 300                  * Ensure that the queue is written before updating
 301                  * the doorbell in memory
 302                  */
 303                 wmb();
 304
 305                 old_value = *dbbuf_db;
 306                 *dbbuf_db = value;
 307
 308                 if (!nvme_dbbuf_need_event(*dbbuf_ei, value, old_value))
 309                         return false;
 310         }
 311
 312         return true;
 313 }
 314
 315 /*
 316  * Max size of iod being embedded in the request payload
 317  */
 318 #define NVME_INT_PAGES          2
 319 #define NVME_INT_BYTES(dev)     (NVME_INT_PAGES * (dev)->ctrl.page_size)
 320
 321 /*
 322  * Will slightly overestimate the number of pages needed.  This is OK
 323  * as it only leads to a small amount of wasted memory for the lifetime of
 324  * the I/O.
 325  */
 326 static int nvme_npages(unsigned size, struct nvme_dev *dev)
 327 {
 328         unsigned nprps = DIV_ROUND_UP(size + dev->ctrl.page_size,
 329                                       dev->ctrl.page_size);
 330         return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
 331 }
 332
 333 static unsigned int nvme_iod_alloc_size(struct nvme_dev *dev,
 334                 unsigned int size, unsigned int nseg)
 335 {
 336         return sizeof(__le64 *) * nvme_npages(size, dev) +
 337                         sizeof(struct scatterlist) * nseg;
 338 }
 339
 340 static unsigned int nvme_cmd_size(struct nvme_dev *dev)
 341 {
 342         return sizeof(struct nvme_iod) +
 343                 nvme_iod_alloc_size(dev, NVME_INT_BYTES(dev), NVME_INT_PAGES);
 344 }
 345
 346 static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 347                                 unsigned int hctx_idx)
 348 {
 349         struct nvme_dev *dev = data;
 350         struct nvme_queue *nvmeq = dev->queues[0];
 351
 352         WARN_ON(hctx_idx != 0);
 353         WARN_ON(dev->admin_tagset.tags[0] != hctx->tags);
 354         WARN_ON(nvmeq->tags);
 355
 356         hctx->driver_data = nvmeq;
 357         nvmeq->tags = &dev->admin_tagset.tags[0];
 358         return 0;
 359 }
 360
 361 static void nvme_admin_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
 362 {
 363         struct nvme_queue *nvmeq = hctx->driver_data;
 364
 365         nvmeq->tags = NULL;
 366 }
 367
 368 static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 369                           unsigned int hctx_idx)
 370 {
 371         struct nvme_dev *dev = data;
 372         struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];
 373
 374         if (!nvmeq->tags)
 375                 nvmeq->tags = &dev->tagset.tags[hctx_idx];
 376
 377         WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
 378         hctx->driver_data = nvmeq;
 379         return 0;
 380 }
 381
 382 static int nvme_init_request(struct blk_mq_tag_set *set, struct request *req,
 383                 unsigned int hctx_idx, unsigned int numa_node)
 384 {
 385         struct nvme_dev *dev = set->driver_data;
 386         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 387         int queue_idx = (set == &dev->tagset) ? hctx_idx + 1 : 0;
 388         struct nvme_queue *nvmeq = dev->queues[queue_idx];
 389
 390         BUG_ON(!nvmeq);
 391         iod->nvmeq = nvmeq;
 392         return 0;
 393 }
 394
 395 static int nvme_pci_map_queues(struct blk_mq_tag_set *set)
 396 {
 397         struct nvme_dev *dev = set->driver_data;
 398
 399         return blk_mq_pci_map_queues(set, to_pci_dev(dev->dev));
 400 }
 401
 402 /**
 403  * __nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
 404  * @nvmeq: The queue to use
 405  * @cmd: The command to send
 406  *
 407  * Safe to use from interrupt context
 408  */
 409 static void __nvme_submit_cmd(struct nvme_queue *nvmeq,
 410                                                 struct nvme_command *cmd)
 411 {
 412         u16 tail = nvmeq->sq_tail;
 413
 414         if (nvmeq->sq_cmds_io)
 415                 memcpy_toio(&nvmeq->sq_cmds_io[tail], cmd, sizeof(*cmd));
 416         else
 417                 memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));
 418
 419         if (++tail == nvmeq->q_depth)
 420                 tail = 0;
 421         if (nvme_dbbuf_update_and_check_event(tail, nvmeq->dbbuf_sq_db,
 422                                               nvmeq->dbbuf_sq_ei))
 423                 writel(tail, nvmeq->q_db);
 424         nvmeq->sq_tail = tail;
 425 }
 426
 427 static __le64 **iod_list(struct request *req)
 428 {
 429         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 430         return (__le64 **)(iod->sg + blk_rq_nr_phys_segments(req));
 431 }
 432
 433 static blk_status_t nvme_init_iod(struct request *rq, struct nvme_dev *dev)
 434 {
 435         struct nvme_iod *iod = blk_mq_rq_to_pdu(rq);
 436         int nseg = blk_rq_nr_phys_segments(rq);
 437         unsigned int size = blk_rq_payload_bytes(rq);
 438
 439         if (nseg > NVME_INT_PAGES || size > NVME_INT_BYTES(dev)) {
 440                 iod->sg = kmalloc(nvme_iod_alloc_size(dev, size, nseg), GFP_ATOMIC);
 441                 if (!iod->sg)
 442                         return BLK_STS_RESOURCE;
 443         } else {
 444                 iod->sg = iod->inline_sg;
 445         }
 446
 447         iod->aborted = 0;
 448         iod->npages = -1;
 449         iod->nents = 0;
 450         iod->length = size;
 451
 452         return BLK_STS_OK;
 453 }
 454
 455 static void nvme_free_iod(struct nvme_dev *dev, struct request *req)
 456 {
 457         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 458         const int last_prp = dev->ctrl.page_size / 8 - 1;
 459         int i;
 460         __le64 **list = iod_list(req);
 461         dma_addr_t prp_dma = iod->first_dma;
 462
 463         if (iod->npages == 0)
 464                 dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
 465         for (i = 0; i < iod->npages; i++) {
 466                 __le64 *prp_list = list[i];
 467                 dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
 468                 dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
 469                 prp_dma = next_prp_dma;
 470         }
 471
 472         if (iod->sg != iod->inline_sg)
 473                 kfree(iod->sg);
 474 }
 475
 476 #ifdef CONFIG_BLK_DEV_INTEGRITY
 477 static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
 478 {
 479         if (be32_to_cpu(pi->ref_tag) == v)
 480                 pi->ref_tag = cpu_to_be32(p);
 481 }
 482
 483 static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
 484 {
 485         if (be32_to_cpu(pi->ref_tag) == p)
 486                 pi->ref_tag = cpu_to_be32(v);
 487 }
 488
 489 /**
 490  * nvme_dif_remap - remaps ref tags to bip seed and physical lba
 491  *
 492  * The virtual start sector is the one that was originally submitted by the
 493  * block layer. Due to partitioning, MD/DM cloning, etc. the actual physical
 494  * start sector may be different. Remap protection information to match the
 495  * physical LBA on writes, and back to the original seed on reads.
 496  *
 497  * Type 0 and 3 do not have a ref tag, so no remapping required.
 498  */
 499 static void nvme_dif_remap(struct request *req,
 500                         void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
 501 {
 502         struct nvme_ns *ns = req->rq_disk->private_data;
 503         struct bio_integrity_payload *bip;
 504         struct t10_pi_tuple *pi;
 505         void *p, *pmap;
 506         u32 i, nlb, ts, phys, virt;
 507
 508         if (!ns->pi_type || ns->pi_type == NVME_NS_DPS_PI_TYPE3)
 509                 return;
 510
 511         bip = bio_integrity(req->bio);
 512         if (!bip)
 513                 return;
 514
 515         pmap = kmap_atomic(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset;
 516
 517         p = pmap;
 518         virt = bip_get_seed(bip);
 519         phys = nvme_block_nr(ns, blk_rq_pos(req));
 520         nlb = (blk_rq_bytes(req) >> ns->lba_shift);
 521         ts = ns->disk->queue->integrity.tuple_size;
 522
 523         for (i = 0; i < nlb; i++, virt++, phys++) {
 524                 pi = (struct t10_pi_tuple *)p;
 525                 dif_swap(phys, virt, pi);
 526                 p += ts;
 527         }
 528         kunmap_atomic(pmap);
 529 }
 530 #else /* CONFIG_BLK_DEV_INTEGRITY */
 531 static void nvme_dif_remap(struct request *req,
 532                         void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
 533 {
 534 }
 535 static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
 536 {
 537 }
 538 static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
 539 {
 540 }
 541 #endif
 542
 543 static blk_status_t nvme_setup_prps(struct nvme_dev *dev, struct request *req)
 544 {
 545         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 546         struct dma_pool *pool;
 547         int length = blk_rq_payload_bytes(req);
 548         struct scatterlist *sg = iod->sg;
 549         int dma_len = sg_dma_len(sg);
 550         u64 dma_addr = sg_dma_address(sg);
 551         u32 page_size = dev->ctrl.page_size;
 552         int offset = dma_addr & (page_size - 1);
 553         __le64 *prp_list;
 554         __le64 **list = iod_list(req);
 555         dma_addr_t prp_dma;
 556         int nprps, i;
 557
 558         length -= (page_size - offset);
 559         if (length <= 0) {
 560                 iod->first_dma = 0;
 561                 return BLK_STS_OK;
 562         }
 563
 564         dma_len -= (page_size - offset);
 565         if (dma_len) {
 566                 dma_addr += (page_size - offset);
 567         } else {
 568                 sg = sg_next(sg);
 569                 dma_addr = sg_dma_address(sg);
 570                 dma_len = sg_dma_len(sg);
 571         }
 572
 573         if (length <= page_size) {
 574                 iod->first_dma = dma_addr;
 575                 return BLK_STS_OK;
 576         }
 577
 578         nprps = DIV_ROUND_UP(length, page_size);
 579         if (nprps <= (256 / 8)) {
 580                 pool = dev->prp_small_pool;
 581                 iod->npages = 0;
 582         } else {
 583                 pool = dev->prp_page_pool;
 584                 iod->npages = 1;
 585         }
 586
 587         prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
 588         if (!prp_list) {
 589                 iod->first_dma = dma_addr;
 590                 iod->npages = -1;
 591                 return BLK_STS_RESOURCE;
 592         }
 593         list[0] = prp_list;
 594         iod->first_dma = prp_dma;
 595         i = 0;
 596         for (;;) {
 597                 if (i == page_size >> 3) {
 598                         __le64 *old_prp_list = prp_list;
 599                         prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
 600                         if (!prp_list)
 601                                 return BLK_STS_RESOURCE;
 602                         list[iod->npages++] = prp_list;
 603                         prp_list[0] = old_prp_list[i - 1];
 604                         old_prp_list[i - 1] = cpu_to_le64(prp_dma);
 605                         i = 1;
 606                 }
 607                 prp_list[i++] = cpu_to_le64(dma_addr);
 608                 dma_len -= page_size;
 609                 dma_addr += page_size;
 610                 length -= page_size;
 611                 if (length <= 0)
 612                         break;
 613                 if (dma_len > 0)
 614                         continue;
 615                 if (unlikely(dma_len < 0))
 616                         goto bad_sgl;
 617                 sg = sg_next(sg);
 618                 dma_addr = sg_dma_address(sg);
 619                 dma_len = sg_dma_len(sg);
 620         }
 621
 622         return BLK_STS_OK;
 623
 624  bad_sgl:
 625         if (WARN_ONCE(1, "Invalid SGL for payload:%d nents:%d\n",
 626                                 blk_rq_payload_bytes(req), iod->nents)) {
 627                 for_each_sg(iod->sg, sg, iod->nents, i) {
 628                         dma_addr_t phys = sg_phys(sg);
 629                         pr_warn("sg[%d] phys_addr:%pad offset:%d length:%d "
 630                                "dma_address:%pad dma_length:%d\n", i, &phys,
 631                                         sg->offset, sg->length,
 632                                         &sg_dma_address(sg),
 633                                         sg_dma_len(sg));
 634                 }
 635         }
 636         return BLK_STS_IOERR;
 637
 638 }
 639
 640 static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
 641                 struct nvme_command *cmnd)
 642 {
 643         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 644         struct request_queue *q = req->q;
 645         enum dma_data_direction dma_dir = rq_data_dir(req) ?
 646                         DMA_TO_DEVICE : DMA_FROM_DEVICE;
 647         blk_status_t ret = BLK_STS_IOERR;
 648
 649         sg_init_table(iod->sg, blk_rq_nr_phys_segments(req));
 650         iod->nents = blk_rq_map_sg(q, req, iod->sg);
 651         if (!iod->nents)
 652                 goto out;
 653
 654         ret = BLK_STS_RESOURCE;
 655         if (!dma_map_sg_attrs(dev->dev, iod->sg, iod->nents, dma_dir,
 656                                 DMA_ATTR_NO_WARN))
 657                 goto out;
 658
 659         ret = nvme_setup_prps(dev, req);
 660         if (ret != BLK_STS_OK)
 661                 goto out_unmap;
 662
 663         ret = BLK_STS_IOERR;
 664         if (blk_integrity_rq(req)) {
 665                 if (blk_rq_count_integrity_sg(q, req->bio) != 1)
 666                         goto out_unmap;
 667
 668                 sg_init_table(&iod->meta_sg, 1);
 669                 if (blk_rq_map_integrity_sg(q, req->bio, &iod->meta_sg) != 1)
 670                         goto out_unmap;
 671
 672                 if (req_op(req) == REQ_OP_WRITE)
 673                         nvme_dif_remap(req, nvme_dif_prep);
 674
 675                 if (!dma_map_sg(dev->dev, &iod->meta_sg, 1, dma_dir))
 676                         goto out_unmap;
 677         }
 678
 679         cmnd->rw.dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
 680         cmnd->rw.dptr.prp2 = cpu_to_le64(iod->first_dma);
 681         if (blk_integrity_rq(req))
 682                 cmnd->rw.metadata = cpu_to_le64(sg_dma_address(&iod->meta_sg));
 683         return BLK_STS_OK;
 684
 685 out_unmap:
 686         dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
 687 out:
 688         return ret;
 689 }
 690
 691 static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
 692 {
 693         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 694         enum dma_data_direction dma_dir = rq_data_dir(req) ?
 695                         DMA_TO_DEVICE : DMA_FROM_DEVICE;
 696
 697         if (iod->nents) {
 698                 dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
 699                 if (blk_integrity_rq(req)) {
 700                         if (req_op(req) == REQ_OP_READ)
 701                                 nvme_dif_remap(req, nvme_dif_complete);
 702                         dma_unmap_sg(dev->dev, &iod->meta_sg, 1, dma_dir);
 703                 }
 704         }
 705
 706         nvme_cleanup_cmd(req);
 707         nvme_free_iod(dev, req);
 708 }
 709
 710 /*
 711  * NOTE: ns is NULL when called on the admin queue.
 712  */
 713 static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
 714                          const struct blk_mq_queue_data *bd)
 715 {
 716         struct nvme_ns *ns = hctx->queue->queuedata;
 717         struct nvme_queue *nvmeq = hctx->driver_data;
 718         struct nvme_dev *dev = nvmeq->dev;
 719         struct request *req = bd->rq;
 720         struct nvme_command cmnd;
 721         blk_status_t ret;
 722
 723         ret = nvme_setup_cmd(ns, req, &cmnd);
 724         if (ret)
 725                 return ret;
 726
 727         ret = nvme_init_iod(req, dev);
 728         if (ret)
 729                 goto out_free_cmd;
 730
 731         if (blk_rq_nr_phys_segments(req)) {
 732                 ret = nvme_map_data(dev, req, &cmnd);
 733                 if (ret)
 734                         goto out_cleanup_iod;
 735         }
 736
 737         blk_mq_start_request(req);
 738
 739         spin_lock_irq(&nvmeq->q_lock);
 740         if (unlikely(nvmeq->cq_vector < 0)) {
 741                 ret = BLK_STS_IOERR;
 742                 spin_unlock_irq(&nvmeq->q_lock);
 743                 goto out_cleanup_iod;
 744         }
 745         __nvme_submit_cmd(nvmeq, &cmnd);
 746         nvme_process_cq(nvmeq);
 747         spin_unlock_irq(&nvmeq->q_lock);
 748         return BLK_STS_OK;
 749 out_cleanup_iod:
 750         nvme_free_iod(dev, req);
 751 out_free_cmd:
 752         nvme_cleanup_cmd(req);
 753         return ret;
 754 }
 755
 756 static void nvme_pci_complete_rq(struct request *req)
 757 {
 758         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 759
 760         nvme_unmap_data(iod->nvmeq->dev, req);
 761         nvme_complete_rq(req);
 762 }
 763
 764 /* We read the CQE phase first to check if the rest of the entry is valid */
 765 static inline bool nvme_cqe_valid(struct nvme_queue *nvmeq, u16 head,
 766                 u16 phase)
 767 {
 768         return (le16_to_cpu(nvmeq->cqes[head].status) & 1) == phase;
 769 }
 770
 771 static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq)
 772 {
 773         u16 head = nvmeq->cq_head;
 774
 775         if (likely(nvmeq->cq_vector >= 0)) {
 776                 if (nvme_dbbuf_update_and_check_event(head, nvmeq->dbbuf_cq_db,
 777                                                       nvmeq->dbbuf_cq_ei))
 778                         writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
 779         }
 780 }
 781
 782 static inline void nvme_handle_cqe(struct nvme_queue *nvmeq,
 783                 struct nvme_completion *cqe)
 784 {
 785         struct request *req;
 786
 787         if (unlikely(cqe->command_id >= nvmeq->q_depth)) {
 788                 dev_warn(nvmeq->dev->ctrl.device,
 789                         "invalid id %d completed on queue %d\n",
 790                         cqe->command_id, le16_to_cpu(cqe->sq_id));
 791                 return;
 792         }
 793
 794         /*
 795          * AEN requests are special as they don't time out and can
 796          * survive any kind of queue freeze and often don't respond to
 797          * aborts.  We don't even bother to allocate a struct request
 798          * for them but rather special case them here.
 799          */
 800         if (unlikely(nvmeq->qid == 0 &&
 801                         cqe->command_id >= NVME_AQ_BLKMQ_DEPTH)) {
 802                 nvme_complete_async_event(&nvmeq->dev->ctrl,
 803                                 cqe->status, &cqe->result);
 804                 return;
 805         }
 806
 807         nvmeq->cqe_seen = 1;
 808         req = blk_mq_tag_to_rq(*nvmeq->tags, cqe->command_id);
 809         nvme_end_request(req, cqe->status, cqe->result);
 810 }
 811
 812 static inline bool nvme_read_cqe(struct nvme_queue *nvmeq,
 813                 struct nvme_completion *cqe)
 814 {
 815         if (nvme_cqe_valid(nvmeq, nvmeq->cq_head, nvmeq->cq_phase)) {
 816                 *cqe = nvmeq->cqes[nvmeq->cq_head];
 817
 818                 if (++nvmeq->cq_head == nvmeq->q_depth) {
 819                         nvmeq->cq_head = 0;
 820                         nvmeq->cq_phase = !nvmeq->cq_phase;
 821                 }
 822                 return true;
 823         }
 824         return false;
 825 }
 826
 827 static void nvme_process_cq(struct nvme_queue *nvmeq)
 828 {
 829         struct nvme_completion cqe;
 830         int consumed = 0;
 831
 832         while (nvme_read_cqe(nvmeq, &cqe)) {
 833                 nvme_handle_cqe(nvmeq, &cqe);
 834                 consumed++;
 835         }
 836
 837         if (consumed)
 838                 nvme_ring_cq_doorbell(nvmeq);
 839 }
 840
 841 static irqreturn_t nvme_irq(int irq, void *data)
 842 {
 843         irqreturn_t result;
 844         struct nvme_queue *nvmeq = data;
 845         spin_lock(&nvmeq->q_lock);
 846         nvme_process_cq(nvmeq);
 847         result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE;
 848         nvmeq->cqe_seen = 0;
 849         spin_unlock(&nvmeq->q_lock);
 850         return result;
 851 }
 852
 853 static irqreturn_t nvme_irq_check(int irq, void *data)
 854 {
 855         struct nvme_queue *nvmeq = data;
 856         if (nvme_cqe_valid(nvmeq, nvmeq->cq_head, nvmeq->cq_phase))
 857                 return IRQ_WAKE_THREAD;
 858         return IRQ_NONE;
 859 }
 860
 861 static int __nvme_poll(struct nvme_queue *nvmeq, unsigned int tag)
 862 {
 863         struct nvme_completion cqe;
 864         int found = 0, consumed = 0;
 865
 866         if (!nvme_cqe_valid(nvmeq, nvmeq->cq_head, nvmeq->cq_phase))
 867                 return 0;
 868
 869         spin_lock_irq(&nvmeq->q_lock);
 870         while (nvme_read_cqe(nvmeq, &cqe)) {
 871                 nvme_handle_cqe(nvmeq, &cqe);
 872                 consumed++;
 873
 874                 if (tag == cqe.command_id) {
 875                         found = 1;
 876                         break;
 877                 }
 878        }
 879
 880         if (consumed)
 881                 nvme_ring_cq_doorbell(nvmeq);
 882         spin_unlock_irq(&nvmeq->q_lock);
 883
 884         return found;
 885 }
 886
 887 static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
 888 {
 889         struct nvme_queue *nvmeq = hctx->driver_data;
 890
 891         return __nvme_poll(nvmeq, tag);
 892 }
 893
 894 static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl, int aer_idx)
 895 {
 896         struct nvme_dev *dev = to_nvme_dev(ctrl);
 897         struct nvme_queue *nvmeq = dev->queues[0];
 898         struct nvme_command c;
 899
 900         memset(&c, 0, sizeof(c));
 901         c.common.opcode = nvme_admin_async_event;
 902         c.common.command_id = NVME_AQ_BLKMQ_DEPTH + aer_idx;
 903
 904         spin_lock_irq(&nvmeq->q_lock);
 905         __nvme_submit_cmd(nvmeq, &c);
 906         spin_unlock_irq(&nvmeq->q_lock);
 907 }
 908
 909 static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
 910 {
 911         struct nvme_command c;
 912
 913         memset(&c, 0, sizeof(c));
 914         c.delete_queue.opcode = opcode;
 915         c.delete_queue.qid = cpu_to_le16(id);
 916
 917         return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
 918 }
 919
 920 static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
 921                                                 struct nvme_queue *nvmeq)
 922 {
 923         struct nvme_command c;
 924         int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
 925
 926         /*
 927          * Note: we (ab)use the fact the the prp fields survive if no data
 928          * is attached to the request.
 929          */
 930         memset(&c, 0, sizeof(c));
 931         c.create_cq.opcode = nvme_admin_create_cq;
 932         c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
 933         c.create_cq.cqid = cpu_to_le16(qid);
 934         c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
 935         c.create_cq.cq_flags = cpu_to_le16(flags);
 936         c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
 937
 938         return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
 939 }
 940
 941 static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
 942                                                 struct nvme_queue *nvmeq)
 943 {
 944         struct nvme_command c;
 945         int flags = NVME_QUEUE_PHYS_CONTIG;
 946
 947         /*
 948          * Note: we (ab)use the fact the the prp fields survive if no data
 949          * is attached to the request.
 950          */
 951         memset(&c, 0, sizeof(c));
 952         c.create_sq.opcode = nvme_admin_create_sq;
 953         c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
 954         c.create_sq.sqid = cpu_to_le16(qid);
 955         c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
 956         c.create_sq.sq_flags = cpu_to_le16(flags);
 957         c.create_sq.cqid = cpu_to_le16(qid);
 958
 959         return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
 960 }
 961
 962 static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
 963 {
 964         return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
 965 }
 966
 967 static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
 968 {
 969         return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
 970 }
 971
 972 static void abort_endio(struct request *req, blk_status_t error)
 973 {
 974         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
 975         struct nvme_queue *nvmeq = iod->nvmeq;
 976
 977         dev_warn(nvmeq->dev->ctrl.device,
 978                  "Abort status: 0x%x", nvme_req(req)->status);
 979         atomic_inc(&nvmeq->dev->ctrl.abort_limit);
 980         blk_mq_free_request(req);
 981 }
 982
 983 static bool nvme_should_reset(struct nvme_dev *dev, u32 csts)
 984 {
 985
 986         /* If true, indicates loss of adapter communication, possibly by a
 987          * NVMe Subsystem reset.
 988          */
 989         bool nssro = dev->subsystem && (csts & NVME_CSTS_NSSRO);
 990
 991         /* If there is a reset ongoing, we shouldn't reset again. */
 992         if (dev->ctrl.state == NVME_CTRL_RESETTING)
 993                 return false;
 994
 995         /* We shouldn't reset unless the controller is on fatal error state
 996          * _or_ if we lost the communication with it.
 997          */
 998         if (!(csts & NVME_CSTS_CFS) && !nssro)
 999                 return false;
1000
1001         /* If PCI error recovery process is happening, we cannot reset or
1002          * the recovery mechanism will surely fail.
1003          */
1004         if (pci_channel_offline(to_pci_dev(dev->dev)))
1005                 return false;
1006
1007         return true;
1008 }
1009
1010 static void nvme_warn_reset(struct nvme_dev *dev, u32 csts)
1011 {
1012         /* Read a config register to help see what died. */
1013         u16 pci_status;
1014         int result;
1015
1016         result = pci_read_config_word(to_pci_dev(dev->dev), PCI_STATUS,
1017                                       &pci_status);
1018         if (result == PCIBIOS_SUCCESSFUL)
1019                 dev_warn(dev->ctrl.device,
1020                          "controller is down; will reset: CSTS=0x%x, PCI_STATUS=0x%hx\n",
1021                          csts, pci_status);
1022         else
1023                 dev_warn(dev->ctrl.device,
1024                          "controller is down; will reset: CSTS=0x%x, PCI_STATUS read failed (%d)\n",
1025                          csts, result);
1026 }
1027
1028 static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
1029 {
1030         struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
1031         struct nvme_queue *nvmeq = iod->nvmeq;
1032         struct nvme_dev *dev = nvmeq->dev;
1033         struct request *abort_req;
1034         struct nvme_command cmd;
1035         u32 csts = readl(dev->bar + NVME_REG_CSTS);
1036
1037         /*
1038          * Reset immediately if the controller is failed
1039          */
1040         if (nvme_should_reset(dev, csts)) {
1041                 nvme_warn_reset(dev, csts);
1042                 nvme_dev_disable(dev, false);
1043                 nvme_reset_ctrl(&dev->ctrl);
1044                 return BLK_EH_HANDLED;
1045         }
1046
1047         /*
1048          * Did we miss an interrupt?
1049          */
1050         if (__nvme_poll(nvmeq, req->tag)) {
1051                 dev_warn(dev->ctrl.device,
1052                          "I/O %d QID %d timeout, completion polled\n",
1053                          req->tag, nvmeq->qid);
1054                 return BLK_EH_HANDLED;
1055         }
1056
1057         /*
1058          * Shutdown immediately if controller times out while starting. The
1059          * reset work will see the pci device disabled when it gets the forced
1060          * cancellation error. All outstanding requests are completed on
1061          * shutdown, so we return BLK_EH_HANDLED.
1062          */
1063         if (dev->ctrl.state == NVME_CTRL_RESETTING) {
1064                 dev_warn(dev->ctrl.device,
1065                          "I/O %d QID %d timeout, disable controller\n",
1066                          req->tag, nvmeq->qid);
1067                 nvme_dev_disable(dev, false);
1068                 nvme_req(req)->flags |= NVME_REQ_CANCELLED;
1069                 return BLK_EH_HANDLED;
1070         }
1071
1072         /*
1073          * Shutdown the controller immediately and schedule a reset if the
1074          * command was already aborted once before and still hasn't been
1075          * returned to the driver, or if this is the admin queue.
1076          */
1077         if (!nvmeq->qid || iod->aborted) {
1078                 dev_warn(dev->ctrl.device,
1079                          "I/O %d QID %d timeout, reset controller\n",
1080                          req->tag, nvmeq->qid);
1081                 nvme_dev_disable(dev, false);
1082                 nvme_reset_ctrl(&dev->ctrl);
1083
1084                 /*
1085                  * Mark the request as handled, since the inline shutdown
1086                  * forces all outstanding requests to complete.
1087                  */
1088                 nvme_req(req)->flags |= NVME_REQ_CANCELLED;
1089                 return BLK_EH_HANDLED;
1090         }
1091
1092         if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) {
1093                 atomic_inc(&dev->ctrl.abort_limit);
1094                 return BLK_EH_RESET_TIMER;
1095         }
1096         iod->aborted = 1;
1097
1098         memset(&cmd, 0, sizeof(cmd));
1099         cmd.abort.opcode = nvme_admin_abort_cmd;
1100         cmd.abort.cid = req->tag;
1101         cmd.abort.sqid = cpu_to_le16(nvmeq->qid);
1102
1103         dev_warn(nvmeq->dev->ctrl.device,
1104                 "I/O %d QID %d timeout, aborting\n",
1105                  req->tag, nvmeq->qid);
1106
1107         abort_req = nvme_alloc_request(dev->ctrl.admin_q, &cmd,
1108                         BLK_MQ_REQ_NOWAIT, NVME_QID_ANY);
1109         if (IS_ERR(abort_req)) {
1110                 atomic_inc(&dev->ctrl.abort_limit);
1111                 return BLK_EH_RESET_TIMER;
1112         }
1113
1114         abort_req->timeout = ADMIN_TIMEOUT;
1115         abort_req->end_io_data = NULL;
1116         blk_execute_rq_nowait(abort_req->q, NULL, abort_req, 0, abort_endio);
1117
1118         /*
1119          * The aborted req will be completed on receiving the abort req.
1120          * We enable the timer again. If hit twice, it'll cause a device reset,
1121          * as the device then is in a faulty state.
1122          */
1123         return BLK_EH_RESET_TIMER;
1124 }
1125
1126 static void nvme_free_queue(struct nvme_queue *nvmeq)
1127 {
1128         dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
1129                                 (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1130         if (nvmeq->sq_cmds)
1131                 dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
1132                                         nvmeq->sq_cmds, nvmeq->sq_dma_addr);
1133         kfree(nvmeq);
1134 }
1135
1136 static void nvme_free_queues(struct nvme_dev *dev, int lowest)
1137 {
1138         int i;
1139
1140         for (i = dev->ctrl.queue_count - 1; i >= lowest; i--) {
1141                 struct nvme_queue *nvmeq = dev->queues[i];
1142                 dev->ctrl.queue_count--;
1143                 dev->queues[i] = NULL;
1144                 nvme_free_queue(nvmeq);
1145         }
1146 }
1147
1148 /**
1149  * nvme_suspend_queue - put queue into suspended state
1150  * @nvmeq - queue to suspend
1151  */
1152 static int nvme_suspend_queue(struct nvme_queue *nvmeq)
1153 {
1154         int vector;
1155
1156         spin_lock_irq(&nvmeq->q_lock);
1157         if (nvmeq->cq_vector == -1) {
1158                 spin_unlock_irq(&nvmeq->q_lock);
1159                 return 1;
1160         }
1161         vector = nvmeq->cq_vector;
1162         nvmeq->dev->online_queues--;
1163         nvmeq->cq_vector = -1;
1164         spin_unlock_irq(&nvmeq->q_lock);
1165
1166         if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
1167                 blk_mq_quiesce_queue(nvmeq->dev->ctrl.admin_q);
1168
1169         pci_free_irq(to_pci_dev(nvmeq->dev->dev), vector, nvmeq);
1170
1171         return 0;
1172 }
1173
1174 static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown)
1175 {
1176         struct nvme_queue *nvmeq = dev->queues[0];
1177
1178         if (!nvmeq)
1179                 return;
1180         if (nvme_suspend_queue(nvmeq))
1181                 return;
1182
1183         if (shutdown)
1184                 nvme_shutdown_ctrl(&dev->ctrl);
1185         else
1186                 nvme_disable_ctrl(&dev->ctrl, dev->ctrl.cap);
1187
1188         spin_lock_irq(&nvmeq->q_lock);
1189         nvme_process_cq(nvmeq);
1190         spin_unlock_irq(&nvmeq->q_lock);
1191 }
1192
1193 static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
1194                                 int entry_size)
1195 {
1196         int q_depth = dev->q_depth;
1197         unsigned q_size_aligned = roundup(q_depth * entry_size,
1198                                           dev->ctrl.page_size);
1199
1200         if (q_size_aligned * nr_io_queues > dev->cmb_size) {
1201                 u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
1202                 mem_per_q = round_down(mem_per_q, dev->ctrl.page_size);
1203                 q_depth = div_u64(mem_per_q, entry_size);
1204
1205                 /*
1206                  * Ensure the reduced q_depth is above some threshold where it
1207                  * would be better to map queues in system memory with the
1208                  * original depth
1209                  */
1210                 if (q_depth < 64)
1211                         return -ENOMEM;
1212         }
1213
1214         return q_depth;
1215 }
1216
1217 static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
1218                                 int qid, int depth)
1219 {
1220         if (qid && dev->cmb && use_cmb_sqes && NVME_CMB_SQS(dev->cmbsz)) {
1221                 unsigned offset = (qid - 1) * roundup(SQ_SIZE(depth),
1222                                                       dev->ctrl.page_size);
1223                 nvmeq->sq_dma_addr = dev->cmb_dma_addr + offset;
1224                 nvmeq->sq_cmds_io = dev->cmb + offset;
1225         } else {
1226                 nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(depth),
1227                                         &nvmeq->sq_dma_addr, GFP_KERNEL);
1228                 if (!nvmeq->sq_cmds)
1229                         return -ENOMEM;
1230         }
1231
1232         return 0;
1233 }
1234
1235 static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
1236                                                         int depth, int node)
1237 {
1238         struct nvme_queue *nvmeq = kzalloc_node(sizeof(*nvmeq), GFP_KERNEL,
1239                                                         node);
1240         if (!nvmeq)
1241                 return NULL;
1242
1243         nvmeq->cqes = dma_zalloc_coherent(dev->dev, CQ_SIZE(depth),
1244                                           &nvmeq->cq_dma_addr, GFP_KERNEL);
1245         if (!nvmeq->cqes)
1246                 goto free_nvmeq;
1247
1248         if (nvme_alloc_sq_cmds(dev, nvmeq, qid, depth))
1249                 goto free_cqdma;
1250
1251         nvmeq->q_dmadev = dev->dev;
1252         nvmeq->dev = dev;
1253         spin_lock_init(&nvmeq->q_lock);
1254         nvmeq->cq_head = 0;
1255         nvmeq->cq_phase = 1;
1256         nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1257         nvmeq->q_depth = depth;
1258         nvmeq->qid = qid;
1259         nvmeq->cq_vector = -1;
1260         dev->queues[qid] = nvmeq;
1261         dev->ctrl.queue_count++;
1262
1263         return nvmeq;
1264
1265  free_cqdma:
1266         dma_free_coherent(dev->dev, CQ_SIZE(depth), (void *)nvmeq->cqes,
1267                                                         nvmeq->cq_dma_addr);
1268  free_nvmeq:
1269         kfree(nvmeq);
1270         return NULL;
1271 }
1272
1273 static int queue_request_irq(struct nvme_queue *nvmeq)
1274 {
1275         struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
1276         int nr = nvmeq->dev->ctrl.instance;
1277
1278         if (use_threaded_interrupts) {
1279                 return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check,
1280                                 nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
1281         } else {
1282                 return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq,
1283                                 NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
1284         }
1285 }
1286
1287 static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
1288 {
1289         struct nvme_dev *dev = nvmeq->dev;
1290
1291         spin_lock_irq(&nvmeq->q_lock);
1292         nvmeq->sq_tail = 0;
1293         nvmeq->cq_head = 0;
1294         nvmeq->cq_phase = 1;
1295         nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1296         memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
1297         nvme_dbbuf_init(dev, nvmeq, qid);
1298         dev->online_queues++;
1299         spin_unlock_irq(&nvmeq->q_lock);
1300 }
1301
1302 static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
1303 {
1304         struct nvme_dev *dev = nvmeq->dev;
1305         int result;
1306
1307         nvmeq->cq_vector = qid - 1;
1308         result = adapter_alloc_cq(dev, qid, nvmeq);
1309         if (result < 0)
1310                 return result;
1311
1312         result = adapter_alloc_sq(dev, qid, nvmeq);
1313         if (result < 0)
1314                 goto release_cq;
1315
1316         result = queue_request_irq(nvmeq);
1317         if (result < 0)
1318                 goto release_sq;
1319
1320         nvme_init_queue(nvmeq, qid);
1321         return result;
1322
1323  release_sq:
1324         adapter_delete_sq(dev, qid);
1325  release_cq:
1326         adapter_delete_cq(dev, qid);
1327         return result;
1328 }
1329
1330 static const struct blk_mq_ops nvme_mq_admin_ops = {
1331         .queue_rq       = nvme_queue_rq,
1332         .complete       = nvme_pci_complete_rq,
1333         .init_hctx      = nvme_admin_init_hctx,
1334         .exit_hctx      = nvme_admin_exit_hctx,
1335         .init_request   = nvme_init_request,
1336         .timeout        = nvme_timeout,
1337 };
1338
1339 static const struct blk_mq_ops nvme_mq_ops = {
1340         .queue_rq       = nvme_queue_rq,
1341         .complete       = nvme_pci_complete_rq,
1342         .init_hctx      = nvme_init_hctx,
1343         .init_request   = nvme_init_request,
1344         .map_queues     = nvme_pci_map_queues,
1345         .timeout        = nvme_timeout,
1346         .poll           = nvme_poll,
1347 };
1348
1349 static void nvme_dev_remove_admin(struct nvme_dev *dev)
1350 {
1351         if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
1352                 /*
1353                  * If the controller was reset during removal, it's possible
1354                  * user requests may be waiting on a stopped queue. Start the
1355                  * queue to flush these to completion.
1356                  */
1357                 blk_mq_unquiesce_queue(dev->ctrl.admin_q);
1358                 blk_cleanup_queue(dev->ctrl.admin_q);
1359                 blk_mq_free_tag_set(&dev->admin_tagset);
1360         }
1361 }
1362
1363 static int nvme_alloc_admin_tags(struct nvme_dev *dev)
1364 {
1365         if (!dev->ctrl.admin_q) {
1366                 dev->admin_tagset.ops = &nvme_mq_admin_ops;
1367                 dev->admin_tagset.nr_hw_queues = 1;
1368
1369                 /*
1370                  * Subtract one to leave an empty queue entry for 'Full Queue'
1371                  * condition. See NVM-Express 1.2 specification, section 4.1.2.
1372                  */
1373                 dev->admin_tagset.queue_depth = NVME_AQ_BLKMQ_DEPTH - 1;
1374                 dev->admin_tagset.timeout = ADMIN_TIMEOUT;
1375                 dev->admin_tagset.numa_node = dev_to_node(dev->dev);
1376                 dev->admin_tagset.cmd_size = nvme_cmd_size(dev);
1377                 dev->admin_tagset.flags = BLK_MQ_F_NO_SCHED;
1378                 dev->admin_tagset.driver_data = dev;
1379
1380                 if (blk_mq_alloc_tag_set(&dev->admin_tagset))
1381                         return -ENOMEM;
1382                 dev->ctrl.admin_tagset = &dev->admin_tagset;
1383
1384                 dev->ctrl.admin_q = blk_mq_init_queue(&dev->admin_tagset);
1385                 if (IS_ERR(dev->ctrl.admin_q)) {
1386                         blk_mq_free_tag_set(&dev->admin_tagset);
1387                         return -ENOMEM;
1388                 }
1389                 if (!blk_get_queue(dev->ctrl.admin_q)) {
1390                         nvme_dev_remove_admin(dev);
1391                         dev->ctrl.admin_q = NULL;
1392                         return -ENODEV;
1393                 }
1394         } else
1395                 blk_mq_unquiesce_queue(dev->ctrl.admin_q);
1396
1397         return 0;
1398 }
1399
1400 static unsigned long db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
1401 {
1402         return NVME_REG_DBS + ((nr_io_queues + 1) * 8 * dev->db_stride);
1403 }
1404
1405 static int nvme_remap_bar(struct nvme_dev *dev, unsigned long size)
1406 {
1407         struct pci_dev *pdev = to_pci_dev(dev->dev);
1408
1409         if (size <= dev->bar_mapped_size)
1410                 return 0;
1411         if (size > pci_resource_len(pdev, 0))
1412                 return -ENOMEM;
1413         if (dev->bar)
1414                 iounmap(dev->bar);
1415         dev->bar = ioremap(pci_resource_start(pdev, 0), size);
1416         if (!dev->bar) {
1417                 dev->bar_mapped_size = 0;
1418                 return -ENOMEM;
1419         }
1420         dev->bar_mapped_size = size;
1421         dev->dbs = dev->bar + NVME_REG_DBS;
1422
1423         return 0;
1424 }
1425
1426 static int nvme_pci_configure_admin_queue(struct nvme_dev *dev)
1427 {
1428         int result;
1429         u32 aqa;
1430         struct nvme_queue *nvmeq;
1431
1432         result = nvme_remap_bar(dev, db_bar_size(dev, 0));
1433         if (result < 0)
1434                 return result;
1435
1436         dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1, 0) ?
1437                                 NVME_CAP_NSSRC(dev->ctrl.cap) : 0;
1438
1439         if (dev->subsystem &&
1440             (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
1441                 writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
1442
1443         result = nvme_disable_ctrl(&dev->ctrl, dev->ctrl.cap);
1444         if (result < 0)
1445                 return result;
1446
1447         nvmeq = dev->queues[0];
1448         if (!nvmeq) {
1449                 nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH,
1450                                         dev_to_node(dev->dev));
1451                 if (!nvmeq)
1452                         return -ENOMEM;
1453         }
1454
1455         aqa = nvmeq->q_depth - 1;
1456         aqa |= aqa << 16;
1457
1458         writel(aqa, dev->bar + NVME_REG_AQA);
1459         lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
1460         lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
1461
1462         result = nvme_enable_ctrl(&dev->ctrl, dev->ctrl.cap);
1463         if (result)
1464                 return result;
1465
1466         nvmeq->cq_vector = 0;
1467         result = queue_request_irq(nvmeq);
1468         if (result) {
1469                 nvmeq->cq_vector = -1;
1470                 return result;
1471         }
1472
1473         return result;
1474 }
1475
1476 static int nvme_create_io_queues(struct nvme_dev *dev)
1477 {
1478         unsigned i, max;
1479         int ret = 0;
1480
1481         for (i = dev->ctrl.queue_count; i <= dev->max_qid; i++) {
1482                 /* vector == qid - 1, match nvme_create_queue */
1483                 if (!nvme_alloc_queue(dev, i, dev->q_depth,
1484                      pci_irq_get_node(to_pci_dev(dev->dev), i - 1))) {
1485                         ret = -ENOMEM;
1486                         break;
1487                 }
1488         }
1489
1490         max = min(dev->max_qid, dev->ctrl.queue_count - 1);
1491         for (i = dev->online_queues; i <= max; i++) {
1492                 ret = nvme_create_queue(dev->queues[i], i);
1493                 if (ret)
1494                         break;
1495         }
1496
1497         /*
1498          * Ignore failing Create SQ/CQ commands, we can continue with less
1499          * than the desired aount of queues, and even a controller without
1500          * I/O queues an still be used to issue admin commands.  This might
1501          * be useful to upgrade a buggy firmware for example.
1502          */
1503         return ret >= 0 ? 0 : ret;
1504 }
1505
1506 static ssize_t nvme_cmb_show(struct device *dev,
1507                              struct device_attribute *attr,
1508                              char *buf)
1509 {
1510         struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
1511
1512         return scnprintf(buf, PAGE_SIZE, "cmbloc : x%08x\ncmbsz  : x%08x\n",
1513                        ndev->cmbloc, ndev->cmbsz);
1514 }
1515 static DEVICE_ATTR(cmb, S_IRUGO, nvme_cmb_show, NULL);
1516
1517 static void __iomem *nvme_map_cmb(struct nvme_dev *dev)
1518 {
1519         u64 szu, size, offset;
1520         resource_size_t bar_size;
1521         struct pci_dev *pdev = to_pci_dev(dev->dev);
1522         void __iomem *cmb;
1523         dma_addr_t dma_addr;
1524
1525         dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
1526         if (!(NVME_CMB_SZ(dev->cmbsz)))
1527                 return NULL;
1528         dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC);
1529
1530         if (!use_cmb_sqes)
1531                 return NULL;
1532
1533         szu = (u64)1 << (12 + 4 * NVME_CMB_SZU(dev->cmbsz));
1534         size = szu * NVME_CMB_SZ(dev->cmbsz);
1535         offset = szu * NVME_CMB_OFST(dev->cmbloc);
1536         bar_size = pci_resource_len(pdev, NVME_CMB_BIR(dev->cmbloc));
1537
1538         if (offset > bar_size)
1539                 return NULL;
1540
1541         /*
1542          * Controllers may support a CMB size larger than their BAR,
1543          * for example, due to being behind a bridge. Reduce the CMB to
1544          * the reported size of the BAR
1545          */
1546         if (size > bar_size - offset)
1547                 size = bar_size - offset;
1548
1549         dma_addr = pci_resource_start(pdev, NVME_CMB_BIR(dev->cmbloc)) + offset;
1550         cmb = ioremap_wc(dma_addr, size);
1551         if (!cmb)
1552                 return NULL;
1553
1554         dev->cmb_dma_addr = dma_addr;
1555         dev->cmb_size = size;
1556         return cmb;
1557 }
1558
1559 static inline void nvme_release_cmb(struct nvme_dev *dev)
1560 {
1561         if (dev->cmb) {
1562                 iounmap(dev->cmb);
1563                 dev->cmb = NULL;
1564                 sysfs_remove_file_from_group(&dev->ctrl.device->kobj,
1565                                              &dev_attr_cmb.attr, NULL);
1566                 dev->cmbsz = 0;
1567         }
1568 }
1569
1570 static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits)
1571 {
1572         u64 dma_addr = dev->host_mem_descs_dma;
1573         struct nvme_command c;
1574         int ret;
1575
1576         memset(&c, 0, sizeof(c));
1577         c.features.opcode       = nvme_admin_set_features;
1578         c.features.fid          = cpu_to_le32(NVME_FEAT_HOST_MEM_BUF);
1579         c.features.dword11      = cpu_to_le32(bits);
1580         c.features.dword12      = cpu_to_le32(dev->host_mem_size >>
1581                                               ilog2(dev->ctrl.page_size));
1582         c.features.dword13      = cpu_to_le32(lower_32_bits(dma_addr));
1583         c.features.dword14      = cpu_to_le32(upper_32_bits(dma_addr));
1584         c.features.dword15      = cpu_to_le32(dev->nr_host_mem_descs);
1585
1586         ret = nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1587         if (ret) {
1588                 dev_warn(dev->ctrl.device,
1589                          "failed to set host mem (err %d, flags %#x).\n",
1590                          ret, bits);
1591         }
1592         return ret;
1593 }
1594
1595 static void nvme_free_host_mem(struct nvme_dev *dev)
1596 {
1597         int i;
1598
1599         for (i = 0; i < dev->nr_host_mem_descs; i++) {
1600                 struct nvme_host_mem_buf_desc *desc = &dev->host_mem_descs[i];
1601                 size_t size = le32_to_cpu(desc->size) * dev->ctrl.page_size;
1602
1603                 dma_free_coherent(dev->dev, size, dev->host_mem_desc_bufs[i],
1604                                 le64_to_cpu(desc->addr));
1605         }
1606
1607         kfree(dev->host_mem_desc_bufs);
1608         dev->host_mem_desc_bufs = NULL;
1609         dma_free_coherent(dev->dev,
1610                         dev->nr_host_mem_descs * sizeof(*dev->host_mem_descs),
1611                         dev->host_mem_descs, dev->host_mem_descs_dma);
1612         dev->host_mem_descs = NULL;
1613 }
1614
1615 static int __nvme_alloc_host_mem(struct nvme_dev *dev, u64 preferred,
1616                 u32 chunk_size)
1617 {
1618         struct nvme_host_mem_buf_desc *descs;
1619         u32 max_entries, len;
1620         dma_addr_t descs_dma;
1621         int i = 0;
1622         void **bufs;
1623         u64 size = 0, tmp;
1624
1625         tmp = (preferred + chunk_size - 1);
1626         do_div(tmp, chunk_size);
1627         max_entries = tmp;
1628
1629         if (dev->ctrl.hmmaxd && dev->ctrl.hmmaxd < max_entries)
1630                 max_entries = dev->ctrl.hmmaxd;
1631
1632         descs = dma_zalloc_coherent(dev->dev, max_entries * sizeof(*descs),
1633                         &descs_dma, GFP_KERNEL);
1634         if (!descs)
1635                 goto out;
1636
1637         bufs = kcalloc(max_entries, sizeof(*bufs), GFP_KERNEL);
1638         if (!bufs)
1639                 goto out_free_descs;
1640
1641         for (size = 0; size < preferred; size += len) {
1642                 dma_addr_t dma_addr;
1643
1644                 len = min_t(u64, chunk_size, preferred - size);
1645                 bufs[i] = dma_alloc_attrs(dev->dev, len, &dma_addr, GFP_KERNEL,
1646                                 DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
1647                 if (!bufs[i])
1648                         break;
1649
1650                 descs[i].addr = cpu_to_le64(dma_addr);
1651                 descs[i].size = cpu_to_le32(len / dev->ctrl.page_size);
1652                 i++;
1653         }
1654
1655         if (!size)
1656                 goto out_free_bufs;
1657
1658         dev->nr_host_mem_descs = i;
1659         dev->host_mem_size = size;
1660         dev->host_mem_descs = descs;
1661         dev->host_mem_descs_dma = descs_dma;
1662         dev->host_mem_desc_bufs = bufs;
1663         return 0;
1664
1665 out_free_bufs:
1666         while (--i >= 0) {
1667                 size_t size = le32_to_cpu(descs[i].size) * dev->ctrl.page_size;
1668
1669                 dma_free_coherent(dev->dev, size, bufs[i],
1670                                 le64_to_cpu(descs[i].addr));
1671         }
1672
1673         kfree(bufs);
1674 out_free_descs:
1675         dma_free_coherent(dev->dev, max_entries * sizeof(*descs), descs,
1676                         descs_dma);
1677 out:
1678         dev->host_mem_descs = NULL;
1679         return -ENOMEM;
1680 }
1681
1682 static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred)
1683 {
1684         u32 chunk_size;
1685
1686         /* start big and work our way down */
1687         for (chunk_size = min_t(u64, preferred, PAGE_SIZE * MAX_ORDER_NR_PAGES);
1688              chunk_size >= max_t(u32, dev->ctrl.hmminds * 4096, PAGE_SIZE * 2);
1689              chunk_size /= 2) {
1690                 if (!__nvme_alloc_host_mem(dev, preferred, chunk_size)) {
1691                         if (!min || dev->host_mem_size >= min)
1692                                 return 0;
1693                         nvme_free_host_mem(dev);
1694                 }
1695         }
1696
1697         return -ENOMEM;
1698 }
1699
1700 static int nvme_setup_host_mem(struct nvme_dev *dev)
1701 {
1702         u64 max = (u64)max_host_mem_size_mb * SZ_1M;
1703         u64 preferred = (u64)dev->ctrl.hmpre * 4096;
1704         u64 min = (u64)dev->ctrl.hmmin * 4096;
1705         u32 enable_bits = NVME_HOST_MEM_ENABLE;
1706         int ret = 0;
1707
1708         preferred = min(preferred, max);
1709         if (min > max) {
1710                 dev_warn(dev->ctrl.device,
1711                         "min host memory (%lld MiB) above limit (%d MiB).\n",
1712                         min >> ilog2(SZ_1M), max_host_mem_size_mb);
1713                 nvme_free_host_mem(dev);
1714                 return 0;
1715         }
1716
1717         /*
1718          * If we already have a buffer allocated check if we can reuse it.
1719          */
1720         if (dev->host_mem_descs) {
1721                 if (dev->host_mem_size >= min)
1722                         enable_bits |= NVME_HOST_MEM_RETURN;
1723                 else
1724                         nvme_free_host_mem(dev);
1725         }
1726
1727         if (!dev->host_mem_descs) {
1728                 if (nvme_alloc_host_mem(dev, min, preferred)) {
1729                         dev_warn(dev->ctrl.device,
1730                                 "failed to allocate host memory buffer.\n");
1731                         return 0; /* controller must work without HMB */
1732                 }
1733
1734                 dev_info(dev->ctrl.device,
1735                         "allocated %lld MiB host memory buffer.\n",
1736                         dev->host_mem_size >> ilog2(SZ_1M));
1737         }
1738
1739         ret = nvme_set_host_mem(dev, enable_bits);
1740         if (ret)
1741                 nvme_free_host_mem(dev);
1742         return ret;
1743 }
1744
1745 static int nvme_setup_io_queues(struct nvme_dev *dev)
1746 {
1747         struct nvme_queue *adminq = dev->queues[0];
1748         struct pci_dev *pdev = to_pci_dev(dev->dev);
1749         int result, nr_io_queues;
1750         unsigned long size;
1751
1752         nr_io_queues = num_present_cpus();
1753         result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
1754         if (result < 0)
1755                 return result;
1756
1757         if (nr_io_queues == 0)
1758                 return 0;
1759
1760         if (dev->cmb && NVME_CMB_SQS(dev->cmbsz)) {
1761                 result = nvme_cmb_qdepth(dev, nr_io_queues,
1762                                 sizeof(struct nvme_command));
1763                 if (result > 0)
1764                         dev->q_depth = result;
1765                 else
1766                         nvme_release_cmb(dev);
1767         }
1768
1769         do {
1770                 size = db_bar_size(dev, nr_io_queues);
1771                 result = nvme_remap_bar(dev, size);
1772                 if (!result)
1773                         break;
1774                 if (!--nr_io_queues)
1775                         return -ENOMEM;
1776         } while (1);
1777         adminq->q_db = dev->dbs;
1778
1779         /* Deregister the admin queue's interrupt */
1780         pci_free_irq(pdev, 0, adminq);
1781
1782         /*
1783          * If we enable msix early due to not intx, disable it again before
1784          * setting up the full range we need.
1785          */
1786         pci_free_irq_vectors(pdev);
1787         nr_io_queues = pci_alloc_irq_vectors(pdev, 1, nr_io_queues,
1788                         PCI_IRQ_ALL_TYPES | PCI_IRQ_AFFINITY);
1789         if (nr_io_queues <= 0)
1790                 return -EIO;
1791         dev->max_qid = nr_io_queues;
1792
1793         /*
1794          * Should investigate if there's a performance win from allocating
1795          * more queues than interrupt vectors; it might allow the submission
1796          * path to scale better, even if the receive path is limited by the
1797          * number of interrupts.
1798          */
1799
1800         result = queue_request_irq(adminq);
1801         if (result) {
1802                 adminq->cq_vector = -1;
1803                 return result;
1804         }
1805         return nvme_create_io_queues(dev);
1806 }
1807
1808 static void nvme_del_queue_end(struct request *req, blk_status_t error)
1809 {
1810         struct nvme_queue *nvmeq = req->end_io_data;
1811
1812         blk_mq_free_request(req);
1813         complete(&nvmeq->dev->ioq_wait);
1814 }
1815
1816 static void nvme_del_cq_end(struct request *req, blk_status_t error)
1817 {
1818         struct nvme_queue *nvmeq = req->end_io_data;
1819
1820         if (!error) {
1821                 unsigned long flags;
1822
1823                 /*
1824                  * We might be called with the AQ q_lock held
1825                  * and the I/O queue q_lock should always
1826                  * nest inside the AQ one.
1827                  */
1828                 spin_lock_irqsave_nested(&nvmeq->q_lock, flags,
1829                                         SINGLE_DEPTH_NESTING);
1830                 nvme_process_cq(nvmeq);
1831                 spin_unlock_irqrestore(&nvmeq->q_lock, flags);
1832         }
1833
1834         nvme_del_queue_end(req, error);
1835 }
1836
1837 static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode)
1838 {
1839         struct request_queue *q = nvmeq->dev->ctrl.admin_q;
1840         struct request *req;
1841         struct nvme_command cmd;
1842
1843         memset(&cmd, 0, sizeof(cmd));
1844         cmd.delete_queue.opcode = opcode;
1845         cmd.delete_queue.qid = cpu_to_le16(nvmeq->qid);
1846
1847         req = nvme_alloc_request(q, &cmd, BLK_MQ_REQ_NOWAIT, NVME_QID_ANY);
1848         if (IS_ERR(req))
1849                 return PTR_ERR(req);
1850
1851         req->timeout = ADMIN_TIMEOUT;
1852         req->end_io_data = nvmeq;
1853
1854         blk_execute_rq_nowait(q, NULL, req, false,
1855                         opcode == nvme_admin_delete_cq ?
1856                                 nvme_del_cq_end : nvme_del_queue_end);
1857         return 0;
1858 }
1859
1860 static void nvme_disable_io_queues(struct nvme_dev *dev, int queues)
1861 {
1862         int pass;
1863         unsigned long timeout;
1864         u8 opcode = nvme_admin_delete_sq;
1865
1866         for (pass = 0; pass < 2; pass++) {
1867                 int sent = 0, i = queues;
1868
1869                 reinit_completion(&dev->ioq_wait);
1870  retry:
1871                 timeout = ADMIN_TIMEOUT;
1872                 for (; i > 0; i--, sent++)
1873                         if (nvme_delete_queue(dev->queues[i], opcode))
1874                                 break;
1875
1876                 while (sent--) {
1877                         timeout = wait_for_completion_io_timeout(&dev->ioq_wait, timeout);
1878                         if (timeout == 0)
1879                                 return;
1880                         if (i)
1881                                 goto retry;
1882                 }
1883                 opcode = nvme_admin_delete_cq;
1884         }
1885 }
1886
1887 /*
1888  * Return: error value if an error occurred setting up the queues or calling
1889  * Identify Device.  0 if these succeeded, even if adding some of the
1890  * namespaces failed.  At the moment, these failures are silent.  TBD which
1891  * failures should be reported.
1892  */
1893 static int nvme_dev_add(struct nvme_dev *dev)
1894 {
1895         if (!dev->ctrl.tagset) {
1896                 dev->tagset.ops = &nvme_mq_ops;
1897                 dev->tagset.nr_hw_queues = dev->online_queues - 1;
1898                 dev->tagset.timeout = NVME_IO_TIMEOUT;
1899                 dev->tagset.numa_node = dev_to_node(dev->dev);
1900                 dev->tagset.queue_depth =
1901                                 min_t(int, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1;
1902                 dev->tagset.cmd_size = nvme_cmd_size(dev);
1903                 dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
1904                 dev->tagset.driver_data = dev;
1905
1906                 if (blk_mq_alloc_tag_set(&dev->tagset))
1907                         return 0;
1908                 dev->ctrl.tagset = &dev->tagset;
1909
1910                 nvme_dbbuf_set(dev);
1911         } else {
1912                 blk_mq_update_nr_hw_queues(&dev->tagset, dev->online_queues - 1);
1913
1914                 /* Free previously allocated queues that are no longer usable */
1915                 nvme_free_queues(dev, dev->online_queues);
1916         }
1917
1918         return 0;
1919 }
1920
1921 static int nvme_pci_enable(struct nvme_dev *dev)
1922 {
1923         int result = -ENOMEM;
1924         struct pci_dev *pdev = to_pci_dev(dev->dev);
1925
1926         if (pci_enable_device_mem(pdev))
1927                 return result;
1928
1929         pci_set_master(pdev);
1930
1931         if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64)) &&
1932             dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(32)))
1933                 goto disable;
1934
1935         if (readl(dev->bar + NVME_REG_CSTS) == -1) {
1936                 result = -ENODEV;
1937                 goto disable;
1938         }
1939
1940         /*
1941          * Some devices and/or platforms don't advertise or work with INTx
1942          * interrupts. Pre-enable a single MSIX or MSI vec for setup. We'll
1943          * adjust this later.
1944          */
1945         result = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES);
1946         if (result < 0)
1947                 return result;
1948
1949         dev->ctrl.cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
1950
1951         dev->q_depth = min_t(int, NVME_CAP_MQES(dev->ctrl.cap) + 1,
1952                                 io_queue_depth);
1953         dev->db_stride = 1 << NVME_CAP_STRIDE(dev->ctrl.cap);
1954         dev->dbs = dev->bar + 4096;
1955
1956         /*
1957          * Temporary fix for the Apple controller found in the MacBook8,1 and
1958          * some MacBook7,1 to avoid controller resets and data loss.
1959          */
1960         if (pdev->vendor == PCI_VENDOR_ID_APPLE && pdev->device == 0x2001) {
1961                 dev->q_depth = 2;
1962                 dev_warn(dev->ctrl.device, "detected Apple NVMe controller, "
1963                         "set queue depth=%u to work around controller resets\n",
1964                         dev->q_depth);
1965         } else if (pdev->vendor == PCI_VENDOR_ID_SAMSUNG &&
1966                    (pdev->device == 0xa821 || pdev->device == 0xa822) &&
1967                    NVME_CAP_MQES(dev->ctrl.cap) == 0) {
1968                 dev->q_depth = 64;
1969                 dev_err(dev->ctrl.device, "detected PM1725 NVMe controller, "
1970                         "set queue depth=%u\n", dev->q_depth);
1971         }
1972
1973         /*
1974          * CMBs can currently only exist on >=1.2 PCIe devices. We only
1975          * populate sysfs if a CMB is implemented. Since nvme_dev_attrs_group
1976          * has no name we can pass NULL as final argument to
1977          * sysfs_add_file_to_group.
1978          */
1979
1980         if (readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 2, 0)) {
1981                 dev->cmb = nvme_map_cmb(dev);
1982                 if (dev->cmb) {
1983                         if (sysfs_add_file_to_group(&dev->ctrl.device->kobj,
1984                                                     &dev_attr_cmb.attr, NULL))
1985                                 dev_warn(dev->ctrl.device,
1986                                          "failed to add sysfs attribute for CMB\n");
1987                 }
1988         }
1989
1990         pci_enable_pcie_error_reporting(pdev);
1991         pci_save_state(pdev);
1992         return 0;
1993
1994  disable:
1995         pci_disable_device(pdev);
1996         return result;
1997 }
1998
1999 static void nvme_dev_unmap(struct nvme_dev *dev)
2000 {
2001         if (dev->bar)
2002                 iounmap(dev->bar);
2003         pci_release_mem_regions(to_pci_dev(dev->dev));
2004 }
2005
2006 static void nvme_pci_disable(struct nvme_dev *dev)
2007 {
2008         struct pci_dev *pdev = to_pci_dev(dev->dev);
2009
2010         nvme_release_cmb(dev);
2011         pci_free_irq_vectors(pdev);
2012
2013         if (pci_is_enabled(pdev)) {
2014                 pci_disable_pcie_error_reporting(pdev);
2015                 pci_disable_device(pdev);
2016         }
2017 }
2018
2019 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
2020 {
2021         int i, queues;
2022         bool dead = true;
2023         struct pci_dev *pdev = to_pci_dev(dev->dev);
2024
2025         mutex_lock(&dev->shutdown_lock);
2026         if (pci_is_enabled(pdev)) {
2027                 u32 csts = readl(dev->bar + NVME_REG_CSTS);
2028
2029                 if (dev->ctrl.state == NVME_CTRL_LIVE ||
2030                     dev->ctrl.state == NVME_CTRL_RESETTING)
2031                         nvme_start_freeze(&dev->ctrl);
2032                 dead = !!((csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY) ||
2033                         pdev->error_state  != pci_channel_io_normal);
2034         }
2035
2036         /*
2037          * Give the controller a chance to complete all entered requests if
2038          * doing a safe shutdown.
2039          */
2040         if (!dead) {
2041                 if (shutdown)
2042                         nvme_wait_freeze_timeout(&dev->ctrl, NVME_IO_TIMEOUT);
2043
2044                 /*
2045                  * If the controller is still alive tell it to stop using the
2046                  * host memory buffer.  In theory the shutdown / reset should
2047                  * make sure that it doesn't access the host memoery anymore,
2048                  * but I'd rather be safe than sorry..
2049                  */
2050                 if (dev->host_mem_descs)
2051                         nvme_set_host_mem(dev, 0);
2052
2053         }
2054         nvme_stop_queues(&dev->ctrl);
2055
2056         queues = dev->online_queues - 1;
2057         for (i = dev->ctrl.queue_count - 1; i > 0; i--)
2058                 nvme_suspend_queue(dev->queues[i]);
2059
2060         if (dead) {
2061                 /* A device might become IO incapable very soon during
2062                  * probe, before the admin queue is configured. Thus,
2063                  * queue_count can be 0 here.
2064                  */
2065                 if (dev->ctrl.queue_count)
2066                         nvme_suspend_queue(dev->queues[0]);
2067         } else {
2068                 nvme_disable_io_queues(dev, queues);
2069                 nvme_disable_admin_queue(dev, shutdown);
2070         }
2071         nvme_pci_disable(dev);
2072
2073         blk_mq_tagset_busy_iter(&dev->tagset, nvme_cancel_request, &dev->ctrl);
2074         blk_mq_tagset_busy_iter(&dev->admin_tagset, nvme_cancel_request, &dev->ctrl);
2075
2076         /*
2077          * The driver will not be starting up queues again if shutting down so
2078          * must flush all entered requests to their failed completion to avoid
2079          * deadlocking blk-mq hot-cpu notifier.
2080          */
2081         if (shutdown)
2082                 nvme_start_queues(&dev->ctrl);
2083         mutex_unlock(&dev->shutdown_lock);
2084 }
2085
2086 static int nvme_setup_prp_pools(struct nvme_dev *dev)
2087 {
2088         dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
2089                                                 PAGE_SIZE, PAGE_SIZE, 0);
2090         if (!dev->prp_page_pool)
2091                 return -ENOMEM;
2092
2093         /* Optimisation for I/Os between 4k and 128k */
2094         dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev,
2095                                                 256, 256, 0);
2096         if (!dev->prp_small_pool) {
2097                 dma_pool_destroy(dev->prp_page_pool);
2098                 return -ENOMEM;
2099         }
2100         return 0;
2101 }
2102
2103 static void nvme_release_prp_pools(struct nvme_dev *dev)
2104 {
2105         dma_pool_destroy(dev->prp_page_pool);
2106         dma_pool_destroy(dev->prp_small_pool);
2107 }
2108
2109 static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
2110 {
2111         struct nvme_dev *dev = to_nvme_dev(ctrl);
2112
2113         nvme_dbbuf_dma_free(dev);
2114         put_device(dev->dev);
2115         if (dev->tagset.tags)
2116                 blk_mq_free_tag_set(&dev->tagset);
2117         if (dev->ctrl.admin_q)
2118                 blk_put_queue(dev->ctrl.admin_q);
2119         kfree(dev->queues);
2120         free_opal_dev(dev->ctrl.opal_dev);
2121         kfree(dev);
2122 }
2123
2124 static void nvme_remove_dead_ctrl(struct nvme_dev *dev, int status)
2125 {
2126         dev_warn(dev->ctrl.device, "Removing after probe failure status: %d\n", status);
2127
2128         kref_get(&dev->ctrl.kref);
2129         nvme_dev_disable(dev, false);
2130         if (!schedule_work(&dev->remove_work))
2131                 nvme_put_ctrl(&dev->ctrl);
2132 }
2133
2134 static void nvme_reset_work(struct work_struct *work)
2135 {
2136         struct nvme_dev *dev =
2137                 container_of(work, struct nvme_dev, ctrl.reset_work);
2138         bool was_suspend = !!(dev->ctrl.ctrl_config & NVME_CC_SHN_NORMAL);
2139         int result = -ENODEV;
2140
2141         if (WARN_ON(dev->ctrl.state != NVME_CTRL_RESETTING))
2142                 goto out;
2143
2144         /*
2145          * If we're called to reset a live controller first shut it down before
2146          * moving on.
2147          */
2148         if (dev->ctrl.ctrl_config & NVME_CC_ENABLE)
2149                 nvme_dev_disable(dev, false);
2150
2151         result = nvme_pci_enable(dev);
2152         if (result)
2153                 goto out;
2154
2155         result = nvme_pci_configure_admin_queue(dev);
2156         if (result)
2157                 goto out;
2158
2159         nvme_init_queue(dev->queues[0], 0);
2160         result = nvme_alloc_admin_tags(dev);
2161         if (result)
2162                 goto out;
2163
2164         result = nvme_init_identify(&dev->ctrl);
2165         if (result)
2166                 goto out;
2167
2168         if (dev->ctrl.oacs & NVME_CTRL_OACS_SEC_SUPP) {
2169                 if (!dev->ctrl.opal_dev)
2170                         dev->ctrl.opal_dev =
2171                                 init_opal_dev(&dev->ctrl, &nvme_sec_submit);
2172                 else if (was_suspend)
2173                         opal_unlock_from_suspend(dev->ctrl.opal_dev);
2174         } else {
2175                 free_opal_dev(dev->ctrl.opal_dev);
2176                 dev->ctrl.opal_dev = NULL;
2177         }
2178
2179         if (dev->ctrl.oacs & NVME_CTRL_OACS_DBBUF_SUPP) {
2180                 result = nvme_dbbuf_dma_alloc(dev);
2181                 if (result)
2182                         dev_warn(dev->dev,
2183                                  "unable to allocate dma for dbbuf\n");
2184         }
2185
2186         if (dev->ctrl.hmpre) {
2187                 result = nvme_setup_host_mem(dev);
2188                 if (result < 0)
2189                         goto out;
2190         }
2191
2192         result = nvme_setup_io_queues(dev);
2193         if (result)
2194                 goto out;
2195
2196         /*
2197          * Keep the controller around but remove all namespaces if we don't have
2198          * any working I/O queue.
2199          */
2200         if (dev->online_queues < 2) {
2201                 dev_warn(dev->ctrl.device, "IO queues not created\n");
2202                 nvme_kill_queues(&dev->ctrl);
2203                 nvme_remove_namespaces(&dev->ctrl);
2204         } else {
2205                 nvme_start_queues(&dev->ctrl);
2206                 nvme_wait_freeze(&dev->ctrl);
2207                 nvme_dev_add(dev);
2208                 nvme_unfreeze(&dev->ctrl);
2209         }
2210
2211         if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) {
2212                 dev_warn(dev->ctrl.device, "failed to mark controller live\n");
2213                 goto out;
2214         }
2215
2216         nvme_start_ctrl(&dev->ctrl);
2217         return;
2218
2219  out:
2220         nvme_remove_dead_ctrl(dev, result);
2221 }
2222
2223 static void nvme_remove_dead_ctrl_work(struct work_struct *work)
2224 {
2225         struct nvme_dev *dev = container_of(work, struct nvme_dev, remove_work);
2226         struct pci_dev *pdev = to_pci_dev(dev->dev);
2227
2228         nvme_kill_queues(&dev->ctrl);
2229         if (pci_get_drvdata(pdev))
2230                 device_release_driver(&pdev->dev);
2231         nvme_put_ctrl(&dev->ctrl);
2232 }
2233
2234 static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
2235 {
2236         *val = readl(to_nvme_dev(ctrl)->bar + off);
2237         return 0;
2238 }
2239
2240 static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
2241 {
2242         writel(val, to_nvme_dev(ctrl)->bar + off);
2243         return 0;
2244 }
2245
2246 static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
2247 {
2248         *val = readq(to_nvme_dev(ctrl)->bar + off);
2249         return 0;
2250 }
2251
2252 static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
2253         .name                   = "pcie",
2254         .module                 = THIS_MODULE,
2255         .flags                  = NVME_F_METADATA_SUPPORTED,
2256         .reg_read32             = nvme_pci_reg_read32,
2257         .reg_write32            = nvme_pci_reg_write32,
2258         .reg_read64             = nvme_pci_reg_read64,
2259         .free_ctrl              = nvme_pci_free_ctrl,
2260         .submit_async_event     = nvme_pci_submit_async_event,
2261 };
2262
2263 static int nvme_dev_map(struct nvme_dev *dev)
2264 {
2265         struct pci_dev *pdev = to_pci_dev(dev->dev);
2266
2267         if (pci_request_mem_regions(pdev, "nvme"))
2268                 return -ENODEV;
2269
2270         if (nvme_remap_bar(dev, NVME_REG_DBS + 4096))
2271                 goto release;
2272
2273         return 0;
2274   release:
2275         pci_release_mem_regions(pdev);
2276         return -ENODEV;
2277 }
2278
2279 static unsigned long check_dell_samsung_bug(struct pci_dev *pdev)
2280 {
2281         if (pdev->vendor == 0x144d && pdev->device == 0xa802) {
2282                 /*
2283                  * Several Samsung devices seem to drop off the PCIe bus
2284                  * randomly when APST is on and uses the deepest sleep state.
2285                  * This has been observed on a Samsung "SM951 NVMe SAMSUNG
2286                  * 256GB", a "PM951 NVMe SAMSUNG 512GB", and a "Samsung SSD
2287                  * 950 PRO 256GB", but it seems to be restricted to two Dell
2288                  * laptops.
2289                  */
2290                 if (dmi_match(DMI_SYS_VENDOR, "Dell Inc.") &&
2291                     (dmi_match(DMI_PRODUCT_NAME, "XPS 15 9550") ||
2292                      dmi_match(DMI_PRODUCT_NAME, "Precision 5510")))
2293                         return NVME_QUIRK_NO_DEEPEST_PS;
2294         }
2295
2296         return 0;
2297 }
2298
2299 static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2300 {
2301         int node, result = -ENOMEM;
2302         struct nvme_dev *dev;
2303         unsigned long quirks = id->driver_data;
2304
2305         node = dev_to_node(&pdev->dev);
2306         if (node == NUMA_NO_NODE)
2307                 set_dev_node(&pdev->dev, first_memory_node);
2308
2309         dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
2310         if (!dev)
2311                 return -ENOMEM;
2312         dev->queues = kzalloc_node((num_possible_cpus() + 1) * sizeof(void *),
2313                                                         GFP_KERNEL, node);
2314         if (!dev->queues)
2315                 goto free;
2316
2317         dev->dev = get_device(&pdev->dev);
2318         pci_set_drvdata(pdev, dev);
2319
2320         result = nvme_dev_map(dev);
2321         if (result)
2322                 goto put_pci;
2323
2324         INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work);
2325         INIT_WORK(&dev->remove_work, nvme_remove_dead_ctrl_work);
2326         mutex_init(&dev->shutdown_lock);
2327         init_completion(&dev->ioq_wait);
2328
2329         result = nvme_setup_prp_pools(dev);
2330         if (result)
2331                 goto unmap;
2332
2333         quirks |= check_dell_samsung_bug(pdev);
2334
2335         result = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops,
2336                         quirks);
2337         if (result)
2338                 goto release_pools;
2339
2340         nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING);
2341         dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev));
2342
2343         queue_work(nvme_wq, &dev->ctrl.reset_work);
2344         return 0;
2345
2346  release_pools:
2347         nvme_release_prp_pools(dev);
2348  unmap:
2349         nvme_dev_unmap(dev);
2350  put_pci:
2351         put_device(dev->dev);
2352  free:
2353         kfree(dev->queues);
2354         kfree(dev);
2355         return result;
2356 }
2357
2358 static void nvme_reset_prepare(struct pci_dev *pdev)
2359 {
2360         struct nvme_dev *dev = pci_get_drvdata(pdev);
2361         nvme_dev_disable(dev, false);
2362 }
2363
2364 static void nvme_reset_done(struct pci_dev *pdev)
2365 {
2366         struct nvme_dev *dev = pci_get_drvdata(pdev);
2367         nvme_reset_ctrl(&dev->ctrl);
2368 }
2369
2370 static void nvme_shutdown(struct pci_dev *pdev)
2371 {
2372         struct nvme_dev *dev = pci_get_drvdata(pdev);
2373         nvme_dev_disable(dev, true);
2374 }
2375
2376 /*
2377  * The driver's remove may be called on a device in a partially initialized
2378  * state. This function must not have any dependencies on the device state in
2379  * order to proceed.
2380  */
2381 static void nvme_remove(struct pci_dev *pdev)
2382 {
2383         struct nvme_dev *dev = pci_get_drvdata(pdev);
2384
2385         nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
2386
2387         cancel_work_sync(&dev->ctrl.reset_work);
2388         pci_set_drvdata(pdev, NULL);
2389
2390         if (!pci_device_is_present(pdev)) {
2391                 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD);
2392                 nvme_dev_disable(dev, false);
2393         }
2394
2395         flush_work(&dev->ctrl.reset_work);
2396         nvme_stop_ctrl(&dev->ctrl);
2397         nvme_remove_namespaces(&dev->ctrl);
2398         nvme_dev_disable(dev, true);
2399         nvme_free_host_mem(dev);
2400         nvme_dev_remove_admin(dev);
2401         nvme_free_queues(dev, 0);
2402         nvme_uninit_ctrl(&dev->ctrl);
2403         nvme_release_prp_pools(dev);
2404         nvme_dev_unmap(dev);
2405         nvme_put_ctrl(&dev->ctrl);
2406 }
2407
2408 static int nvme_pci_sriov_configure(struct pci_dev *pdev, int numvfs)
2409 {
2410         int ret = 0;
2411
2412         if (numvfs == 0) {
2413                 if (pci_vfs_assigned(pdev)) {
2414                         dev_warn(&pdev->dev,
2415                                 "Cannot disable SR-IOV VFs while assigned\n");
2416                         return -EPERM;
2417                 }
2418                 pci_disable_sriov(pdev);
2419                 return 0;
2420         }
2421
2422         ret = pci_enable_sriov(pdev, numvfs);
2423         return ret ? ret : numvfs;
2424 }
2425
2426 #ifdef CONFIG_PM_SLEEP
2427 static int nvme_suspend(struct device *dev)
2428 {
2429         struct pci_dev *pdev = to_pci_dev(dev);
2430         struct nvme_dev *ndev = pci_get_drvdata(pdev);
2431
2432         nvme_dev_disable(ndev, true);
2433         return 0;
2434 }
2435
2436 static int nvme_resume(struct device *dev)
2437 {
2438         struct pci_dev *pdev = to_pci_dev(dev);
2439         struct nvme_dev *ndev = pci_get_drvdata(pdev);
2440
2441         nvme_reset_ctrl(&ndev->ctrl);
2442         return 0;
2443 }
2444 #endif
2445
2446 static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
2447
2448 static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev,
2449                                                 pci_channel_state_t state)
2450 {
2451         struct nvme_dev *dev = pci_get_drvdata(pdev);
2452
2453         /*
2454          * A frozen channel requires a reset. When detected, this method will
2455          * shutdown the controller to quiesce. The controller will be restarted
2456          * after the slot reset through driver's slot_reset callback.
2457          */
2458         switch (state) {
2459         case pci_channel_io_normal:
2460                 return PCI_ERS_RESULT_CAN_RECOVER;
2461         case pci_channel_io_frozen:
2462                 dev_warn(dev->ctrl.device,
2463                         "frozen state error detected, reset controller\n");
2464                 nvme_dev_disable(dev, false);
2465                 return PCI_ERS_RESULT_NEED_RESET;
2466         case pci_channel_io_perm_failure:
2467                 dev_warn(dev->ctrl.device,
2468                         "failure state error detected, request disconnect\n");
2469                 return PCI_ERS_RESULT_DISCONNECT;
2470         }
2471         return PCI_ERS_RESULT_NEED_RESET;
2472 }
2473
2474 static pci_ers_result_t nvme_slot_reset(struct pci_dev *pdev)
2475 {
2476         struct nvme_dev *dev = pci_get_drvdata(pdev);
2477
2478         dev_info(dev->ctrl.device, "restart after slot reset\n");
2479         pci_restore_state(pdev);
2480         nvme_reset_ctrl(&dev->ctrl);
2481         return PCI_ERS_RESULT_RECOVERED;
2482 }
2483
2484 static void nvme_error_resume(struct pci_dev *pdev)
2485 {
2486         pci_cleanup_aer_uncorrect_error_status(pdev);
2487 }
2488
2489 static const struct pci_error_handlers nvme_err_handler = {
2490         .error_detected = nvme_error_detected,
2491         .slot_reset     = nvme_slot_reset,
2492         .resume         = nvme_error_resume,
2493         .reset_prepare  = nvme_reset_prepare,
2494         .reset_done     = nvme_reset_done,
2495 };
2496
2497 static const struct pci_device_id nvme_id_table[] = {
2498         { PCI_VDEVICE(INTEL, 0x0953),
2499                 .driver_data = NVME_QUIRK_STRIPE_SIZE |
2500                                 NVME_QUIRK_DEALLOCATE_ZEROES, },
2501         { PCI_VDEVICE(INTEL, 0x0a53),
2502                 .driver_data = NVME_QUIRK_STRIPE_SIZE |
2503                                 NVME_QUIRK_DEALLOCATE_ZEROES, },
2504         { PCI_VDEVICE(INTEL, 0x0a54),
2505                 .driver_data = NVME_QUIRK_STRIPE_SIZE |
2506                                 NVME_QUIRK_DEALLOCATE_ZEROES, },
2507         { PCI_VDEVICE(INTEL, 0x0a55),
2508                 .driver_data = NVME_QUIRK_STRIPE_SIZE |
2509                                 NVME_QUIRK_DEALLOCATE_ZEROES, },
2510         { PCI_VDEVICE(INTEL, 0xf1a5),   /* Intel 600P/P3100 */
2511                 .driver_data = NVME_QUIRK_NO_DEEPEST_PS },
2512         { PCI_VDEVICE(INTEL, 0x5845),   /* Qemu emulated controller */
2513                 .driver_data = NVME_QUIRK_IDENTIFY_CNS, },
2514         { PCI_DEVICE(0x1c58, 0x0003),   /* HGST adapter */
2515                 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
2516         { PCI_DEVICE(0x1c5f, 0x0540),   /* Memblaze Pblaze4 adapter */
2517                 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
2518         { PCI_DEVICE(0x144d, 0xa821),   /* Samsung PM1725 */
2519                 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
2520         { PCI_DEVICE(0x144d, 0xa822),   /* Samsung PM1725a */
2521                 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
2522         { PCI_DEVICE(0x1d1d, 0x1f1f),   /* LighNVM qemu device */
2523                 .driver_data = NVME_QUIRK_LIGHTNVM, },
2524         { PCI_DEVICE(0x1d1d, 0x2807),   /* CNEX WL */
2525                 .driver_data = NVME_QUIRK_LIGHTNVM, },
2526         { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
2527         { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) },
2528         { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) },
2529         { 0, }
2530 };
2531 MODULE_DEVICE_TABLE(pci, nvme_id_table);
2532
2533 static struct pci_driver nvme_driver = {
2534         .name           = "nvme",
2535         .id_table       = nvme_id_table,
2536         .probe          = nvme_probe,
2537         .remove         = nvme_remove,
2538         .shutdown       = nvme_shutdown,
2539         .driver         = {
2540                 .pm     = &nvme_dev_pm_ops,
2541         },
2542         .sriov_configure = nvme_pci_sriov_configure,
2543         .err_handler    = &nvme_err_handler,
2544 };
2545
2546 static int __init nvme_init(void)
2547 {
2548         return pci_register_driver(&nvme_driver);
2549 }
2550
2551 static void __exit nvme_exit(void)
2552 {
2553         pci_unregister_driver(&nvme_driver);
2554         _nvme_check_size();
2555 }
2556
2557 MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
2558 MODULE_LICENSE("GPL");
2559 MODULE_VERSION("1.0");
2560 module_init(nvme_init);
2561 module_exit(nvme_exit);