drivers/net/ethernet/fungible/funcore/fun_dev.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
   2
   3 #include <linux/aer.h>
   4 #include <linux/bitmap.h>
   5 #include <linux/delay.h>
   6 #include <linux/interrupt.h>
   7 #include <linux/io.h>
   8 #include <linux/io-64-nonatomic-lo-hi.h>
   9 #include <linux/mm.h>
  10 #include <linux/module.h>
  11 #include <linux/nvme.h>
  12 #include <linux/pci.h>
  13 #include <linux/wait.h>
  14 #include <linux/sched/signal.h>
  15
  16 #include "fun_queue.h"
  17 #include "fun_dev.h"
  18
  19 #define FUN_ADMIN_CMD_TO_MS 3000
  20
  21 enum {
  22         AQA_ASQS_SHIFT = 0,
  23         AQA_ACQS_SHIFT = 16,
  24         AQA_MIN_QUEUE_SIZE = 2,
  25         AQA_MAX_QUEUE_SIZE = 4096
  26 };
  27
  28 /* context for admin commands */
  29 struct fun_cmd_ctx {
  30         fun_admin_callback_t cb;  /* callback to invoke on completion */
  31         void *cb_data;            /* user data provided to callback */
  32         int cpu;                  /* CPU where the cmd's tag was allocated */
  33 };
  34
  35 /* Context for synchronous admin commands. */
  36 struct fun_sync_cmd_ctx {
  37         struct completion compl;
  38         u8 *rsp_buf;              /* caller provided response buffer */
  39         unsigned int rsp_len;     /* response buffer size */
  40         u8 rsp_status;            /* command response status */
  41 };
  42
  43 /* Wait for the CSTS.RDY bit to match @enabled. */
  44 static int fun_wait_ready(struct fun_dev *fdev, bool enabled)
  45 {
  46         unsigned int cap_to = NVME_CAP_TIMEOUT(fdev->cap_reg);
  47         u32 bit = enabled ? NVME_CSTS_RDY : 0;
  48         unsigned long deadline;
  49
  50         deadline = ((cap_to + 1) * HZ / 2) + jiffies; /* CAP.TO is in 500ms */
  51
  52         for (;;) {
  53                 u32 csts = readl(fdev->bar + NVME_REG_CSTS);
  54
  55                 if (csts == ~0) {
  56                         dev_err(fdev->dev, "CSTS register read %#x\n", csts);
  57                         return -EIO;
  58                 }
  59
  60                 if ((csts & NVME_CSTS_RDY) == bit)
  61                         return 0;
  62
  63                 if (time_is_before_jiffies(deadline))
  64                         break;
  65
  66                 msleep(100);
  67         }
  68
  69         dev_err(fdev->dev,
  70                 "Timed out waiting for device to indicate RDY %u; aborting %s\n",
  71                 enabled, enabled ? "initialization" : "reset");
  72         return -ETIMEDOUT;
  73 }
  74
  75 /* Check CSTS and return an error if it is unreadable or has unexpected
  76  * RDY value.
  77  */
  78 static int fun_check_csts_rdy(struct fun_dev *fdev, unsigned int expected_rdy)
  79 {
  80         u32 csts = readl(fdev->bar + NVME_REG_CSTS);
  81         u32 actual_rdy = csts & NVME_CSTS_RDY;
  82
  83         if (csts == ~0) {
  84                 dev_err(fdev->dev, "CSTS register read %#x\n", csts);
  85                 return -EIO;
  86         }
  87         if (actual_rdy != expected_rdy) {
  88                 dev_err(fdev->dev, "Unexpected CSTS RDY %u\n", actual_rdy);
  89                 return -EINVAL;
  90         }
  91         return 0;
  92 }
  93
  94 /* Check that CSTS RDY has the expected value. Then write a new value to the CC
  95  * register and wait for CSTS RDY to match the new CC ENABLE state.
  96  */
  97 static int fun_update_cc_enable(struct fun_dev *fdev, unsigned int initial_rdy)
  98 {
  99         int rc = fun_check_csts_rdy(fdev, initial_rdy);
 100
 101         if (rc)
 102                 return rc;
 103         writel(fdev->cc_reg, fdev->bar + NVME_REG_CC);
 104         return fun_wait_ready(fdev, !!(fdev->cc_reg & NVME_CC_ENABLE));
 105 }
 106
 107 static int fun_disable_ctrl(struct fun_dev *fdev)
 108 {
 109         fdev->cc_reg &= ~(NVME_CC_SHN_MASK | NVME_CC_ENABLE);
 110         return fun_update_cc_enable(fdev, 1);
 111 }
 112
 113 static int fun_enable_ctrl(struct fun_dev *fdev, u32 admin_cqesz_log2,
 114                            u32 admin_sqesz_log2)
 115 {
 116         fdev->cc_reg = (admin_cqesz_log2 << NVME_CC_IOCQES_SHIFT) |
 117                        (admin_sqesz_log2 << NVME_CC_IOSQES_SHIFT) |
 118                        ((PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT) |
 119                        NVME_CC_ENABLE;
 120
 121         return fun_update_cc_enable(fdev, 0);
 122 }
 123
 124 static int fun_map_bars(struct fun_dev *fdev, const char *name)
 125 {
 126         struct pci_dev *pdev = to_pci_dev(fdev->dev);
 127         int err;
 128
 129         err = pci_request_mem_regions(pdev, name);
 130         if (err) {
 131                 dev_err(&pdev->dev,
 132                         "Couldn't get PCI memory resources, err %d\n", err);
 133                 return err;
 134         }
 135
 136         fdev->bar = pci_ioremap_bar(pdev, 0);
 137         if (!fdev->bar) {
 138                 dev_err(&pdev->dev, "Couldn't map BAR 0\n");
 139                 pci_release_mem_regions(pdev);
 140                 return -ENOMEM;
 141         }
 142
 143         return 0;
 144 }
 145
 146 static void fun_unmap_bars(struct fun_dev *fdev)
 147 {
 148         struct pci_dev *pdev = to_pci_dev(fdev->dev);
 149
 150         if (fdev->bar) {
 151                 iounmap(fdev->bar);
 152                 fdev->bar = NULL;
 153                 pci_release_mem_regions(pdev);
 154         }
 155 }
 156
 157 static int fun_set_dma_masks(struct device *dev)
 158 {
 159         int err;
 160
 161         err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
 162         if (err)
 163                 dev_err(dev, "DMA mask configuration failed, err %d\n", err);
 164         return err;
 165 }
 166
 167 static irqreturn_t fun_admin_irq(int irq, void *data)
 168 {
 169         struct fun_queue *funq = data;
 170
 171         return fun_process_cq(funq, 0) ? IRQ_HANDLED : IRQ_NONE;
 172 }
 173
 174 static void fun_complete_admin_cmd(struct fun_queue *funq, void *data,
 175                                    void *entry, const struct fun_cqe_info *info)
 176 {
 177         const struct fun_admin_rsp_common *rsp_common = entry;
 178         struct fun_dev *fdev = funq->fdev;
 179         struct fun_cmd_ctx *cmd_ctx;
 180         int cpu;
 181         u16 cid;
 182
 183         if (info->sqhd == cpu_to_be16(0xffff)) {
 184                 dev_dbg(fdev->dev, "adminq event");
 185                 if (fdev->adminq_cb)
 186                         fdev->adminq_cb(fdev, entry);
 187                 return;
 188         }
 189
 190         cid = be16_to_cpu(rsp_common->cid);
 191         dev_dbg(fdev->dev, "admin CQE cid %u, op %u, ret %u\n", cid,
 192                 rsp_common->op, rsp_common->ret);
 193
 194         cmd_ctx = &fdev->cmd_ctx[cid];
 195         if (cmd_ctx->cpu < 0) {
 196                 dev_err(fdev->dev,
 197                         "admin CQE with CID=%u, op=%u does not match a pending command\n",
 198                         cid, rsp_common->op);
 199                 return;
 200         }
 201
 202         if (cmd_ctx->cb)
 203                 cmd_ctx->cb(fdev, entry, xchg(&cmd_ctx->cb_data, NULL));
 204
 205         cpu = cmd_ctx->cpu;
 206         cmd_ctx->cpu = -1;
 207         sbitmap_queue_clear(&fdev->admin_sbq, cid, cpu);
 208 }
 209
 210 static int fun_init_cmd_ctx(struct fun_dev *fdev, unsigned int ntags)
 211 {
 212         unsigned int i;
 213
 214         fdev->cmd_ctx = kvcalloc(ntags, sizeof(*fdev->cmd_ctx), GFP_KERNEL);
 215         if (!fdev->cmd_ctx)
 216                 return -ENOMEM;
 217
 218         for (i = 0; i < ntags; i++)
 219                 fdev->cmd_ctx[i].cpu = -1;
 220
 221         return 0;
 222 }
 223
 224 /* Allocate and enable an admin queue and assign it the first IRQ vector. */
 225 static int fun_enable_admin_queue(struct fun_dev *fdev,
 226                                   const struct fun_dev_params *areq)
 227 {
 228         struct fun_queue_alloc_req qreq = {
 229                 .cqe_size_log2 = areq->cqe_size_log2,
 230                 .sqe_size_log2 = areq->sqe_size_log2,
 231                 .cq_depth = areq->cq_depth,
 232                 .sq_depth = areq->sq_depth,
 233                 .rq_depth = areq->rq_depth,
 234         };
 235         unsigned int ntags = areq->sq_depth - 1;
 236         struct fun_queue *funq;
 237         int rc;
 238
 239         if (fdev->admin_q)
 240                 return -EEXIST;
 241
 242         if (areq->sq_depth < AQA_MIN_QUEUE_SIZE ||
 243             areq->sq_depth > AQA_MAX_QUEUE_SIZE ||
 244             areq->cq_depth < AQA_MIN_QUEUE_SIZE ||
 245             areq->cq_depth > AQA_MAX_QUEUE_SIZE)
 246                 return -EINVAL;
 247
 248         fdev->admin_q = fun_alloc_queue(fdev, 0, &qreq);
 249         if (!fdev->admin_q)
 250                 return -ENOMEM;
 251
 252         rc = fun_init_cmd_ctx(fdev, ntags);
 253         if (rc)
 254                 goto free_q;
 255
 256         rc = sbitmap_queue_init_node(&fdev->admin_sbq, ntags, -1, false,
 257                                      GFP_KERNEL, dev_to_node(fdev->dev));
 258         if (rc)
 259                 goto free_cmd_ctx;
 260
 261         funq = fdev->admin_q;
 262         funq->cq_vector = 0;
 263         rc = fun_request_irq(funq, dev_name(fdev->dev), fun_admin_irq, funq);
 264         if (rc)
 265                 goto free_sbq;
 266
 267         fun_set_cq_callback(funq, fun_complete_admin_cmd, NULL);
 268         fdev->adminq_cb = areq->event_cb;
 269
 270         writel((funq->sq_depth - 1) << AQA_ASQS_SHIFT |
 271                (funq->cq_depth - 1) << AQA_ACQS_SHIFT,
 272                fdev->bar + NVME_REG_AQA);
 273
 274         writeq(funq->sq_dma_addr, fdev->bar + NVME_REG_ASQ);
 275         writeq(funq->cq_dma_addr, fdev->bar + NVME_REG_ACQ);
 276
 277         rc = fun_enable_ctrl(fdev, areq->cqe_size_log2, areq->sqe_size_log2);
 278         if (rc)
 279                 goto free_irq;
 280
 281         if (areq->rq_depth) {
 282                 rc = fun_create_rq(funq);
 283                 if (rc)
 284                         goto disable_ctrl;
 285
 286                 funq_rq_post(funq);
 287         }
 288
 289         return 0;
 290
 291 disable_ctrl:
 292         fun_disable_ctrl(fdev);
 293 free_irq:
 294         fun_free_irq(funq);
 295 free_sbq:
 296         sbitmap_queue_free(&fdev->admin_sbq);
 297 free_cmd_ctx:
 298         kvfree(fdev->cmd_ctx);
 299         fdev->cmd_ctx = NULL;
 300 free_q:
 301         fun_free_queue(fdev->admin_q);
 302         fdev->admin_q = NULL;
 303         return rc;
 304 }
 305
 306 static void fun_disable_admin_queue(struct fun_dev *fdev)
 307 {
 308         struct fun_queue *admq = fdev->admin_q;
 309
 310         if (!admq)
 311                 return;
 312
 313         fun_disable_ctrl(fdev);
 314
 315         fun_free_irq(admq);
 316         __fun_process_cq(admq, 0);
 317
 318         sbitmap_queue_free(&fdev->admin_sbq);
 319
 320         kvfree(fdev->cmd_ctx);
 321         fdev->cmd_ctx = NULL;
 322
 323         fun_free_queue(admq);
 324         fdev->admin_q = NULL;
 325 }
 326
 327 /* Return %true if the admin queue has stopped servicing commands as can be
 328  * detected through registers. This isn't exhaustive and may provide false
 329  * negatives.
 330  */
 331 static bool fun_adminq_stopped(struct fun_dev *fdev)
 332 {
 333         u32 csts = readl(fdev->bar + NVME_REG_CSTS);
 334
 335         return (csts & (NVME_CSTS_CFS | NVME_CSTS_RDY)) != NVME_CSTS_RDY;
 336 }
 337
 338 static int fun_wait_for_tag(struct fun_dev *fdev, int *cpup)
 339 {
 340         struct sbitmap_queue *sbq = &fdev->admin_sbq;
 341         struct sbq_wait_state *ws = &sbq->ws[0];
 342         DEFINE_SBQ_WAIT(wait);
 343         int tag;
 344
 345         for (;;) {
 346                 sbitmap_prepare_to_wait(sbq, ws, &wait, TASK_UNINTERRUPTIBLE);
 347                 if (fdev->suppress_cmds) {
 348                         tag = -ESHUTDOWN;
 349                         break;
 350                 }
 351                 tag = sbitmap_queue_get(sbq, cpup);
 352                 if (tag >= 0)
 353                         break;
 354                 schedule();
 355         }
 356
 357         sbitmap_finish_wait(sbq, ws, &wait);
 358         return tag;
 359 }
 360
 361 /* Submit an asynchronous admin command. Caller is responsible for implementing
 362  * any waiting or timeout. Upon command completion the callback @cb is called.
 363  */
 364 int fun_submit_admin_cmd(struct fun_dev *fdev, struct fun_admin_req_common *cmd,
 365                          fun_admin_callback_t cb, void *cb_data, bool wait_ok)
 366 {
 367         struct fun_queue *funq = fdev->admin_q;
 368         unsigned int cmdsize = cmd->len8 * 8;
 369         struct fun_cmd_ctx *cmd_ctx;
 370         int tag, cpu, rc = 0;
 371
 372         if (WARN_ON(cmdsize > (1 << funq->sqe_size_log2)))
 373                 return -EMSGSIZE;
 374
 375         tag = sbitmap_queue_get(&fdev->admin_sbq, &cpu);
 376         if (tag < 0) {
 377                 if (!wait_ok)
 378                         return -EAGAIN;
 379                 tag = fun_wait_for_tag(fdev, &cpu);
 380                 if (tag < 0)
 381                         return tag;
 382         }
 383
 384         cmd->cid = cpu_to_be16(tag);
 385
 386         cmd_ctx = &fdev->cmd_ctx[tag];
 387         cmd_ctx->cb = cb;
 388         cmd_ctx->cb_data = cb_data;
 389
 390         spin_lock(&funq->sq_lock);
 391
 392         if (unlikely(fdev->suppress_cmds)) {
 393                 rc = -ESHUTDOWN;
 394                 sbitmap_queue_clear(&fdev->admin_sbq, tag, cpu);
 395         } else {
 396                 cmd_ctx->cpu = cpu;
 397                 memcpy(fun_sqe_at(funq, funq->sq_tail), cmd, cmdsize);
 398
 399                 dev_dbg(fdev->dev, "admin cmd @ %u: %8ph\n", funq->sq_tail,
 400                         cmd);
 401
 402                 if (++funq->sq_tail == funq->sq_depth)
 403                         funq->sq_tail = 0;
 404                 writel(funq->sq_tail, funq->sq_db);
 405         }
 406         spin_unlock(&funq->sq_lock);
 407         return rc;
 408 }
 409
 410 /* Abandon a pending admin command by clearing the issuer's callback data.
 411  * Failure indicates that the command either has already completed or its
 412  * completion is racing with this call.
 413  */
 414 static bool fun_abandon_admin_cmd(struct fun_dev *fd,
 415                                   const struct fun_admin_req_common *cmd,
 416                                   void *cb_data)
 417 {
 418         u16 cid = be16_to_cpu(cmd->cid);
 419         struct fun_cmd_ctx *cmd_ctx = &fd->cmd_ctx[cid];
 420
 421         return cmpxchg(&cmd_ctx->cb_data, cb_data, NULL) == cb_data;
 422 }
 423
 424 /* Stop submission of new admin commands and wake up any processes waiting for
 425  * tags. Already submitted commands are left to complete or time out.
 426  */
 427 static void fun_admin_stop(struct fun_dev *fdev)
 428 {
 429         spin_lock(&fdev->admin_q->sq_lock);
 430         fdev->suppress_cmds = true;
 431         spin_unlock(&fdev->admin_q->sq_lock);
 432         sbitmap_queue_wake_all(&fdev->admin_sbq);
 433 }
 434
 435 /* The callback for synchronous execution of admin commands. It copies the
 436  * command response to the caller's buffer and signals completion.
 437  */
 438 static void fun_admin_cmd_sync_cb(struct fun_dev *fd, void *rsp, void *cb_data)
 439 {
 440         const struct fun_admin_rsp_common *rsp_common = rsp;
 441         struct fun_sync_cmd_ctx *ctx = cb_data;
 442
 443         if (!ctx)
 444                 return;         /* command issuer timed out and left */
 445         if (ctx->rsp_buf) {
 446                 unsigned int rsp_len = rsp_common->len8 * 8;
 447
 448                 if (unlikely(rsp_len > ctx->rsp_len)) {
 449                         dev_err(fd->dev,
 450                                 "response for op %u is %uB > response buffer %uB\n",
 451                                 rsp_common->op, rsp_len, ctx->rsp_len);
 452                         rsp_len = ctx->rsp_len;
 453                 }
 454                 memcpy(ctx->rsp_buf, rsp, rsp_len);
 455         }
 456         ctx->rsp_status = rsp_common->ret;
 457         complete(&ctx->compl);
 458 }
 459
 460 /* Submit a synchronous admin command. */
 461 int fun_submit_admin_sync_cmd(struct fun_dev *fdev,
 462                               struct fun_admin_req_common *cmd, void *rsp,
 463                               size_t rspsize, unsigned int timeout)
 464 {
 465         struct fun_sync_cmd_ctx ctx = {
 466                 .compl = COMPLETION_INITIALIZER_ONSTACK(ctx.compl),
 467                 .rsp_buf = rsp,
 468                 .rsp_len = rspsize,
 469         };
 470         unsigned int cmdlen = cmd->len8 * 8;
 471         unsigned long jiffies_left;
 472         int ret;
 473
 474         ret = fun_submit_admin_cmd(fdev, cmd, fun_admin_cmd_sync_cb, &ctx,
 475                                    true);
 476         if (ret)
 477                 return ret;
 478
 479         if (!timeout)
 480                 timeout = FUN_ADMIN_CMD_TO_MS;
 481
 482         jiffies_left = wait_for_completion_timeout(&ctx.compl,
 483                                                    msecs_to_jiffies(timeout));
 484         if (!jiffies_left) {
 485                 /* The command timed out. Attempt to cancel it so we can return.
 486                  * But if the command is in the process of completing we'll
 487                  * wait for it.
 488                  */
 489                 if (fun_abandon_admin_cmd(fdev, cmd, &ctx)) {
 490                         dev_err(fdev->dev, "admin command timed out: %*ph\n",
 491                                 cmdlen, cmd);
 492                         fun_admin_stop(fdev);
 493                         /* see if the timeout was due to a queue failure */
 494                         if (fun_adminq_stopped(fdev))
 495                                 dev_err(fdev->dev,
 496                                         "device does not accept admin commands\n");
 497
 498                         return -ETIMEDOUT;
 499                 }
 500                 wait_for_completion(&ctx.compl);
 501         }
 502
 503         if (ctx.rsp_status) {
 504                 dev_err(fdev->dev, "admin command failed, err %d: %*ph\n",
 505                         ctx.rsp_status, cmdlen, cmd);
 506         }
 507
 508         return -ctx.rsp_status;
 509 }
 510 EXPORT_SYMBOL_GPL(fun_submit_admin_sync_cmd);
 511
 512 /* Return the number of device resources of the requested type. */
 513 int fun_get_res_count(struct fun_dev *fdev, enum fun_admin_op res)
 514 {
 515         union {
 516                 struct fun_admin_res_count_req req;
 517                 struct fun_admin_res_count_rsp rsp;
 518         } cmd;
 519         int rc;
 520
 521         cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(cmd.req));
 522         cmd.req.count = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_RES_COUNT,
 523                                                     0, 0);
 524
 525         rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common, &cmd.rsp,
 526                                        sizeof(cmd), 0);
 527         return rc ? rc : be32_to_cpu(cmd.rsp.count.data);
 528 }
 529 EXPORT_SYMBOL_GPL(fun_get_res_count);
 530
 531 /* Request that the instance of resource @res with the given id be deleted. */
 532 int fun_res_destroy(struct fun_dev *fdev, enum fun_admin_op res,
 533                     unsigned int flags, u32 id)
 534 {
 535         struct fun_admin_generic_destroy_req req = {
 536                 .common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(req)),
 537                 .destroy = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_DESTROY,
 538                                                        flags, id)
 539         };
 540
 541         return fun_submit_admin_sync_cmd(fdev, &req.common, NULL, 0, 0);
 542 }
 543 EXPORT_SYMBOL_GPL(fun_res_destroy);
 544
 545 /* Bind two entities of the given types and IDs. */
 546 int fun_bind(struct fun_dev *fdev, enum fun_admin_bind_type type0,
 547              unsigned int id0, enum fun_admin_bind_type type1,
 548              unsigned int id1)
 549 {
 550         struct {
 551                 struct fun_admin_bind_req req;
 552                 struct fun_admin_bind_entry entry[2];
 553         } cmd = {
 554                 .req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_BIND,
 555                                                          sizeof(cmd)),
 556                 .entry[0] = FUN_ADMIN_BIND_ENTRY_INIT(type0, id0),
 557                 .entry[1] = FUN_ADMIN_BIND_ENTRY_INIT(type1, id1),
 558         };
 559
 560         return fun_submit_admin_sync_cmd(fdev, &cmd.req.common, NULL, 0, 0);
 561 }
 562 EXPORT_SYMBOL_GPL(fun_bind);
 563
 564 static int fun_get_dev_limits(struct fun_dev *fdev)
 565 {
 566         struct pci_dev *pdev = to_pci_dev(fdev->dev);
 567         unsigned int cq_count, sq_count, num_dbs;
 568         int rc;
 569
 570         rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPCQ);
 571         if (rc < 0)
 572                 return rc;
 573         cq_count = rc;
 574
 575         rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPSQ);
 576         if (rc < 0)
 577                 return rc;
 578         sq_count = rc;
 579
 580         /* The admin queue consumes 1 CQ and at least 1 SQ. To be usable the
 581          * device must provide additional queues.
 582          */
 583         if (cq_count < 2 || sq_count < 2 + !!fdev->admin_q->rq_depth)
 584                 return -EINVAL;
 585
 586         /* Calculate the max QID based on SQ/CQ/doorbell counts.
 587          * SQ/CQ doorbells alternate.
 588          */
 589         num_dbs = (pci_resource_len(pdev, 0) - NVME_REG_DBS) /
 590                   (fdev->db_stride * 4);
 591         fdev->max_qid = min3(cq_count, sq_count, num_dbs / 2) - 1;
 592         fdev->kern_end_qid = fdev->max_qid + 1;
 593         return 0;
 594 }
 595
 596 /* Allocate all MSI-X vectors available on a function and at least @min_vecs. */
 597 static int fun_alloc_irqs(struct pci_dev *pdev, unsigned int min_vecs)
 598 {
 599         int vecs, num_msix = pci_msix_vec_count(pdev);
 600
 601         if (num_msix < 0)
 602                 return num_msix;
 603         if (min_vecs > num_msix)
 604                 return -ERANGE;
 605
 606         vecs = pci_alloc_irq_vectors(pdev, min_vecs, num_msix, PCI_IRQ_MSIX);
 607         if (vecs > 0) {
 608                 dev_info(&pdev->dev,
 609                          "Allocated %d IRQ vectors of %d requested\n",
 610                          vecs, num_msix);
 611         } else {
 612                 dev_err(&pdev->dev,
 613                         "Unable to allocate at least %u IRQ vectors\n",
 614                         min_vecs);
 615         }
 616         return vecs;
 617 }
 618
 619 /* Allocate and initialize the IRQ manager state. */
 620 static int fun_alloc_irq_mgr(struct fun_dev *fdev)
 621 {
 622         fdev->irq_map = bitmap_zalloc(fdev->num_irqs, GFP_KERNEL);
 623         if (!fdev->irq_map)
 624                 return -ENOMEM;
 625
 626         spin_lock_init(&fdev->irqmgr_lock);
 627         /* mark IRQ 0 allocated, it is used by the admin queue */
 628         __set_bit(0, fdev->irq_map);
 629         fdev->irqs_avail = fdev->num_irqs - 1;
 630         return 0;
 631 }
 632
 633 /* Reserve @nirqs of the currently available IRQs and return their indices. */
 634 int fun_reserve_irqs(struct fun_dev *fdev, unsigned int nirqs, u16 *irq_indices)
 635 {
 636         unsigned int b, n = 0;
 637         int err = -ENOSPC;
 638
 639         if (!nirqs)
 640                 return 0;
 641
 642         spin_lock(&fdev->irqmgr_lock);
 643         if (nirqs > fdev->irqs_avail)
 644                 goto unlock;
 645
 646         for_each_clear_bit(b, fdev->irq_map, fdev->num_irqs) {
 647                 __set_bit(b, fdev->irq_map);
 648                 irq_indices[n++] = b;
 649                 if (n >= nirqs)
 650                         break;
 651         }
 652
 653         WARN_ON(n < nirqs);
 654         fdev->irqs_avail -= n;
 655         err = n;
 656 unlock:
 657         spin_unlock(&fdev->irqmgr_lock);
 658         return err;
 659 }
 660 EXPORT_SYMBOL(fun_reserve_irqs);
 661
 662 /* Release @nirqs previously allocated IRQS with the supplied indices. */
 663 void fun_release_irqs(struct fun_dev *fdev, unsigned int nirqs,
 664                       u16 *irq_indices)
 665 {
 666         unsigned int i;
 667
 668         spin_lock(&fdev->irqmgr_lock);
 669         for (i = 0; i < nirqs; i++)
 670                 __clear_bit(irq_indices[i], fdev->irq_map);
 671         fdev->irqs_avail += nirqs;
 672         spin_unlock(&fdev->irqmgr_lock);
 673 }
 674 EXPORT_SYMBOL(fun_release_irqs);
 675
 676 static void fun_serv_handler(struct work_struct *work)
 677 {
 678         struct fun_dev *fd = container_of(work, struct fun_dev, service_task);
 679
 680         if (test_bit(FUN_SERV_DISABLED, &fd->service_flags))
 681                 return;
 682         if (fd->serv_cb)
 683                 fd->serv_cb(fd);
 684 }
 685
 686 void fun_serv_stop(struct fun_dev *fd)
 687 {
 688         set_bit(FUN_SERV_DISABLED, &fd->service_flags);
 689         cancel_work_sync(&fd->service_task);
 690 }
 691 EXPORT_SYMBOL_GPL(fun_serv_stop);
 692
 693 void fun_serv_restart(struct fun_dev *fd)
 694 {
 695         clear_bit(FUN_SERV_DISABLED, &fd->service_flags);
 696         if (fd->service_flags)
 697                 schedule_work(&fd->service_task);
 698 }
 699 EXPORT_SYMBOL_GPL(fun_serv_restart);
 700
 701 void fun_serv_sched(struct fun_dev *fd)
 702 {
 703         if (!test_bit(FUN_SERV_DISABLED, &fd->service_flags))
 704                 schedule_work(&fd->service_task);
 705 }
 706 EXPORT_SYMBOL_GPL(fun_serv_sched);
 707
 708 /* Check and try to get the device into a proper state for initialization,
 709  * i.e., CSTS.RDY = CC.EN = 0.
 710  */
 711 static int sanitize_dev(struct fun_dev *fdev)
 712 {
 713         int rc;
 714
 715         fdev->cap_reg = readq(fdev->bar + NVME_REG_CAP);
 716         fdev->cc_reg = readl(fdev->bar + NVME_REG_CC);
 717
 718         /* First get RDY to agree with the current EN. Give RDY the opportunity
 719          * to complete a potential recent EN change.
 720          */
 721         rc = fun_wait_ready(fdev, fdev->cc_reg & NVME_CC_ENABLE);
 722         if (rc)
 723                 return rc;
 724
 725         /* Next, reset the device if EN is currently 1. */
 726         if (fdev->cc_reg & NVME_CC_ENABLE)
 727                 rc = fun_disable_ctrl(fdev);
 728
 729         return rc;
 730 }
 731
 732 /* Undo the device initialization of fun_dev_enable(). */
 733 void fun_dev_disable(struct fun_dev *fdev)
 734 {
 735         struct pci_dev *pdev = to_pci_dev(fdev->dev);
 736
 737         pci_set_drvdata(pdev, NULL);
 738
 739         if (fdev->fw_handle != FUN_HCI_ID_INVALID) {
 740                 fun_res_destroy(fdev, FUN_ADMIN_OP_SWUPGRADE, 0,
 741                                 fdev->fw_handle);
 742                 fdev->fw_handle = FUN_HCI_ID_INVALID;
 743         }
 744
 745         fun_disable_admin_queue(fdev);
 746
 747         bitmap_free(fdev->irq_map);
 748         pci_free_irq_vectors(pdev);
 749
 750         pci_clear_master(pdev);
 751         pci_disable_pcie_error_reporting(pdev);
 752         pci_disable_device(pdev);
 753
 754         fun_unmap_bars(fdev);
 755 }
 756 EXPORT_SYMBOL(fun_dev_disable);
 757
 758 /* Perform basic initialization of a device, including
 759  * - PCI config space setup and BAR0 mapping
 760  * - interrupt management initialization
 761  * - 1 admin queue setup
 762  * - determination of some device limits, such as number of queues.
 763  */
 764 int fun_dev_enable(struct fun_dev *fdev, struct pci_dev *pdev,
 765                    const struct fun_dev_params *areq, const char *name)
 766 {
 767         int rc;
 768
 769         fdev->dev = &pdev->dev;
 770         rc = fun_map_bars(fdev, name);
 771         if (rc)
 772                 return rc;
 773
 774         rc = fun_set_dma_masks(fdev->dev);
 775         if (rc)
 776                 goto unmap;
 777
 778         rc = pci_enable_device_mem(pdev);
 779         if (rc) {
 780                 dev_err(&pdev->dev, "Couldn't enable device, err %d\n", rc);
 781                 goto unmap;
 782         }
 783
 784         pci_enable_pcie_error_reporting(pdev);
 785
 786         rc = sanitize_dev(fdev);
 787         if (rc)
 788                 goto disable_dev;
 789
 790         fdev->fw_handle = FUN_HCI_ID_INVALID;
 791         fdev->q_depth = NVME_CAP_MQES(fdev->cap_reg) + 1;
 792         fdev->db_stride = 1 << NVME_CAP_STRIDE(fdev->cap_reg);
 793         fdev->dbs = fdev->bar + NVME_REG_DBS;
 794
 795         INIT_WORK(&fdev->service_task, fun_serv_handler);
 796         fdev->service_flags = FUN_SERV_DISABLED;
 797         fdev->serv_cb = areq->serv_cb;
 798
 799         rc = fun_alloc_irqs(pdev, areq->min_msix + 1); /* +1 for admin CQ */
 800         if (rc < 0)
 801                 goto disable_dev;
 802         fdev->num_irqs = rc;
 803
 804         rc = fun_alloc_irq_mgr(fdev);
 805         if (rc)
 806                 goto free_irqs;
 807
 808         pci_set_master(pdev);
 809         rc = fun_enable_admin_queue(fdev, areq);
 810         if (rc)
 811                 goto free_irq_mgr;
 812
 813         rc = fun_get_dev_limits(fdev);
 814         if (rc < 0)
 815                 goto disable_admin;
 816
 817         pci_save_state(pdev);
 818         pci_set_drvdata(pdev, fdev);
 819         pcie_print_link_status(pdev);
 820         dev_dbg(fdev->dev, "q_depth %u, db_stride %u, max qid %d kern_end_qid %d\n",
 821                 fdev->q_depth, fdev->db_stride, fdev->max_qid,
 822                 fdev->kern_end_qid);
 823         return 0;
 824
 825 disable_admin:
 826         fun_disable_admin_queue(fdev);
 827 free_irq_mgr:
 828         pci_clear_master(pdev);
 829         bitmap_free(fdev->irq_map);
 830 free_irqs:
 831         pci_free_irq_vectors(pdev);
 832 disable_dev:
 833         pci_disable_pcie_error_reporting(pdev);
 834         pci_disable_device(pdev);
 835 unmap:
 836         fun_unmap_bars(fdev);
 837         return rc;
 838 }
 839 EXPORT_SYMBOL(fun_dev_enable);
 840
 841 MODULE_AUTHOR("Dimitris Michailidis <dmichail@fungible.com>");
 842 MODULE_DESCRIPTION("Core services driver for Fungible devices");
 843 MODULE_LICENSE("Dual BSD/GPL");