drivers/nvme/host/fc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2016 Avago Technologies.  All rights reserved.
   4  */
   5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   6 #include <linux/module.h>
   7 #include <linux/parser.h>
   8 #include <uapi/scsi/fc/fc_fs.h>
   9 #include <uapi/scsi/fc/fc_els.h>
  10 #include <linux/delay.h>
  11 #include <linux/overflow.h>
  12 #include <linux/blk-cgroup.h>
  13 #include "nvme.h"
  14 #include "fabrics.h"
  15 #include <linux/nvme-fc-driver.h>
  16 #include <linux/nvme-fc.h>
  17 #include "fc.h"
  18 #include <scsi/scsi_transport_fc.h>
  19 #include <linux/blk-mq-pci.h>
  20
  21 /* *************************** Data Structures/Defines ****************** */
  22
  23
  24 enum nvme_fc_queue_flags {
  25         NVME_FC_Q_CONNECTED = 0,
  26         NVME_FC_Q_LIVE,
  27 };
  28
  29 #define NVME_FC_DEFAULT_DEV_LOSS_TMO    60      /* seconds */
  30 #define NVME_FC_DEFAULT_RECONNECT_TMO   2       /* delay between reconnects
  31                                                  * when connected and a
  32                                                  * connection failure.
  33                                                  */
  34
  35 struct nvme_fc_queue {
  36         struct nvme_fc_ctrl     *ctrl;
  37         struct device           *dev;
  38         struct blk_mq_hw_ctx    *hctx;
  39         void                    *lldd_handle;
  40         size_t                  cmnd_capsule_len;
  41         u32                     qnum;
  42         u32                     rqcnt;
  43         u32                     seqno;
  44
  45         u64                     connection_id;
  46         atomic_t                csn;
  47
  48         unsigned long           flags;
  49 } __aligned(sizeof(u64));       /* alignment for other things alloc'd with */
  50
  51 enum nvme_fcop_flags {
  52         FCOP_FLAGS_TERMIO       = (1 << 0),
  53         FCOP_FLAGS_AEN          = (1 << 1),
  54 };
  55
  56 struct nvmefc_ls_req_op {
  57         struct nvmefc_ls_req    ls_req;
  58
  59         struct nvme_fc_rport    *rport;
  60         struct nvme_fc_queue    *queue;
  61         struct request          *rq;
  62         u32                     flags;
  63
  64         int                     ls_error;
  65         struct completion       ls_done;
  66         struct list_head        lsreq_list;     /* rport->ls_req_list */
  67         bool                    req_queued;
  68 };
  69
  70 struct nvmefc_ls_rcv_op {
  71         struct nvme_fc_rport            *rport;
  72         struct nvmefc_ls_rsp            *lsrsp;
  73         union nvmefc_ls_requests        *rqstbuf;
  74         union nvmefc_ls_responses       *rspbuf;
  75         u16                             rqstdatalen;
  76         bool                            handled;
  77         dma_addr_t                      rspdma;
  78         struct list_head                lsrcv_list;     /* rport->ls_rcv_list */
  79 } __aligned(sizeof(u64));       /* alignment for other things alloc'd with */
  80
  81 enum nvme_fcpop_state {
  82         FCPOP_STATE_UNINIT      = 0,
  83         FCPOP_STATE_IDLE        = 1,
  84         FCPOP_STATE_ACTIVE      = 2,
  85         FCPOP_STATE_ABORTED     = 3,
  86         FCPOP_STATE_COMPLETE    = 4,
  87 };
  88
  89 struct nvme_fc_fcp_op {
  90         struct nvme_request     nreq;           /*
  91                                                  * nvme/host/core.c
  92                                                  * requires this to be
  93                                                  * the 1st element in the
  94                                                  * private structure
  95                                                  * associated with the
  96                                                  * request.
  97                                                  */
  98         struct nvmefc_fcp_req   fcp_req;
  99
 100         struct nvme_fc_ctrl     *ctrl;
 101         struct nvme_fc_queue    *queue;
 102         struct request          *rq;
 103
 104         atomic_t                state;
 105         u32                     flags;
 106         u32                     rqno;
 107         u32                     nents;
 108
 109         struct nvme_fc_cmd_iu   cmd_iu;
 110         struct nvme_fc_ersp_iu  rsp_iu;
 111 };
 112
 113 struct nvme_fcp_op_w_sgl {
 114         struct nvme_fc_fcp_op   op;
 115         struct scatterlist      sgl[NVME_INLINE_SG_CNT];
 116         uint8_t                 priv[];
 117 };
 118
 119 struct nvme_fc_lport {
 120         struct nvme_fc_local_port       localport;
 121
 122         struct ida                      endp_cnt;
 123         struct list_head                port_list;      /* nvme_fc_port_list */
 124         struct list_head                endp_list;
 125         struct device                   *dev;   /* physical device for dma */
 126         struct nvme_fc_port_template    *ops;
 127         struct kref                     ref;
 128         atomic_t                        act_rport_cnt;
 129 } __aligned(sizeof(u64));       /* alignment for other things alloc'd with */
 130
 131 struct nvme_fc_rport {
 132         struct nvme_fc_remote_port      remoteport;
 133
 134         struct list_head                endp_list; /* for lport->endp_list */
 135         struct list_head                ctrl_list;
 136         struct list_head                ls_req_list;
 137         struct list_head                ls_rcv_list;
 138         struct list_head                disc_list;
 139         struct device                   *dev;   /* physical device for dma */
 140         struct nvme_fc_lport            *lport;
 141         spinlock_t                      lock;
 142         struct kref                     ref;
 143         atomic_t                        act_ctrl_cnt;
 144         unsigned long                   dev_loss_end;
 145         struct work_struct              lsrcv_work;
 146 } __aligned(sizeof(u64));       /* alignment for other things alloc'd with */
 147
 148 /* fc_ctrl flags values - specified as bit positions */
 149 #define ASSOC_ACTIVE            0
 150 #define ASSOC_FAILED            1
 151 #define FCCTRL_TERMIO           2
 152
 153 struct nvme_fc_ctrl {
 154         spinlock_t              lock;
 155         struct nvme_fc_queue    *queues;
 156         struct device           *dev;
 157         struct nvme_fc_lport    *lport;
 158         struct nvme_fc_rport    *rport;
 159         u32                     cnum;
 160
 161         bool                    ioq_live;
 162         u64                     association_id;
 163         struct nvmefc_ls_rcv_op *rcv_disconn;
 164
 165         struct list_head        ctrl_list;      /* rport->ctrl_list */
 166
 167         struct blk_mq_tag_set   admin_tag_set;
 168         struct blk_mq_tag_set   tag_set;
 169
 170         struct work_struct      ioerr_work;
 171         struct delayed_work     connect_work;
 172
 173         struct kref             ref;
 174         unsigned long           flags;
 175         u32                     iocnt;
 176         wait_queue_head_t       ioabort_wait;
 177
 178         struct nvme_fc_fcp_op   aen_ops[NVME_NR_AEN_COMMANDS];
 179
 180         struct nvme_ctrl        ctrl;
 181 };
 182
 183 static inline struct nvme_fc_ctrl *
 184 to_fc_ctrl(struct nvme_ctrl *ctrl)
 185 {
 186         return container_of(ctrl, struct nvme_fc_ctrl, ctrl);
 187 }
 188
 189 static inline struct nvme_fc_lport *
 190 localport_to_lport(struct nvme_fc_local_port *portptr)
 191 {
 192         return container_of(portptr, struct nvme_fc_lport, localport);
 193 }
 194
 195 static inline struct nvme_fc_rport *
 196 remoteport_to_rport(struct nvme_fc_remote_port *portptr)
 197 {
 198         return container_of(portptr, struct nvme_fc_rport, remoteport);
 199 }
 200
 201 static inline struct nvmefc_ls_req_op *
 202 ls_req_to_lsop(struct nvmefc_ls_req *lsreq)
 203 {
 204         return container_of(lsreq, struct nvmefc_ls_req_op, ls_req);
 205 }
 206
 207 static inline struct nvme_fc_fcp_op *
 208 fcp_req_to_fcp_op(struct nvmefc_fcp_req *fcpreq)
 209 {
 210         return container_of(fcpreq, struct nvme_fc_fcp_op, fcp_req);
 211 }
 212
 213
 214
 215 /* *************************** Globals **************************** */
 216
 217
 218 static DEFINE_SPINLOCK(nvme_fc_lock);
 219
 220 static LIST_HEAD(nvme_fc_lport_list);
 221 static DEFINE_IDA(nvme_fc_local_port_cnt);
 222 static DEFINE_IDA(nvme_fc_ctrl_cnt);
 223
 224 static struct workqueue_struct *nvme_fc_wq;
 225
 226 static bool nvme_fc_waiting_to_unload;
 227 static DECLARE_COMPLETION(nvme_fc_unload_proceed);
 228
 229 /*
 230  * These items are short-term. They will eventually be moved into
 231  * a generic FC class. See comments in module init.
 232  */
 233 static struct device *fc_udev_device;
 234
 235 static void nvme_fc_complete_rq(struct request *rq);
 236
 237 /* *********************** FC-NVME Port Management ************************ */
 238
 239 static void __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *,
 240                         struct nvme_fc_queue *, unsigned int);
 241
 242 static void nvme_fc_handle_ls_rqst_work(struct work_struct *work);
 243
 244
 245 static void
 246 nvme_fc_free_lport(struct kref *ref)
 247 {
 248         struct nvme_fc_lport *lport =
 249                 container_of(ref, struct nvme_fc_lport, ref);
 250         unsigned long flags;
 251
 252         WARN_ON(lport->localport.port_state != FC_OBJSTATE_DELETED);
 253         WARN_ON(!list_empty(&lport->endp_list));
 254
 255         /* remove from transport list */
 256         spin_lock_irqsave(&nvme_fc_lock, flags);
 257         list_del(&lport->port_list);
 258         if (nvme_fc_waiting_to_unload && list_empty(&nvme_fc_lport_list))
 259                 complete(&nvme_fc_unload_proceed);
 260         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 261
 262         ida_free(&nvme_fc_local_port_cnt, lport->localport.port_num);
 263         ida_destroy(&lport->endp_cnt);
 264
 265         put_device(lport->dev);
 266
 267         kfree(lport);
 268 }
 269
 270 static void
 271 nvme_fc_lport_put(struct nvme_fc_lport *lport)
 272 {
 273         kref_put(&lport->ref, nvme_fc_free_lport);
 274 }
 275
 276 static int
 277 nvme_fc_lport_get(struct nvme_fc_lport *lport)
 278 {
 279         return kref_get_unless_zero(&lport->ref);
 280 }
 281
 282
 283 static struct nvme_fc_lport *
 284 nvme_fc_attach_to_unreg_lport(struct nvme_fc_port_info *pinfo,
 285                         struct nvme_fc_port_template *ops,
 286                         struct device *dev)
 287 {
 288         struct nvme_fc_lport *lport;
 289         unsigned long flags;
 290
 291         spin_lock_irqsave(&nvme_fc_lock, flags);
 292
 293         list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
 294                 if (lport->localport.node_name != pinfo->node_name ||
 295                     lport->localport.port_name != pinfo->port_name)
 296                         continue;
 297
 298                 if (lport->dev != dev) {
 299                         lport = ERR_PTR(-EXDEV);
 300                         goto out_done;
 301                 }
 302
 303                 if (lport->localport.port_state != FC_OBJSTATE_DELETED) {
 304                         lport = ERR_PTR(-EEXIST);
 305                         goto out_done;
 306                 }
 307
 308                 if (!nvme_fc_lport_get(lport)) {
 309                         /*
 310                          * fails if ref cnt already 0. If so,
 311                          * act as if lport already deleted
 312                          */
 313                         lport = NULL;
 314                         goto out_done;
 315                 }
 316
 317                 /* resume the lport */
 318
 319                 lport->ops = ops;
 320                 lport->localport.port_role = pinfo->port_role;
 321                 lport->localport.port_id = pinfo->port_id;
 322                 lport->localport.port_state = FC_OBJSTATE_ONLINE;
 323
 324                 spin_unlock_irqrestore(&nvme_fc_lock, flags);
 325
 326                 return lport;
 327         }
 328
 329         lport = NULL;
 330
 331 out_done:
 332         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 333
 334         return lport;
 335 }
 336
 337 /**
 338  * nvme_fc_register_localport - transport entry point called by an
 339  *                              LLDD to register the existence of a NVME
 340  *                              host FC port.
 341  * @pinfo:     pointer to information about the port to be registered
 342  * @template:  LLDD entrypoints and operational parameters for the port
 343  * @dev:       physical hardware device node port corresponds to. Will be
 344  *             used for DMA mappings
 345  * @portptr:   pointer to a local port pointer. Upon success, the routine
 346  *             will allocate a nvme_fc_local_port structure and place its
 347  *             address in the local port pointer. Upon failure, local port
 348  *             pointer will be set to 0.
 349  *
 350  * Returns:
 351  * a completion status. Must be 0 upon success; a negative errno
 352  * (ex: -ENXIO) upon failure.
 353  */
 354 int
 355 nvme_fc_register_localport(struct nvme_fc_port_info *pinfo,
 356                         struct nvme_fc_port_template *template,
 357                         struct device *dev,
 358                         struct nvme_fc_local_port **portptr)
 359 {
 360         struct nvme_fc_lport *newrec;
 361         unsigned long flags;
 362         int ret, idx;
 363
 364         if (!template->localport_delete || !template->remoteport_delete ||
 365             !template->ls_req || !template->fcp_io ||
 366             !template->ls_abort || !template->fcp_abort ||
 367             !template->max_hw_queues || !template->max_sgl_segments ||
 368             !template->max_dif_sgl_segments || !template->dma_boundary) {
 369                 ret = -EINVAL;
 370                 goto out_reghost_failed;
 371         }
 372
 373         /*
 374          * look to see if there is already a localport that had been
 375          * deregistered and in the process of waiting for all the
 376          * references to fully be removed.  If the references haven't
 377          * expired, we can simply re-enable the localport. Remoteports
 378          * and controller reconnections should resume naturally.
 379          */
 380         newrec = nvme_fc_attach_to_unreg_lport(pinfo, template, dev);
 381
 382         /* found an lport, but something about its state is bad */
 383         if (IS_ERR(newrec)) {
 384                 ret = PTR_ERR(newrec);
 385                 goto out_reghost_failed;
 386
 387         /* found existing lport, which was resumed */
 388         } else if (newrec) {
 389                 *portptr = &newrec->localport;
 390                 return 0;
 391         }
 392
 393         /* nothing found - allocate a new localport struct */
 394
 395         newrec = kmalloc((sizeof(*newrec) + template->local_priv_sz),
 396                          GFP_KERNEL);
 397         if (!newrec) {
 398                 ret = -ENOMEM;
 399                 goto out_reghost_failed;
 400         }
 401
 402         idx = ida_alloc(&nvme_fc_local_port_cnt, GFP_KERNEL);
 403         if (idx < 0) {
 404                 ret = -ENOSPC;
 405                 goto out_fail_kfree;
 406         }
 407
 408         if (!get_device(dev) && dev) {
 409                 ret = -ENODEV;
 410                 goto out_ida_put;
 411         }
 412
 413         INIT_LIST_HEAD(&newrec->port_list);
 414         INIT_LIST_HEAD(&newrec->endp_list);
 415         kref_init(&newrec->ref);
 416         atomic_set(&newrec->act_rport_cnt, 0);
 417         newrec->ops = template;
 418         newrec->dev = dev;
 419         ida_init(&newrec->endp_cnt);
 420         if (template->local_priv_sz)
 421                 newrec->localport.private = &newrec[1];
 422         else
 423                 newrec->localport.private = NULL;
 424         newrec->localport.node_name = pinfo->node_name;
 425         newrec->localport.port_name = pinfo->port_name;
 426         newrec->localport.port_role = pinfo->port_role;
 427         newrec->localport.port_id = pinfo->port_id;
 428         newrec->localport.port_state = FC_OBJSTATE_ONLINE;
 429         newrec->localport.port_num = idx;
 430
 431         spin_lock_irqsave(&nvme_fc_lock, flags);
 432         list_add_tail(&newrec->port_list, &nvme_fc_lport_list);
 433         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 434
 435         if (dev)
 436                 dma_set_seg_boundary(dev, template->dma_boundary);
 437
 438         *portptr = &newrec->localport;
 439         return 0;
 440
 441 out_ida_put:
 442         ida_free(&nvme_fc_local_port_cnt, idx);
 443 out_fail_kfree:
 444         kfree(newrec);
 445 out_reghost_failed:
 446         *portptr = NULL;
 447
 448         return ret;
 449 }
 450 EXPORT_SYMBOL_GPL(nvme_fc_register_localport);
 451
 452 /**
 453  * nvme_fc_unregister_localport - transport entry point called by an
 454  *                              LLDD to deregister/remove a previously
 455  *                              registered a NVME host FC port.
 456  * @portptr: pointer to the (registered) local port that is to be deregistered.
 457  *
 458  * Returns:
 459  * a completion status. Must be 0 upon success; a negative errno
 460  * (ex: -ENXIO) upon failure.
 461  */
 462 int
 463 nvme_fc_unregister_localport(struct nvme_fc_local_port *portptr)
 464 {
 465         struct nvme_fc_lport *lport = localport_to_lport(portptr);
 466         unsigned long flags;
 467
 468         if (!portptr)
 469                 return -EINVAL;
 470
 471         spin_lock_irqsave(&nvme_fc_lock, flags);
 472
 473         if (portptr->port_state != FC_OBJSTATE_ONLINE) {
 474                 spin_unlock_irqrestore(&nvme_fc_lock, flags);
 475                 return -EINVAL;
 476         }
 477         portptr->port_state = FC_OBJSTATE_DELETED;
 478
 479         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 480
 481         if (atomic_read(&lport->act_rport_cnt) == 0)
 482                 lport->ops->localport_delete(&lport->localport);
 483
 484         nvme_fc_lport_put(lport);
 485
 486         return 0;
 487 }
 488 EXPORT_SYMBOL_GPL(nvme_fc_unregister_localport);
 489
 490 /*
 491  * TRADDR strings, per FC-NVME are fixed format:
 492  *   "nn-0x<16hexdigits>:pn-0x<16hexdigits>" - 43 characters
 493  * udev event will only differ by prefix of what field is
 494  * being specified:
 495  *    "NVMEFC_HOST_TRADDR=" or "NVMEFC_TRADDR=" - 19 max characters
 496  *  19 + 43 + null_fudge = 64 characters
 497  */
 498 #define FCNVME_TRADDR_LENGTH            64
 499
 500 static void
 501 nvme_fc_signal_discovery_scan(struct nvme_fc_lport *lport,
 502                 struct nvme_fc_rport *rport)
 503 {
 504         char hostaddr[FCNVME_TRADDR_LENGTH];    /* NVMEFC_HOST_TRADDR=...*/
 505         char tgtaddr[FCNVME_TRADDR_LENGTH];     /* NVMEFC_TRADDR=...*/
 506         char *envp[4] = { "FC_EVENT=nvmediscovery", hostaddr, tgtaddr, NULL };
 507
 508         if (!(rport->remoteport.port_role & FC_PORT_ROLE_NVME_DISCOVERY))
 509                 return;
 510
 511         snprintf(hostaddr, sizeof(hostaddr),
 512                 "NVMEFC_HOST_TRADDR=nn-0x%016llx:pn-0x%016llx",
 513                 lport->localport.node_name, lport->localport.port_name);
 514         snprintf(tgtaddr, sizeof(tgtaddr),
 515                 "NVMEFC_TRADDR=nn-0x%016llx:pn-0x%016llx",
 516                 rport->remoteport.node_name, rport->remoteport.port_name);
 517         kobject_uevent_env(&fc_udev_device->kobj, KOBJ_CHANGE, envp);
 518 }
 519
 520 static void
 521 nvme_fc_free_rport(struct kref *ref)
 522 {
 523         struct nvme_fc_rport *rport =
 524                 container_of(ref, struct nvme_fc_rport, ref);
 525         struct nvme_fc_lport *lport =
 526                         localport_to_lport(rport->remoteport.localport);
 527         unsigned long flags;
 528
 529         WARN_ON(rport->remoteport.port_state != FC_OBJSTATE_DELETED);
 530         WARN_ON(!list_empty(&rport->ctrl_list));
 531
 532         /* remove from lport list */
 533         spin_lock_irqsave(&nvme_fc_lock, flags);
 534         list_del(&rport->endp_list);
 535         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 536
 537         WARN_ON(!list_empty(&rport->disc_list));
 538         ida_free(&lport->endp_cnt, rport->remoteport.port_num);
 539
 540         kfree(rport);
 541
 542         nvme_fc_lport_put(lport);
 543 }
 544
 545 static void
 546 nvme_fc_rport_put(struct nvme_fc_rport *rport)
 547 {
 548         kref_put(&rport->ref, nvme_fc_free_rport);
 549 }
 550
 551 static int
 552 nvme_fc_rport_get(struct nvme_fc_rport *rport)
 553 {
 554         return kref_get_unless_zero(&rport->ref);
 555 }
 556
 557 static void
 558 nvme_fc_resume_controller(struct nvme_fc_ctrl *ctrl)
 559 {
 560         switch (ctrl->ctrl.state) {
 561         case NVME_CTRL_NEW:
 562         case NVME_CTRL_CONNECTING:
 563                 /*
 564                  * As all reconnects were suppressed, schedule a
 565                  * connect.
 566                  */
 567                 dev_info(ctrl->ctrl.device,
 568                         "NVME-FC{%d}: connectivity re-established. "
 569                         "Attempting reconnect\n", ctrl->cnum);
 570
 571                 queue_delayed_work(nvme_wq, &ctrl->connect_work, 0);
 572                 break;
 573
 574         case NVME_CTRL_RESETTING:
 575                 /*
 576                  * Controller is already in the process of terminating the
 577                  * association. No need to do anything further. The reconnect
 578                  * step will naturally occur after the reset completes.
 579                  */
 580                 break;
 581
 582         default:
 583                 /* no action to take - let it delete */
 584                 break;
 585         }
 586 }
 587
 588 static struct nvme_fc_rport *
 589 nvme_fc_attach_to_suspended_rport(struct nvme_fc_lport *lport,
 590                                 struct nvme_fc_port_info *pinfo)
 591 {
 592         struct nvme_fc_rport *rport;
 593         struct nvme_fc_ctrl *ctrl;
 594         unsigned long flags;
 595
 596         spin_lock_irqsave(&nvme_fc_lock, flags);
 597
 598         list_for_each_entry(rport, &lport->endp_list, endp_list) {
 599                 if (rport->remoteport.node_name != pinfo->node_name ||
 600                     rport->remoteport.port_name != pinfo->port_name)
 601                         continue;
 602
 603                 if (!nvme_fc_rport_get(rport)) {
 604                         rport = ERR_PTR(-ENOLCK);
 605                         goto out_done;
 606                 }
 607
 608                 spin_unlock_irqrestore(&nvme_fc_lock, flags);
 609
 610                 spin_lock_irqsave(&rport->lock, flags);
 611
 612                 /* has it been unregistered */
 613                 if (rport->remoteport.port_state != FC_OBJSTATE_DELETED) {
 614                         /* means lldd called us twice */
 615                         spin_unlock_irqrestore(&rport->lock, flags);
 616                         nvme_fc_rport_put(rport);
 617                         return ERR_PTR(-ESTALE);
 618                 }
 619
 620                 rport->remoteport.port_role = pinfo->port_role;
 621                 rport->remoteport.port_id = pinfo->port_id;
 622                 rport->remoteport.port_state = FC_OBJSTATE_ONLINE;
 623                 rport->dev_loss_end = 0;
 624
 625                 /*
 626                  * kick off a reconnect attempt on all associations to the
 627                  * remote port. A successful reconnects will resume i/o.
 628                  */
 629                 list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list)
 630                         nvme_fc_resume_controller(ctrl);
 631
 632                 spin_unlock_irqrestore(&rport->lock, flags);
 633
 634                 return rport;
 635         }
 636
 637         rport = NULL;
 638
 639 out_done:
 640         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 641
 642         return rport;
 643 }
 644
 645 static inline void
 646 __nvme_fc_set_dev_loss_tmo(struct nvme_fc_rport *rport,
 647                         struct nvme_fc_port_info *pinfo)
 648 {
 649         if (pinfo->dev_loss_tmo)
 650                 rport->remoteport.dev_loss_tmo = pinfo->dev_loss_tmo;
 651         else
 652                 rport->remoteport.dev_loss_tmo = NVME_FC_DEFAULT_DEV_LOSS_TMO;
 653 }
 654
 655 /**
 656  * nvme_fc_register_remoteport - transport entry point called by an
 657  *                              LLDD to register the existence of a NVME
 658  *                              subsystem FC port on its fabric.
 659  * @localport: pointer to the (registered) local port that the remote
 660  *             subsystem port is connected to.
 661  * @pinfo:     pointer to information about the port to be registered
 662  * @portptr:   pointer to a remote port pointer. Upon success, the routine
 663  *             will allocate a nvme_fc_remote_port structure and place its
 664  *             address in the remote port pointer. Upon failure, remote port
 665  *             pointer will be set to 0.
 666  *
 667  * Returns:
 668  * a completion status. Must be 0 upon success; a negative errno
 669  * (ex: -ENXIO) upon failure.
 670  */
 671 int
 672 nvme_fc_register_remoteport(struct nvme_fc_local_port *localport,
 673                                 struct nvme_fc_port_info *pinfo,
 674                                 struct nvme_fc_remote_port **portptr)
 675 {
 676         struct nvme_fc_lport *lport = localport_to_lport(localport);
 677         struct nvme_fc_rport *newrec;
 678         unsigned long flags;
 679         int ret, idx;
 680
 681         if (!nvme_fc_lport_get(lport)) {
 682                 ret = -ESHUTDOWN;
 683                 goto out_reghost_failed;
 684         }
 685
 686         /*
 687          * look to see if there is already a remoteport that is waiting
 688          * for a reconnect (within dev_loss_tmo) with the same WWN's.
 689          * If so, transition to it and reconnect.
 690          */
 691         newrec = nvme_fc_attach_to_suspended_rport(lport, pinfo);
 692
 693         /* found an rport, but something about its state is bad */
 694         if (IS_ERR(newrec)) {
 695                 ret = PTR_ERR(newrec);
 696                 goto out_lport_put;
 697
 698         /* found existing rport, which was resumed */
 699         } else if (newrec) {
 700                 nvme_fc_lport_put(lport);
 701                 __nvme_fc_set_dev_loss_tmo(newrec, pinfo);
 702                 nvme_fc_signal_discovery_scan(lport, newrec);
 703                 *portptr = &newrec->remoteport;
 704                 return 0;
 705         }
 706
 707         /* nothing found - allocate a new remoteport struct */
 708
 709         newrec = kmalloc((sizeof(*newrec) + lport->ops->remote_priv_sz),
 710                          GFP_KERNEL);
 711         if (!newrec) {
 712                 ret = -ENOMEM;
 713                 goto out_lport_put;
 714         }
 715
 716         idx = ida_alloc(&lport->endp_cnt, GFP_KERNEL);
 717         if (idx < 0) {
 718                 ret = -ENOSPC;
 719                 goto out_kfree_rport;
 720         }
 721
 722         INIT_LIST_HEAD(&newrec->endp_list);
 723         INIT_LIST_HEAD(&newrec->ctrl_list);
 724         INIT_LIST_HEAD(&newrec->ls_req_list);
 725         INIT_LIST_HEAD(&newrec->disc_list);
 726         kref_init(&newrec->ref);
 727         atomic_set(&newrec->act_ctrl_cnt, 0);
 728         spin_lock_init(&newrec->lock);
 729         newrec->remoteport.localport = &lport->localport;
 730         INIT_LIST_HEAD(&newrec->ls_rcv_list);
 731         newrec->dev = lport->dev;
 732         newrec->lport = lport;
 733         if (lport->ops->remote_priv_sz)
 734                 newrec->remoteport.private = &newrec[1];
 735         else
 736                 newrec->remoteport.private = NULL;
 737         newrec->remoteport.port_role = pinfo->port_role;
 738         newrec->remoteport.node_name = pinfo->node_name;
 739         newrec->remoteport.port_name = pinfo->port_name;
 740         newrec->remoteport.port_id = pinfo->port_id;
 741         newrec->remoteport.port_state = FC_OBJSTATE_ONLINE;
 742         newrec->remoteport.port_num = idx;
 743         __nvme_fc_set_dev_loss_tmo(newrec, pinfo);
 744         INIT_WORK(&newrec->lsrcv_work, nvme_fc_handle_ls_rqst_work);
 745
 746         spin_lock_irqsave(&nvme_fc_lock, flags);
 747         list_add_tail(&newrec->endp_list, &lport->endp_list);
 748         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 749
 750         nvme_fc_signal_discovery_scan(lport, newrec);
 751
 752         *portptr = &newrec->remoteport;
 753         return 0;
 754
 755 out_kfree_rport:
 756         kfree(newrec);
 757 out_lport_put:
 758         nvme_fc_lport_put(lport);
 759 out_reghost_failed:
 760         *portptr = NULL;
 761         return ret;
 762 }
 763 EXPORT_SYMBOL_GPL(nvme_fc_register_remoteport);
 764
 765 static int
 766 nvme_fc_abort_lsops(struct nvme_fc_rport *rport)
 767 {
 768         struct nvmefc_ls_req_op *lsop;
 769         unsigned long flags;
 770
 771 restart:
 772         spin_lock_irqsave(&rport->lock, flags);
 773
 774         list_for_each_entry(lsop, &rport->ls_req_list, lsreq_list) {
 775                 if (!(lsop->flags & FCOP_FLAGS_TERMIO)) {
 776                         lsop->flags |= FCOP_FLAGS_TERMIO;
 777                         spin_unlock_irqrestore(&rport->lock, flags);
 778                         rport->lport->ops->ls_abort(&rport->lport->localport,
 779                                                 &rport->remoteport,
 780                                                 &lsop->ls_req);
 781                         goto restart;
 782                 }
 783         }
 784         spin_unlock_irqrestore(&rport->lock, flags);
 785
 786         return 0;
 787 }
 788
 789 static void
 790 nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl)
 791 {
 792         dev_info(ctrl->ctrl.device,
 793                 "NVME-FC{%d}: controller connectivity lost. Awaiting "
 794                 "Reconnect", ctrl->cnum);
 795
 796         switch (ctrl->ctrl.state) {
 797         case NVME_CTRL_NEW:
 798         case NVME_CTRL_LIVE:
 799                 /*
 800                  * Schedule a controller reset. The reset will terminate the
 801                  * association and schedule the reconnect timer.  Reconnects
 802                  * will be attempted until either the ctlr_loss_tmo
 803                  * (max_retries * connect_delay) expires or the remoteport's
 804                  * dev_loss_tmo expires.
 805                  */
 806                 if (nvme_reset_ctrl(&ctrl->ctrl)) {
 807                         dev_warn(ctrl->ctrl.device,
 808                                 "NVME-FC{%d}: Couldn't schedule reset.\n",
 809                                 ctrl->cnum);
 810                         nvme_delete_ctrl(&ctrl->ctrl);
 811                 }
 812                 break;
 813
 814         case NVME_CTRL_CONNECTING:
 815                 /*
 816                  * The association has already been terminated and the
 817                  * controller is attempting reconnects.  No need to do anything
 818                  * futher.  Reconnects will be attempted until either the
 819                  * ctlr_loss_tmo (max_retries * connect_delay) expires or the
 820                  * remoteport's dev_loss_tmo expires.
 821                  */
 822                 break;
 823
 824         case NVME_CTRL_RESETTING:
 825                 /*
 826                  * Controller is already in the process of terminating the
 827                  * association.  No need to do anything further. The reconnect
 828                  * step will kick in naturally after the association is
 829                  * terminated.
 830                  */
 831                 break;
 832
 833         case NVME_CTRL_DELETING:
 834         case NVME_CTRL_DELETING_NOIO:
 835         default:
 836                 /* no action to take - let it delete */
 837                 break;
 838         }
 839 }
 840
 841 /**
 842  * nvme_fc_unregister_remoteport - transport entry point called by an
 843  *                              LLDD to deregister/remove a previously
 844  *                              registered a NVME subsystem FC port.
 845  * @portptr: pointer to the (registered) remote port that is to be
 846  *           deregistered.
 847  *
 848  * Returns:
 849  * a completion status. Must be 0 upon success; a negative errno
 850  * (ex: -ENXIO) upon failure.
 851  */
 852 int
 853 nvme_fc_unregister_remoteport(struct nvme_fc_remote_port *portptr)
 854 {
 855         struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
 856         struct nvme_fc_ctrl *ctrl;
 857         unsigned long flags;
 858
 859         if (!portptr)
 860                 return -EINVAL;
 861
 862         spin_lock_irqsave(&rport->lock, flags);
 863
 864         if (portptr->port_state != FC_OBJSTATE_ONLINE) {
 865                 spin_unlock_irqrestore(&rport->lock, flags);
 866                 return -EINVAL;
 867         }
 868         portptr->port_state = FC_OBJSTATE_DELETED;
 869
 870         rport->dev_loss_end = jiffies + (portptr->dev_loss_tmo * HZ);
 871
 872         list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
 873                 /* if dev_loss_tmo==0, dev loss is immediate */
 874                 if (!portptr->dev_loss_tmo) {
 875                         dev_warn(ctrl->ctrl.device,
 876                                 "NVME-FC{%d}: controller connectivity lost.\n",
 877                                 ctrl->cnum);
 878                         nvme_delete_ctrl(&ctrl->ctrl);
 879                 } else
 880                         nvme_fc_ctrl_connectivity_loss(ctrl);
 881         }
 882
 883         spin_unlock_irqrestore(&rport->lock, flags);
 884
 885         nvme_fc_abort_lsops(rport);
 886
 887         if (atomic_read(&rport->act_ctrl_cnt) == 0)
 888                 rport->lport->ops->remoteport_delete(portptr);
 889
 890         /*
 891          * release the reference, which will allow, if all controllers
 892          * go away, which should only occur after dev_loss_tmo occurs,
 893          * for the rport to be torn down.
 894          */
 895         nvme_fc_rport_put(rport);
 896
 897         return 0;
 898 }
 899 EXPORT_SYMBOL_GPL(nvme_fc_unregister_remoteport);
 900
 901 /**
 902  * nvme_fc_rescan_remoteport - transport entry point called by an
 903  *                              LLDD to request a nvme device rescan.
 904  * @remoteport: pointer to the (registered) remote port that is to be
 905  *              rescanned.
 906  *
 907  * Returns: N/A
 908  */
 909 void
 910 nvme_fc_rescan_remoteport(struct nvme_fc_remote_port *remoteport)
 911 {
 912         struct nvme_fc_rport *rport = remoteport_to_rport(remoteport);
 913
 914         nvme_fc_signal_discovery_scan(rport->lport, rport);
 915 }
 916 EXPORT_SYMBOL_GPL(nvme_fc_rescan_remoteport);
 917
 918 int
 919 nvme_fc_set_remoteport_devloss(struct nvme_fc_remote_port *portptr,
 920                         u32 dev_loss_tmo)
 921 {
 922         struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
 923         unsigned long flags;
 924
 925         spin_lock_irqsave(&rport->lock, flags);
 926
 927         if (portptr->port_state != FC_OBJSTATE_ONLINE) {
 928                 spin_unlock_irqrestore(&rport->lock, flags);
 929                 return -EINVAL;
 930         }
 931
 932         /* a dev_loss_tmo of 0 (immediate) is allowed to be set */
 933         rport->remoteport.dev_loss_tmo = dev_loss_tmo;
 934
 935         spin_unlock_irqrestore(&rport->lock, flags);
 936
 937         return 0;
 938 }
 939 EXPORT_SYMBOL_GPL(nvme_fc_set_remoteport_devloss);
 940
 941
 942 /* *********************** FC-NVME DMA Handling **************************** */
 943
 944 /*
 945  * The fcloop device passes in a NULL device pointer. Real LLD's will
 946  * pass in a valid device pointer. If NULL is passed to the dma mapping
 947  * routines, depending on the platform, it may or may not succeed, and
 948  * may crash.
 949  *
 950  * As such:
 951  * Wrapper all the dma routines and check the dev pointer.
 952  *
 953  * If simple mappings (return just a dma address, we'll noop them,
 954  * returning a dma address of 0.
 955  *
 956  * On more complex mappings (dma_map_sg), a pseudo routine fills
 957  * in the scatter list, setting all dma addresses to 0.
 958  */
 959
 960 static inline dma_addr_t
 961 fc_dma_map_single(struct device *dev, void *ptr, size_t size,
 962                 enum dma_data_direction dir)
 963 {
 964         return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
 965 }
 966
 967 static inline int
 968 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 969 {
 970         return dev ? dma_mapping_error(dev, dma_addr) : 0;
 971 }
 972
 973 static inline void
 974 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
 975         enum dma_data_direction dir)
 976 {
 977         if (dev)
 978                 dma_unmap_single(dev, addr, size, dir);
 979 }
 980
 981 static inline void
 982 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
 983                 enum dma_data_direction dir)
 984 {
 985         if (dev)
 986                 dma_sync_single_for_cpu(dev, addr, size, dir);
 987 }
 988
 989 static inline void
 990 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
 991                 enum dma_data_direction dir)
 992 {
 993         if (dev)
 994                 dma_sync_single_for_device(dev, addr, size, dir);
 995 }
 996
 997 /* pseudo dma_map_sg call */
 998 static int
 999 fc_map_sg(struct scatterlist *sg, int nents)
1000 {
1001         struct scatterlist *s;
1002         int i;
1003
1004         WARN_ON(nents == 0 || sg[0].length == 0);
1005
1006         for_each_sg(sg, s, nents, i) {
1007                 s->dma_address = 0L;
1008 #ifdef CONFIG_NEED_SG_DMA_LENGTH
1009                 s->dma_length = s->length;
1010 #endif
1011         }
1012         return nents;
1013 }
1014
1015 static inline int
1016 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1017                 enum dma_data_direction dir)
1018 {
1019         return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
1020 }
1021
1022 static inline void
1023 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1024                 enum dma_data_direction dir)
1025 {
1026         if (dev)
1027                 dma_unmap_sg(dev, sg, nents, dir);
1028 }
1029
1030 /* *********************** FC-NVME LS Handling **************************** */
1031
1032 static void nvme_fc_ctrl_put(struct nvme_fc_ctrl *);
1033 static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *);
1034
1035 static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg);
1036
1037 static void
1038 __nvme_fc_finish_ls_req(struct nvmefc_ls_req_op *lsop)
1039 {
1040         struct nvme_fc_rport *rport = lsop->rport;
1041         struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1042         unsigned long flags;
1043
1044         spin_lock_irqsave(&rport->lock, flags);
1045
1046         if (!lsop->req_queued) {
1047                 spin_unlock_irqrestore(&rport->lock, flags);
1048                 return;
1049         }
1050
1051         list_del(&lsop->lsreq_list);
1052
1053         lsop->req_queued = false;
1054
1055         spin_unlock_irqrestore(&rport->lock, flags);
1056
1057         fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
1058                                   (lsreq->rqstlen + lsreq->rsplen),
1059                                   DMA_BIDIRECTIONAL);
1060
1061         nvme_fc_rport_put(rport);
1062 }
1063
1064 static int
1065 __nvme_fc_send_ls_req(struct nvme_fc_rport *rport,
1066                 struct nvmefc_ls_req_op *lsop,
1067                 void (*done)(struct nvmefc_ls_req *req, int status))
1068 {
1069         struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1070         unsigned long flags;
1071         int ret = 0;
1072
1073         if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
1074                 return -ECONNREFUSED;
1075
1076         if (!nvme_fc_rport_get(rport))
1077                 return -ESHUTDOWN;
1078
1079         lsreq->done = done;
1080         lsop->rport = rport;
1081         lsop->req_queued = false;
1082         INIT_LIST_HEAD(&lsop->lsreq_list);
1083         init_completion(&lsop->ls_done);
1084
1085         lsreq->rqstdma = fc_dma_map_single(rport->dev, lsreq->rqstaddr,
1086                                   lsreq->rqstlen + lsreq->rsplen,
1087                                   DMA_BIDIRECTIONAL);
1088         if (fc_dma_mapping_error(rport->dev, lsreq->rqstdma)) {
1089                 ret = -EFAULT;
1090                 goto out_putrport;
1091         }
1092         lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;
1093
1094         spin_lock_irqsave(&rport->lock, flags);
1095
1096         list_add_tail(&lsop->lsreq_list, &rport->ls_req_list);
1097
1098         lsop->req_queued = true;
1099
1100         spin_unlock_irqrestore(&rport->lock, flags);
1101
1102         ret = rport->lport->ops->ls_req(&rport->lport->localport,
1103                                         &rport->remoteport, lsreq);
1104         if (ret)
1105                 goto out_unlink;
1106
1107         return 0;
1108
1109 out_unlink:
1110         lsop->ls_error = ret;
1111         spin_lock_irqsave(&rport->lock, flags);
1112         lsop->req_queued = false;
1113         list_del(&lsop->lsreq_list);
1114         spin_unlock_irqrestore(&rport->lock, flags);
1115         fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
1116                                   (lsreq->rqstlen + lsreq->rsplen),
1117                                   DMA_BIDIRECTIONAL);
1118 out_putrport:
1119         nvme_fc_rport_put(rport);
1120
1121         return ret;
1122 }
1123
1124 static void
1125 nvme_fc_send_ls_req_done(struct nvmefc_ls_req *lsreq, int status)
1126 {
1127         struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1128
1129         lsop->ls_error = status;
1130         complete(&lsop->ls_done);
1131 }
1132
1133 static int
1134 nvme_fc_send_ls_req(struct nvme_fc_rport *rport, struct nvmefc_ls_req_op *lsop)
1135 {
1136         struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1137         struct fcnvme_ls_rjt *rjt = lsreq->rspaddr;
1138         int ret;
1139
1140         ret = __nvme_fc_send_ls_req(rport, lsop, nvme_fc_send_ls_req_done);
1141
1142         if (!ret) {
1143                 /*
1144                  * No timeout/not interruptible as we need the struct
1145                  * to exist until the lldd calls us back. Thus mandate
1146                  * wait until driver calls back. lldd responsible for
1147                  * the timeout action
1148                  */
1149                 wait_for_completion(&lsop->ls_done);
1150
1151                 __nvme_fc_finish_ls_req(lsop);
1152
1153                 ret = lsop->ls_error;
1154         }
1155
1156         if (ret)
1157                 return ret;
1158
1159         /* ACC or RJT payload ? */
1160         if (rjt->w0.ls_cmd == FCNVME_LS_RJT)
1161                 return -ENXIO;
1162
1163         return 0;
1164 }
1165
1166 static int
1167 nvme_fc_send_ls_req_async(struct nvme_fc_rport *rport,
1168                 struct nvmefc_ls_req_op *lsop,
1169                 void (*done)(struct nvmefc_ls_req *req, int status))
1170 {
1171         /* don't wait for completion */
1172
1173         return __nvme_fc_send_ls_req(rport, lsop, done);
1174 }
1175
1176 static int
1177 nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl,
1178         struct nvme_fc_queue *queue, u16 qsize, u16 ersp_ratio)
1179 {
1180         struct nvmefc_ls_req_op *lsop;
1181         struct nvmefc_ls_req *lsreq;
1182         struct fcnvme_ls_cr_assoc_rqst *assoc_rqst;
1183         struct fcnvme_ls_cr_assoc_acc *assoc_acc;
1184         unsigned long flags;
1185         int ret, fcret = 0;
1186
1187         lsop = kzalloc((sizeof(*lsop) +
1188                          sizeof(*assoc_rqst) + sizeof(*assoc_acc) +
1189                          ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1190         if (!lsop) {
1191                 dev_info(ctrl->ctrl.device,
1192                         "NVME-FC{%d}: send Create Association failed: ENOMEM\n",
1193                         ctrl->cnum);
1194                 ret = -ENOMEM;
1195                 goto out_no_memory;
1196         }
1197
1198         assoc_rqst = (struct fcnvme_ls_cr_assoc_rqst *)&lsop[1];
1199         assoc_acc = (struct fcnvme_ls_cr_assoc_acc *)&assoc_rqst[1];
1200         lsreq = &lsop->ls_req;
1201         if (ctrl->lport->ops->lsrqst_priv_sz)
1202                 lsreq->private = &assoc_acc[1];
1203         else
1204                 lsreq->private = NULL;
1205
1206         assoc_rqst->w0.ls_cmd = FCNVME_LS_CREATE_ASSOCIATION;
1207         assoc_rqst->desc_list_len =
1208                         cpu_to_be32(sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1209
1210         assoc_rqst->assoc_cmd.desc_tag =
1211                         cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD);
1212         assoc_rqst->assoc_cmd.desc_len =
1213                         fcnvme_lsdesc_len(
1214                                 sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1215
1216         assoc_rqst->assoc_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1217         assoc_rqst->assoc_cmd.sqsize = cpu_to_be16(qsize - 1);
1218         /* Linux supports only Dynamic controllers */
1219         assoc_rqst->assoc_cmd.cntlid = cpu_to_be16(0xffff);
1220         uuid_copy(&assoc_rqst->assoc_cmd.hostid, &ctrl->ctrl.opts->host->id);
1221         strncpy(assoc_rqst->assoc_cmd.hostnqn, ctrl->ctrl.opts->host->nqn,
1222                 min(FCNVME_ASSOC_HOSTNQN_LEN, NVMF_NQN_SIZE));
1223         strncpy(assoc_rqst->assoc_cmd.subnqn, ctrl->ctrl.opts->subsysnqn,
1224                 min(FCNVME_ASSOC_SUBNQN_LEN, NVMF_NQN_SIZE));
1225
1226         lsop->queue = queue;
1227         lsreq->rqstaddr = assoc_rqst;
1228         lsreq->rqstlen = sizeof(*assoc_rqst);
1229         lsreq->rspaddr = assoc_acc;
1230         lsreq->rsplen = sizeof(*assoc_acc);
1231         lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1232
1233         ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
1234         if (ret)
1235                 goto out_free_buffer;
1236
1237         /* process connect LS completion */
1238
1239         /* validate the ACC response */
1240         if (assoc_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1241                 fcret = VERR_LSACC;
1242         else if (assoc_acc->hdr.desc_list_len !=
1243                         fcnvme_lsdesc_len(
1244                                 sizeof(struct fcnvme_ls_cr_assoc_acc)))
1245                 fcret = VERR_CR_ASSOC_ACC_LEN;
1246         else if (assoc_acc->hdr.rqst.desc_tag !=
1247                         cpu_to_be32(FCNVME_LSDESC_RQST))
1248                 fcret = VERR_LSDESC_RQST;
1249         else if (assoc_acc->hdr.rqst.desc_len !=
1250                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
1251                 fcret = VERR_LSDESC_RQST_LEN;
1252         else if (assoc_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_ASSOCIATION)
1253                 fcret = VERR_CR_ASSOC;
1254         else if (assoc_acc->associd.desc_tag !=
1255                         cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1256                 fcret = VERR_ASSOC_ID;
1257         else if (assoc_acc->associd.desc_len !=
1258                         fcnvme_lsdesc_len(
1259                                 sizeof(struct fcnvme_lsdesc_assoc_id)))
1260                 fcret = VERR_ASSOC_ID_LEN;
1261         else if (assoc_acc->connectid.desc_tag !=
1262                         cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1263                 fcret = VERR_CONN_ID;
1264         else if (assoc_acc->connectid.desc_len !=
1265                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
1266                 fcret = VERR_CONN_ID_LEN;
1267
1268         if (fcret) {
1269                 ret = -EBADF;
1270                 dev_err(ctrl->dev,
1271                         "q %d Create Association LS failed: %s\n",
1272                         queue->qnum, validation_errors[fcret]);
1273         } else {
1274                 spin_lock_irqsave(&ctrl->lock, flags);
1275                 ctrl->association_id =
1276                         be64_to_cpu(assoc_acc->associd.association_id);
1277                 queue->connection_id =
1278                         be64_to_cpu(assoc_acc->connectid.connection_id);
1279                 set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
1280                 spin_unlock_irqrestore(&ctrl->lock, flags);
1281         }
1282
1283 out_free_buffer:
1284         kfree(lsop);
1285 out_no_memory:
1286         if (ret)
1287                 dev_err(ctrl->dev,
1288                         "queue %d connect admin queue failed (%d).\n",
1289                         queue->qnum, ret);
1290         return ret;
1291 }
1292
1293 static int
1294 nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
1295                         u16 qsize, u16 ersp_ratio)
1296 {
1297         struct nvmefc_ls_req_op *lsop;
1298         struct nvmefc_ls_req *lsreq;
1299         struct fcnvme_ls_cr_conn_rqst *conn_rqst;
1300         struct fcnvme_ls_cr_conn_acc *conn_acc;
1301         int ret, fcret = 0;
1302
1303         lsop = kzalloc((sizeof(*lsop) +
1304                          sizeof(*conn_rqst) + sizeof(*conn_acc) +
1305                          ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1306         if (!lsop) {
1307                 dev_info(ctrl->ctrl.device,
1308                         "NVME-FC{%d}: send Create Connection failed: ENOMEM\n",
1309                         ctrl->cnum);
1310                 ret = -ENOMEM;
1311                 goto out_no_memory;
1312         }
1313
1314         conn_rqst = (struct fcnvme_ls_cr_conn_rqst *)&lsop[1];
1315         conn_acc = (struct fcnvme_ls_cr_conn_acc *)&conn_rqst[1];
1316         lsreq = &lsop->ls_req;
1317         if (ctrl->lport->ops->lsrqst_priv_sz)
1318                 lsreq->private = (void *)&conn_acc[1];
1319         else
1320                 lsreq->private = NULL;
1321
1322         conn_rqst->w0.ls_cmd = FCNVME_LS_CREATE_CONNECTION;
1323         conn_rqst->desc_list_len = cpu_to_be32(
1324                                 sizeof(struct fcnvme_lsdesc_assoc_id) +
1325                                 sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1326
1327         conn_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1328         conn_rqst->associd.desc_len =
1329                         fcnvme_lsdesc_len(
1330                                 sizeof(struct fcnvme_lsdesc_assoc_id));
1331         conn_rqst->associd.association_id = cpu_to_be64(ctrl->association_id);
1332         conn_rqst->connect_cmd.desc_tag =
1333                         cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD);
1334         conn_rqst->connect_cmd.desc_len =
1335                         fcnvme_lsdesc_len(
1336                                 sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1337         conn_rqst->connect_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1338         conn_rqst->connect_cmd.qid  = cpu_to_be16(queue->qnum);
1339         conn_rqst->connect_cmd.sqsize = cpu_to_be16(qsize - 1);
1340
1341         lsop->queue = queue;
1342         lsreq->rqstaddr = conn_rqst;
1343         lsreq->rqstlen = sizeof(*conn_rqst);
1344         lsreq->rspaddr = conn_acc;
1345         lsreq->rsplen = sizeof(*conn_acc);
1346         lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1347
1348         ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
1349         if (ret)
1350                 goto out_free_buffer;
1351
1352         /* process connect LS completion */
1353
1354         /* validate the ACC response */
1355         if (conn_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1356                 fcret = VERR_LSACC;
1357         else if (conn_acc->hdr.desc_list_len !=
1358                         fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)))
1359                 fcret = VERR_CR_CONN_ACC_LEN;
1360         else if (conn_acc->hdr.rqst.desc_tag != cpu_to_be32(FCNVME_LSDESC_RQST))
1361                 fcret = VERR_LSDESC_RQST;
1362         else if (conn_acc->hdr.rqst.desc_len !=
1363                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
1364                 fcret = VERR_LSDESC_RQST_LEN;
1365         else if (conn_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_CONNECTION)
1366                 fcret = VERR_CR_CONN;
1367         else if (conn_acc->connectid.desc_tag !=
1368                         cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1369                 fcret = VERR_CONN_ID;
1370         else if (conn_acc->connectid.desc_len !=
1371                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
1372                 fcret = VERR_CONN_ID_LEN;
1373
1374         if (fcret) {
1375                 ret = -EBADF;
1376                 dev_err(ctrl->dev,
1377                         "q %d Create I/O Connection LS failed: %s\n",
1378                         queue->qnum, validation_errors[fcret]);
1379         } else {
1380                 queue->connection_id =
1381                         be64_to_cpu(conn_acc->connectid.connection_id);
1382                 set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
1383         }
1384
1385 out_free_buffer:
1386         kfree(lsop);
1387 out_no_memory:
1388         if (ret)
1389                 dev_err(ctrl->dev,
1390                         "queue %d connect I/O queue failed (%d).\n",
1391                         queue->qnum, ret);
1392         return ret;
1393 }
1394
1395 static void
1396 nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)
1397 {
1398         struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1399
1400         __nvme_fc_finish_ls_req(lsop);
1401
1402         /* fc-nvme initiator doesn't care about success or failure of cmd */
1403
1404         kfree(lsop);
1405 }
1406
1407 /*
1408  * This routine sends a FC-NVME LS to disconnect (aka terminate)
1409  * the FC-NVME Association.  Terminating the association also
1410  * terminates the FC-NVME connections (per queue, both admin and io
1411  * queues) that are part of the association. E.g. things are torn
1412  * down, and the related FC-NVME Association ID and Connection IDs
1413  * become invalid.
1414  *
1415  * The behavior of the fc-nvme initiator is such that it's
1416  * understanding of the association and connections will implicitly
1417  * be torn down. The action is implicit as it may be due to a loss of
1418  * connectivity with the fc-nvme target, so you may never get a
1419  * response even if you tried.  As such, the action of this routine
1420  * is to asynchronously send the LS, ignore any results of the LS, and
1421  * continue on with terminating the association. If the fc-nvme target
1422  * is present and receives the LS, it too can tear down.
1423  */
1424 static void
1425 nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl)
1426 {
1427         struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst;
1428         struct fcnvme_ls_disconnect_assoc_acc *discon_acc;
1429         struct nvmefc_ls_req_op *lsop;
1430         struct nvmefc_ls_req *lsreq;
1431         int ret;
1432
1433         lsop = kzalloc((sizeof(*lsop) +
1434                         sizeof(*discon_rqst) + sizeof(*discon_acc) +
1435                         ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1436         if (!lsop) {
1437                 dev_info(ctrl->ctrl.device,
1438                         "NVME-FC{%d}: send Disconnect Association "
1439                         "failed: ENOMEM\n",
1440                         ctrl->cnum);
1441                 return;
1442         }
1443
1444         discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1];
1445         discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1];
1446         lsreq = &lsop->ls_req;
1447         if (ctrl->lport->ops->lsrqst_priv_sz)
1448                 lsreq->private = (void *)&discon_acc[1];
1449         else
1450                 lsreq->private = NULL;
1451
1452         nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc,
1453                                 ctrl->association_id);
1454
1455         ret = nvme_fc_send_ls_req_async(ctrl->rport, lsop,
1456                                 nvme_fc_disconnect_assoc_done);
1457         if (ret)
1458                 kfree(lsop);
1459 }
1460
1461 static void
1462 nvme_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp)
1463 {
1464         struct nvmefc_ls_rcv_op *lsop = lsrsp->nvme_fc_private;
1465         struct nvme_fc_rport *rport = lsop->rport;
1466         struct nvme_fc_lport *lport = rport->lport;
1467         unsigned long flags;
1468
1469         spin_lock_irqsave(&rport->lock, flags);
1470         list_del(&lsop->lsrcv_list);
1471         spin_unlock_irqrestore(&rport->lock, flags);
1472
1473         fc_dma_sync_single_for_cpu(lport->dev, lsop->rspdma,
1474                                 sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1475         fc_dma_unmap_single(lport->dev, lsop->rspdma,
1476                         sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1477
1478         kfree(lsop);
1479
1480         nvme_fc_rport_put(rport);
1481 }
1482
1483 static void
1484 nvme_fc_xmt_ls_rsp(struct nvmefc_ls_rcv_op *lsop)
1485 {
1486         struct nvme_fc_rport *rport = lsop->rport;
1487         struct nvme_fc_lport *lport = rport->lport;
1488         struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1489         int ret;
1490
1491         fc_dma_sync_single_for_device(lport->dev, lsop->rspdma,
1492                                   sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1493
1494         ret = lport->ops->xmt_ls_rsp(&lport->localport, &rport->remoteport,
1495                                      lsop->lsrsp);
1496         if (ret) {
1497                 dev_warn(lport->dev,
1498                         "LLDD rejected LS RSP xmt: LS %d status %d\n",
1499                         w0->ls_cmd, ret);
1500                 nvme_fc_xmt_ls_rsp_done(lsop->lsrsp);
1501                 return;
1502         }
1503 }
1504
1505 static struct nvme_fc_ctrl *
1506 nvme_fc_match_disconn_ls(struct nvme_fc_rport *rport,
1507                       struct nvmefc_ls_rcv_op *lsop)
1508 {
1509         struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1510                                         &lsop->rqstbuf->rq_dis_assoc;
1511         struct nvme_fc_ctrl *ctrl, *ret = NULL;
1512         struct nvmefc_ls_rcv_op *oldls = NULL;
1513         u64 association_id = be64_to_cpu(rqst->associd.association_id);
1514         unsigned long flags;
1515
1516         spin_lock_irqsave(&rport->lock, flags);
1517
1518         list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
1519                 if (!nvme_fc_ctrl_get(ctrl))
1520                         continue;
1521                 spin_lock(&ctrl->lock);
1522                 if (association_id == ctrl->association_id) {
1523                         oldls = ctrl->rcv_disconn;
1524                         ctrl->rcv_disconn = lsop;
1525                         ret = ctrl;
1526                 }
1527                 spin_unlock(&ctrl->lock);
1528                 if (ret)
1529                         /* leave the ctrl get reference */
1530                         break;
1531                 nvme_fc_ctrl_put(ctrl);
1532         }
1533
1534         spin_unlock_irqrestore(&rport->lock, flags);
1535
1536         /* transmit a response for anything that was pending */
1537         if (oldls) {
1538                 dev_info(rport->lport->dev,
1539                         "NVME-FC{%d}: Multiple Disconnect Association "
1540                         "LS's received\n", ctrl->cnum);
1541                 /* overwrite good response with bogus failure */
1542                 oldls->lsrsp->rsplen = nvme_fc_format_rjt(oldls->rspbuf,
1543                                                 sizeof(*oldls->rspbuf),
1544                                                 rqst->w0.ls_cmd,
1545                                                 FCNVME_RJT_RC_UNAB,
1546                                                 FCNVME_RJT_EXP_NONE, 0);
1547                 nvme_fc_xmt_ls_rsp(oldls);
1548         }
1549
1550         return ret;
1551 }
1552
1553 /*
1554  * returns true to mean LS handled and ls_rsp can be sent
1555  * returns false to defer ls_rsp xmt (will be done as part of
1556  *     association termination)
1557  */
1558 static bool
1559 nvme_fc_ls_disconnect_assoc(struct nvmefc_ls_rcv_op *lsop)
1560 {
1561         struct nvme_fc_rport *rport = lsop->rport;
1562         struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1563                                         &lsop->rqstbuf->rq_dis_assoc;
1564         struct fcnvme_ls_disconnect_assoc_acc *acc =
1565                                         &lsop->rspbuf->rsp_dis_assoc;
1566         struct nvme_fc_ctrl *ctrl = NULL;
1567         int ret = 0;
1568
1569         memset(acc, 0, sizeof(*acc));
1570
1571         ret = nvmefc_vldt_lsreq_discon_assoc(lsop->rqstdatalen, rqst);
1572         if (!ret) {
1573                 /* match an active association */
1574                 ctrl = nvme_fc_match_disconn_ls(rport, lsop);
1575                 if (!ctrl)
1576                         ret = VERR_NO_ASSOC;
1577         }
1578
1579         if (ret) {
1580                 dev_info(rport->lport->dev,
1581                         "Disconnect LS failed: %s\n",
1582                         validation_errors[ret]);
1583                 lsop->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1584                                         sizeof(*acc), rqst->w0.ls_cmd,
1585                                         (ret == VERR_NO_ASSOC) ?
1586                                                 FCNVME_RJT_RC_INV_ASSOC :
1587                                                 FCNVME_RJT_RC_LOGIC,
1588                                         FCNVME_RJT_EXP_NONE, 0);
1589                 return true;
1590         }
1591
1592         /* format an ACCept response */
1593
1594         lsop->lsrsp->rsplen = sizeof(*acc);
1595
1596         nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1597                         fcnvme_lsdesc_len(
1598                                 sizeof(struct fcnvme_ls_disconnect_assoc_acc)),
1599                         FCNVME_LS_DISCONNECT_ASSOC);
1600
1601         /*
1602          * the transmit of the response will occur after the exchanges
1603          * for the association have been ABTS'd by
1604          * nvme_fc_delete_association().
1605          */
1606
1607         /* fail the association */
1608         nvme_fc_error_recovery(ctrl, "Disconnect Association LS received");
1609
1610         /* release the reference taken by nvme_fc_match_disconn_ls() */
1611         nvme_fc_ctrl_put(ctrl);
1612
1613         return false;
1614 }
1615
1616 /*
1617  * Actual Processing routine for received FC-NVME LS Requests from the LLD
1618  * returns true if a response should be sent afterward, false if rsp will
1619  * be sent asynchronously.
1620  */
1621 static bool
1622 nvme_fc_handle_ls_rqst(struct nvmefc_ls_rcv_op *lsop)
1623 {
1624         struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1625         bool ret = true;
1626
1627         lsop->lsrsp->nvme_fc_private = lsop;
1628         lsop->lsrsp->rspbuf = lsop->rspbuf;
1629         lsop->lsrsp->rspdma = lsop->rspdma;
1630         lsop->lsrsp->done = nvme_fc_xmt_ls_rsp_done;
1631         /* Be preventative. handlers will later set to valid length */
1632         lsop->lsrsp->rsplen = 0;
1633
1634         /*
1635          * handlers:
1636          *   parse request input, execute the request, and format the
1637          *   LS response
1638          */
1639         switch (w0->ls_cmd) {
1640         case FCNVME_LS_DISCONNECT_ASSOC:
1641                 ret = nvme_fc_ls_disconnect_assoc(lsop);
1642                 break;
1643         case FCNVME_LS_DISCONNECT_CONN:
1644                 lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1645                                 sizeof(*lsop->rspbuf), w0->ls_cmd,
1646                                 FCNVME_RJT_RC_UNSUP, FCNVME_RJT_EXP_NONE, 0);
1647                 break;
1648         case FCNVME_LS_CREATE_ASSOCIATION:
1649         case FCNVME_LS_CREATE_CONNECTION:
1650                 lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1651                                 sizeof(*lsop->rspbuf), w0->ls_cmd,
1652                                 FCNVME_RJT_RC_LOGIC, FCNVME_RJT_EXP_NONE, 0);
1653                 break;
1654         default:
1655                 lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1656                                 sizeof(*lsop->rspbuf), w0->ls_cmd,
1657                                 FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0);
1658                 break;
1659         }
1660
1661         return(ret);
1662 }
1663
1664 static void
1665 nvme_fc_handle_ls_rqst_work(struct work_struct *work)
1666 {
1667         struct nvme_fc_rport *rport =
1668                 container_of(work, struct nvme_fc_rport, lsrcv_work);
1669         struct fcnvme_ls_rqst_w0 *w0;
1670         struct nvmefc_ls_rcv_op *lsop;
1671         unsigned long flags;
1672         bool sendrsp;
1673
1674 restart:
1675         sendrsp = true;
1676         spin_lock_irqsave(&rport->lock, flags);
1677         list_for_each_entry(lsop, &rport->ls_rcv_list, lsrcv_list) {
1678                 if (lsop->handled)
1679                         continue;
1680
1681                 lsop->handled = true;
1682                 if (rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
1683                         spin_unlock_irqrestore(&rport->lock, flags);
1684                         sendrsp = nvme_fc_handle_ls_rqst(lsop);
1685                 } else {
1686                         spin_unlock_irqrestore(&rport->lock, flags);
1687                         w0 = &lsop->rqstbuf->w0;
1688                         lsop->lsrsp->rsplen = nvme_fc_format_rjt(
1689                                                 lsop->rspbuf,
1690                                                 sizeof(*lsop->rspbuf),
1691                                                 w0->ls_cmd,
1692                                                 FCNVME_RJT_RC_UNAB,
1693                                                 FCNVME_RJT_EXP_NONE, 0);
1694                 }
1695                 if (sendrsp)
1696                         nvme_fc_xmt_ls_rsp(lsop);
1697                 goto restart;
1698         }
1699         spin_unlock_irqrestore(&rport->lock, flags);
1700 }
1701
1702 /**
1703  * nvme_fc_rcv_ls_req - transport entry point called by an LLDD
1704  *                       upon the reception of a NVME LS request.
1705  *
1706  * The nvme-fc layer will copy payload to an internal structure for
1707  * processing.  As such, upon completion of the routine, the LLDD may
1708  * immediately free/reuse the LS request buffer passed in the call.
1709  *
1710  * If this routine returns error, the LLDD should abort the exchange.
1711  *
1712  * @portptr:    pointer to the (registered) remote port that the LS
1713  *              was received from. The remoteport is associated with
1714  *              a specific localport.
1715  * @lsrsp:      pointer to a nvmefc_ls_rsp response structure to be
1716  *              used to reference the exchange corresponding to the LS
1717  *              when issuing an ls response.
1718  * @lsreqbuf:   pointer to the buffer containing the LS Request
1719  * @lsreqbuf_len: length, in bytes, of the received LS request
1720  */
1721 int
1722 nvme_fc_rcv_ls_req(struct nvme_fc_remote_port *portptr,
1723                         struct nvmefc_ls_rsp *lsrsp,
1724                         void *lsreqbuf, u32 lsreqbuf_len)
1725 {
1726         struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
1727         struct nvme_fc_lport *lport = rport->lport;
1728         struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf;
1729         struct nvmefc_ls_rcv_op *lsop;
1730         unsigned long flags;
1731         int ret;
1732
1733         nvme_fc_rport_get(rport);
1734
1735         /* validate there's a routine to transmit a response */
1736         if (!lport->ops->xmt_ls_rsp) {
1737                 dev_info(lport->dev,
1738                         "RCV %s LS failed: no LLDD xmt_ls_rsp\n",
1739                         (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1740                                 nvmefc_ls_names[w0->ls_cmd] : "");
1741                 ret = -EINVAL;
1742                 goto out_put;
1743         }
1744
1745         if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) {
1746                 dev_info(lport->dev,
1747                         "RCV %s LS failed: payload too large\n",
1748                         (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1749                                 nvmefc_ls_names[w0->ls_cmd] : "");
1750                 ret = -E2BIG;
1751                 goto out_put;
1752         }
1753
1754         lsop = kzalloc(sizeof(*lsop) +
1755                         sizeof(union nvmefc_ls_requests) +
1756                         sizeof(union nvmefc_ls_responses),
1757                         GFP_KERNEL);
1758         if (!lsop) {
1759                 dev_info(lport->dev,
1760                         "RCV %s LS failed: No memory\n",
1761                         (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1762                                 nvmefc_ls_names[w0->ls_cmd] : "");
1763                 ret = -ENOMEM;
1764                 goto out_put;
1765         }
1766         lsop->rqstbuf = (union nvmefc_ls_requests *)&lsop[1];
1767         lsop->rspbuf = (union nvmefc_ls_responses *)&lsop->rqstbuf[1];
1768
1769         lsop->rspdma = fc_dma_map_single(lport->dev, lsop->rspbuf,
1770                                         sizeof(*lsop->rspbuf),
1771                                         DMA_TO_DEVICE);
1772         if (fc_dma_mapping_error(lport->dev, lsop->rspdma)) {
1773                 dev_info(lport->dev,
1774                         "RCV %s LS failed: DMA mapping failure\n",
1775                         (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1776                                 nvmefc_ls_names[w0->ls_cmd] : "");
1777                 ret = -EFAULT;
1778                 goto out_free;
1779         }
1780
1781         lsop->rport = rport;
1782         lsop->lsrsp = lsrsp;
1783
1784         memcpy(lsop->rqstbuf, lsreqbuf, lsreqbuf_len);
1785         lsop->rqstdatalen = lsreqbuf_len;
1786
1787         spin_lock_irqsave(&rport->lock, flags);
1788         if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE) {
1789                 spin_unlock_irqrestore(&rport->lock, flags);
1790                 ret = -ENOTCONN;
1791                 goto out_unmap;
1792         }
1793         list_add_tail(&lsop->lsrcv_list, &rport->ls_rcv_list);
1794         spin_unlock_irqrestore(&rport->lock, flags);
1795
1796         schedule_work(&rport->lsrcv_work);
1797
1798         return 0;
1799
1800 out_unmap:
1801         fc_dma_unmap_single(lport->dev, lsop->rspdma,
1802                         sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1803 out_free:
1804         kfree(lsop);
1805 out_put:
1806         nvme_fc_rport_put(rport);
1807         return ret;
1808 }
1809 EXPORT_SYMBOL_GPL(nvme_fc_rcv_ls_req);
1810
1811
1812 /* *********************** NVME Ctrl Routines **************************** */
1813
1814 static void
1815 __nvme_fc_exit_request(struct nvme_fc_ctrl *ctrl,
1816                 struct nvme_fc_fcp_op *op)
1817 {
1818         fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.rspdma,
1819                                 sizeof(op->rsp_iu), DMA_FROM_DEVICE);
1820         fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.cmddma,
1821                                 sizeof(op->cmd_iu), DMA_TO_DEVICE);
1822
1823         atomic_set(&op->state, FCPOP_STATE_UNINIT);
1824 }
1825
1826 static void
1827 nvme_fc_exit_request(struct blk_mq_tag_set *set, struct request *rq,
1828                 unsigned int hctx_idx)
1829 {
1830         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
1831
1832         return __nvme_fc_exit_request(set->driver_data, op);
1833 }
1834
1835 static int
1836 __nvme_fc_abort_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op)
1837 {
1838         unsigned long flags;
1839         int opstate;
1840
1841         spin_lock_irqsave(&ctrl->lock, flags);
1842         opstate = atomic_xchg(&op->state, FCPOP_STATE_ABORTED);
1843         if (opstate != FCPOP_STATE_ACTIVE)
1844                 atomic_set(&op->state, opstate);
1845         else if (test_bit(FCCTRL_TERMIO, &ctrl->flags)) {
1846                 op->flags |= FCOP_FLAGS_TERMIO;
1847                 ctrl->iocnt++;
1848         }
1849         spin_unlock_irqrestore(&ctrl->lock, flags);
1850
1851         if (opstate != FCPOP_STATE_ACTIVE)
1852                 return -ECANCELED;
1853
1854         ctrl->lport->ops->fcp_abort(&ctrl->lport->localport,
1855                                         &ctrl->rport->remoteport,
1856                                         op->queue->lldd_handle,
1857                                         &op->fcp_req);
1858
1859         return 0;
1860 }
1861
1862 static void
1863 nvme_fc_abort_aen_ops(struct nvme_fc_ctrl *ctrl)
1864 {
1865         struct nvme_fc_fcp_op *aen_op = ctrl->aen_ops;
1866         int i;
1867
1868         /* ensure we've initialized the ops once */
1869         if (!(aen_op->flags & FCOP_FLAGS_AEN))
1870                 return;
1871
1872         for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++)
1873                 __nvme_fc_abort_op(ctrl, aen_op);
1874 }
1875
1876 static inline void
1877 __nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl,
1878                 struct nvme_fc_fcp_op *op, int opstate)
1879 {
1880         unsigned long flags;
1881
1882         if (opstate == FCPOP_STATE_ABORTED) {
1883                 spin_lock_irqsave(&ctrl->lock, flags);
1884                 if (test_bit(FCCTRL_TERMIO, &ctrl->flags) &&
1885                     op->flags & FCOP_FLAGS_TERMIO) {
1886                         if (!--ctrl->iocnt)
1887                                 wake_up(&ctrl->ioabort_wait);
1888                 }
1889                 spin_unlock_irqrestore(&ctrl->lock, flags);
1890         }
1891 }
1892
1893 static void
1894 nvme_fc_ctrl_ioerr_work(struct work_struct *work)
1895 {
1896         struct nvme_fc_ctrl *ctrl =
1897                         container_of(work, struct nvme_fc_ctrl, ioerr_work);
1898
1899         nvme_fc_error_recovery(ctrl, "transport detected io error");
1900 }
1901
1902 static void
1903 nvme_fc_fcpio_done(struct nvmefc_fcp_req *req)
1904 {
1905         struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req);
1906         struct request *rq = op->rq;
1907         struct nvmefc_fcp_req *freq = &op->fcp_req;
1908         struct nvme_fc_ctrl *ctrl = op->ctrl;
1909         struct nvme_fc_queue *queue = op->queue;
1910         struct nvme_completion *cqe = &op->rsp_iu.cqe;
1911         struct nvme_command *sqe = &op->cmd_iu.sqe;
1912         __le16 status = cpu_to_le16(NVME_SC_SUCCESS << 1);
1913         union nvme_result result;
1914         bool terminate_assoc = true;
1915         int opstate;
1916
1917         /*
1918          * WARNING:
1919          * The current linux implementation of a nvme controller
1920          * allocates a single tag set for all io queues and sizes
1921          * the io queues to fully hold all possible tags. Thus, the
1922          * implementation does not reference or care about the sqhd
1923          * value as it never needs to use the sqhd/sqtail pointers
1924          * for submission pacing.
1925          *
1926          * This affects the FC-NVME implementation in two ways:
1927          * 1) As the value doesn't matter, we don't need to waste
1928          *    cycles extracting it from ERSPs and stamping it in the
1929          *    cases where the transport fabricates CQEs on successful
1930          *    completions.
1931          * 2) The FC-NVME implementation requires that delivery of
1932          *    ERSP completions are to go back to the nvme layer in order
1933          *    relative to the rsn, such that the sqhd value will always
1934          *    be "in order" for the nvme layer. As the nvme layer in
1935          *    linux doesn't care about sqhd, there's no need to return
1936          *    them in order.
1937          *
1938          * Additionally:
1939          * As the core nvme layer in linux currently does not look at
1940          * every field in the cqe - in cases where the FC transport must
1941          * fabricate a CQE, the following fields will not be set as they
1942          * are not referenced:
1943          *      cqe.sqid,  cqe.sqhd,  cqe.command_id
1944          *
1945          * Failure or error of an individual i/o, in a transport
1946          * detected fashion unrelated to the nvme completion status,
1947          * potentially cause the initiator and target sides to get out
1948          * of sync on SQ head/tail (aka outstanding io count allowed).
1949          * Per FC-NVME spec, failure of an individual command requires
1950          * the connection to be terminated, which in turn requires the
1951          * association to be terminated.
1952          */
1953
1954         opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
1955
1956         fc_dma_sync_single_for_cpu(ctrl->lport->dev, op->fcp_req.rspdma,
1957                                 sizeof(op->rsp_iu), DMA_FROM_DEVICE);
1958
1959         if (opstate == FCPOP_STATE_ABORTED)
1960                 status = cpu_to_le16(NVME_SC_HOST_ABORTED_CMD << 1);
1961         else if (freq->status) {
1962                 status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
1963                 dev_info(ctrl->ctrl.device,
1964                         "NVME-FC{%d}: io failed due to lldd error %d\n",
1965                         ctrl->cnum, freq->status);
1966         }
1967
1968         /*
1969          * For the linux implementation, if we have an unsuccesful
1970          * status, they blk-mq layer can typically be called with the
1971          * non-zero status and the content of the cqe isn't important.
1972          */
1973         if (status)
1974                 goto done;
1975
1976         /*
1977          * command completed successfully relative to the wire
1978          * protocol. However, validate anything received and
1979          * extract the status and result from the cqe (create it
1980          * where necessary).
1981          */
1982
1983         switch (freq->rcv_rsplen) {
1984
1985         case 0:
1986         case NVME_FC_SIZEOF_ZEROS_RSP:
1987                 /*
1988                  * No response payload or 12 bytes of payload (which
1989                  * should all be zeros) are considered successful and
1990                  * no payload in the CQE by the transport.
1991                  */
1992                 if (freq->transferred_length !=
1993                     be32_to_cpu(op->cmd_iu.data_len)) {
1994                         status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
1995                         dev_info(ctrl->ctrl.device,
1996                                 "NVME-FC{%d}: io failed due to bad transfer "
1997                                 "length: %d vs expected %d\n",
1998                                 ctrl->cnum, freq->transferred_length,
1999                                 be32_to_cpu(op->cmd_iu.data_len));
2000                         goto done;
2001                 }
2002                 result.u64 = 0;
2003                 break;
2004
2005         case sizeof(struct nvme_fc_ersp_iu):
2006                 /*
2007                  * The ERSP IU contains a full completion with CQE.
2008                  * Validate ERSP IU and look at cqe.
2009                  */
2010                 if (unlikely(be16_to_cpu(op->rsp_iu.iu_len) !=
2011                                         (freq->rcv_rsplen / 4) ||
2012                              be32_to_cpu(op->rsp_iu.xfrd_len) !=
2013                                         freq->transferred_length ||
2014                              op->rsp_iu.ersp_result ||
2015                              sqe->common.command_id != cqe->command_id)) {
2016                         status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2017                         dev_info(ctrl->ctrl.device,
2018                                 "NVME-FC{%d}: io failed due to bad NVMe_ERSP: "
2019                                 "iu len %d, xfr len %d vs %d, status code "
2020                                 "%d, cmdid %d vs %d\n",
2021                                 ctrl->cnum, be16_to_cpu(op->rsp_iu.iu_len),
2022                                 be32_to_cpu(op->rsp_iu.xfrd_len),
2023                                 freq->transferred_length,
2024                                 op->rsp_iu.ersp_result,
2025                                 sqe->common.command_id,
2026                                 cqe->command_id);
2027                         goto done;
2028                 }
2029                 result = cqe->result;
2030                 status = cqe->status;
2031                 break;
2032
2033         default:
2034                 status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2035                 dev_info(ctrl->ctrl.device,
2036                         "NVME-FC{%d}: io failed due to odd NVMe_xRSP iu "
2037                         "len %d\n",
2038                         ctrl->cnum, freq->rcv_rsplen);
2039                 goto done;
2040         }
2041
2042         terminate_assoc = false;
2043
2044 done:
2045         if (op->flags & FCOP_FLAGS_AEN) {
2046                 nvme_complete_async_event(&queue->ctrl->ctrl, status, &result);
2047                 __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2048                 atomic_set(&op->state, FCPOP_STATE_IDLE);
2049                 op->flags = FCOP_FLAGS_AEN;     /* clear other flags */
2050                 nvme_fc_ctrl_put(ctrl);
2051                 goto check_error;
2052         }
2053
2054         __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2055         if (!nvme_try_complete_req(rq, status, result))
2056                 nvme_fc_complete_rq(rq);
2057
2058 check_error:
2059         if (terminate_assoc && ctrl->ctrl.state != NVME_CTRL_RESETTING)
2060                 queue_work(nvme_reset_wq, &ctrl->ioerr_work);
2061 }
2062
2063 static int
2064 __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl,
2065                 struct nvme_fc_queue *queue, struct nvme_fc_fcp_op *op,
2066                 struct request *rq, u32 rqno)
2067 {
2068         struct nvme_fcp_op_w_sgl *op_w_sgl =
2069                 container_of(op, typeof(*op_w_sgl), op);
2070         struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2071         int ret = 0;
2072
2073         memset(op, 0, sizeof(*op));
2074         op->fcp_req.cmdaddr = &op->cmd_iu;
2075         op->fcp_req.cmdlen = sizeof(op->cmd_iu);
2076         op->fcp_req.rspaddr = &op->rsp_iu;
2077         op->fcp_req.rsplen = sizeof(op->rsp_iu);
2078         op->fcp_req.done = nvme_fc_fcpio_done;
2079         op->ctrl = ctrl;
2080         op->queue = queue;
2081         op->rq = rq;
2082         op->rqno = rqno;
2083
2084         cmdiu->format_id = NVME_CMD_FORMAT_ID;
2085         cmdiu->fc_id = NVME_CMD_FC_ID;
2086         cmdiu->iu_len = cpu_to_be16(sizeof(*cmdiu) / sizeof(u32));
2087         if (queue->qnum)
2088                 cmdiu->rsv_cat = fccmnd_set_cat_css(0,
2089                                         (NVME_CC_CSS_NVM >> NVME_CC_CSS_SHIFT));
2090         else
2091                 cmdiu->rsv_cat = fccmnd_set_cat_admin(0);
2092
2093         op->fcp_req.cmddma = fc_dma_map_single(ctrl->lport->dev,
2094                                 &op->cmd_iu, sizeof(op->cmd_iu), DMA_TO_DEVICE);
2095         if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.cmddma)) {
2096                 dev_err(ctrl->dev,
2097                         "FCP Op failed - cmdiu dma mapping failed.\n");
2098                 ret = -EFAULT;
2099                 goto out_on_error;
2100         }
2101
2102         op->fcp_req.rspdma = fc_dma_map_single(ctrl->lport->dev,
2103                                 &op->rsp_iu, sizeof(op->rsp_iu),
2104                                 DMA_FROM_DEVICE);
2105         if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.rspdma)) {
2106                 dev_err(ctrl->dev,
2107                         "FCP Op failed - rspiu dma mapping failed.\n");
2108                 ret = -EFAULT;
2109         }
2110
2111         atomic_set(&op->state, FCPOP_STATE_IDLE);
2112 out_on_error:
2113         return ret;
2114 }
2115
2116 static int
2117 nvme_fc_init_request(struct blk_mq_tag_set *set, struct request *rq,
2118                 unsigned int hctx_idx, unsigned int numa_node)
2119 {
2120         struct nvme_fc_ctrl *ctrl = set->driver_data;
2121         struct nvme_fcp_op_w_sgl *op = blk_mq_rq_to_pdu(rq);
2122         int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
2123         struct nvme_fc_queue *queue = &ctrl->queues[queue_idx];
2124         int res;
2125
2126         res = __nvme_fc_init_request(ctrl, queue, &op->op, rq, queue->rqcnt++);
2127         if (res)
2128                 return res;
2129         op->op.fcp_req.first_sgl = op->sgl;
2130         op->op.fcp_req.private = &op->priv[0];
2131         nvme_req(rq)->ctrl = &ctrl->ctrl;
2132         nvme_req(rq)->cmd = &op->op.cmd_iu.sqe;
2133         return res;
2134 }
2135
2136 static int
2137 nvme_fc_init_aen_ops(struct nvme_fc_ctrl *ctrl)
2138 {
2139         struct nvme_fc_fcp_op *aen_op;
2140         struct nvme_fc_cmd_iu *cmdiu;
2141         struct nvme_command *sqe;
2142         void *private = NULL;
2143         int i, ret;
2144
2145         aen_op = ctrl->aen_ops;
2146         for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2147                 if (ctrl->lport->ops->fcprqst_priv_sz) {
2148                         private = kzalloc(ctrl->lport->ops->fcprqst_priv_sz,
2149                                                 GFP_KERNEL);
2150                         if (!private)
2151                                 return -ENOMEM;
2152                 }
2153
2154                 cmdiu = &aen_op->cmd_iu;
2155                 sqe = &cmdiu->sqe;
2156                 ret = __nvme_fc_init_request(ctrl, &ctrl->queues[0],
2157                                 aen_op, (struct request *)NULL,
2158                                 (NVME_AQ_BLK_MQ_DEPTH + i));
2159                 if (ret) {
2160                         kfree(private);
2161                         return ret;
2162                 }
2163
2164                 aen_op->flags = FCOP_FLAGS_AEN;
2165                 aen_op->fcp_req.private = private;
2166
2167                 memset(sqe, 0, sizeof(*sqe));
2168                 sqe->common.opcode = nvme_admin_async_event;
2169                 /* Note: core layer may overwrite the sqe.command_id value */
2170                 sqe->common.command_id = NVME_AQ_BLK_MQ_DEPTH + i;
2171         }
2172         return 0;
2173 }
2174
2175 static void
2176 nvme_fc_term_aen_ops(struct nvme_fc_ctrl *ctrl)
2177 {
2178         struct nvme_fc_fcp_op *aen_op;
2179         int i;
2180
2181         cancel_work_sync(&ctrl->ctrl.async_event_work);
2182         aen_op = ctrl->aen_ops;
2183         for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2184                 __nvme_fc_exit_request(ctrl, aen_op);
2185
2186                 kfree(aen_op->fcp_req.private);
2187                 aen_op->fcp_req.private = NULL;
2188         }
2189 }
2190
2191 static inline void
2192 __nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, struct nvme_fc_ctrl *ctrl,
2193                 unsigned int qidx)
2194 {
2195         struct nvme_fc_queue *queue = &ctrl->queues[qidx];
2196
2197         hctx->driver_data = queue;
2198         queue->hctx = hctx;
2199 }
2200
2201 static int
2202 nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
2203                 unsigned int hctx_idx)
2204 {
2205         struct nvme_fc_ctrl *ctrl = data;
2206
2207         __nvme_fc_init_hctx(hctx, ctrl, hctx_idx + 1);
2208
2209         return 0;
2210 }
2211
2212 static int
2213 nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
2214                 unsigned int hctx_idx)
2215 {
2216         struct nvme_fc_ctrl *ctrl = data;
2217
2218         __nvme_fc_init_hctx(hctx, ctrl, hctx_idx);
2219
2220         return 0;
2221 }
2222
2223 static void
2224 nvme_fc_init_queue(struct nvme_fc_ctrl *ctrl, int idx)
2225 {
2226         struct nvme_fc_queue *queue;
2227
2228         queue = &ctrl->queues[idx];
2229         memset(queue, 0, sizeof(*queue));
2230         queue->ctrl = ctrl;
2231         queue->qnum = idx;
2232         atomic_set(&queue->csn, 0);
2233         queue->dev = ctrl->dev;
2234
2235         if (idx > 0)
2236                 queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
2237         else
2238                 queue->cmnd_capsule_len = sizeof(struct nvme_command);
2239
2240         /*
2241          * Considered whether we should allocate buffers for all SQEs
2242          * and CQEs and dma map them - mapping their respective entries
2243          * into the request structures (kernel vm addr and dma address)
2244          * thus the driver could use the buffers/mappings directly.
2245          * It only makes sense if the LLDD would use them for its
2246          * messaging api. It's very unlikely most adapter api's would use
2247          * a native NVME sqe/cqe. More reasonable if FC-NVME IU payload
2248          * structures were used instead.
2249          */
2250 }
2251
2252 /*
2253  * This routine terminates a queue at the transport level.
2254  * The transport has already ensured that all outstanding ios on
2255  * the queue have been terminated.
2256  * The transport will send a Disconnect LS request to terminate
2257  * the queue's connection. Termination of the admin queue will also
2258  * terminate the association at the target.
2259  */
2260 static void
2261 nvme_fc_free_queue(struct nvme_fc_queue *queue)
2262 {
2263         if (!test_and_clear_bit(NVME_FC_Q_CONNECTED, &queue->flags))
2264                 return;
2265
2266         clear_bit(NVME_FC_Q_LIVE, &queue->flags);
2267         /*
2268          * Current implementation never disconnects a single queue.
2269          * It always terminates a whole association. So there is never
2270          * a disconnect(queue) LS sent to the target.
2271          */
2272
2273         queue->connection_id = 0;
2274         atomic_set(&queue->csn, 0);
2275 }
2276
2277 static void
2278 __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *ctrl,
2279         struct nvme_fc_queue *queue, unsigned int qidx)
2280 {
2281         if (ctrl->lport->ops->delete_queue)
2282                 ctrl->lport->ops->delete_queue(&ctrl->lport->localport, qidx,
2283                                 queue->lldd_handle);
2284         queue->lldd_handle = NULL;
2285 }
2286
2287 static void
2288 nvme_fc_free_io_queues(struct nvme_fc_ctrl *ctrl)
2289 {
2290         int i;
2291
2292         for (i = 1; i < ctrl->ctrl.queue_count; i++)
2293                 nvme_fc_free_queue(&ctrl->queues[i]);
2294 }
2295
2296 static int
2297 __nvme_fc_create_hw_queue(struct nvme_fc_ctrl *ctrl,
2298         struct nvme_fc_queue *queue, unsigned int qidx, u16 qsize)
2299 {
2300         int ret = 0;
2301
2302         queue->lldd_handle = NULL;
2303         if (ctrl->lport->ops->create_queue)
2304                 ret = ctrl->lport->ops->create_queue(&ctrl->lport->localport,
2305                                 qidx, qsize, &queue->lldd_handle);
2306
2307         return ret;
2308 }
2309
2310 static void
2311 nvme_fc_delete_hw_io_queues(struct nvme_fc_ctrl *ctrl)
2312 {
2313         struct nvme_fc_queue *queue = &ctrl->queues[ctrl->ctrl.queue_count - 1];
2314         int i;
2315
2316         for (i = ctrl->ctrl.queue_count - 1; i >= 1; i--, queue--)
2317                 __nvme_fc_delete_hw_queue(ctrl, queue, i);
2318 }
2319
2320 static int
2321 nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2322 {
2323         struct nvme_fc_queue *queue = &ctrl->queues[1];
2324         int i, ret;
2325
2326         for (i = 1; i < ctrl->ctrl.queue_count; i++, queue++) {
2327                 ret = __nvme_fc_create_hw_queue(ctrl, queue, i, qsize);
2328                 if (ret)
2329                         goto delete_queues;
2330         }
2331
2332         return 0;
2333
2334 delete_queues:
2335         for (; i > 0; i--)
2336                 __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[i], i);
2337         return ret;
2338 }
2339
2340 static int
2341 nvme_fc_connect_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2342 {
2343         int i, ret = 0;
2344
2345         for (i = 1; i < ctrl->ctrl.queue_count; i++) {
2346                 ret = nvme_fc_connect_queue(ctrl, &ctrl->queues[i], qsize,
2347                                         (qsize / 5));
2348                 if (ret)
2349                         break;
2350                 ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
2351                 if (ret)
2352                         break;
2353
2354                 set_bit(NVME_FC_Q_LIVE, &ctrl->queues[i].flags);
2355         }
2356
2357         return ret;
2358 }
2359
2360 static void
2361 nvme_fc_init_io_queues(struct nvme_fc_ctrl *ctrl)
2362 {
2363         int i;
2364
2365         for (i = 1; i < ctrl->ctrl.queue_count; i++)
2366                 nvme_fc_init_queue(ctrl, i);
2367 }
2368
2369 static void
2370 nvme_fc_ctrl_free(struct kref *ref)
2371 {
2372         struct nvme_fc_ctrl *ctrl =
2373                 container_of(ref, struct nvme_fc_ctrl, ref);
2374         unsigned long flags;
2375
2376         if (ctrl->ctrl.tagset) {
2377                 blk_cleanup_queue(ctrl->ctrl.connect_q);
2378                 blk_mq_free_tag_set(&ctrl->tag_set);
2379         }
2380
2381         /* remove from rport list */
2382         spin_lock_irqsave(&ctrl->rport->lock, flags);
2383         list_del(&ctrl->ctrl_list);
2384         spin_unlock_irqrestore(&ctrl->rport->lock, flags);
2385
2386         nvme_start_admin_queue(&ctrl->ctrl);
2387         blk_cleanup_queue(ctrl->ctrl.admin_q);
2388         blk_cleanup_queue(ctrl->ctrl.fabrics_q);
2389         blk_mq_free_tag_set(&ctrl->admin_tag_set);
2390
2391         kfree(ctrl->queues);
2392
2393         put_device(ctrl->dev);
2394         nvme_fc_rport_put(ctrl->rport);
2395
2396         ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum);
2397         if (ctrl->ctrl.opts)
2398                 nvmf_free_options(ctrl->ctrl.opts);
2399         kfree(ctrl);
2400 }
2401
2402 static void
2403 nvme_fc_ctrl_put(struct nvme_fc_ctrl *ctrl)
2404 {
2405         kref_put(&ctrl->ref, nvme_fc_ctrl_free);
2406 }
2407
2408 static int
2409 nvme_fc_ctrl_get(struct nvme_fc_ctrl *ctrl)
2410 {
2411         return kref_get_unless_zero(&ctrl->ref);
2412 }
2413
2414 /*
2415  * All accesses from nvme core layer done - can now free the
2416  * controller. Called after last nvme_put_ctrl() call
2417  */
2418 static void
2419 nvme_fc_nvme_ctrl_freed(struct nvme_ctrl *nctrl)
2420 {
2421         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
2422
2423         WARN_ON(nctrl != &ctrl->ctrl);
2424
2425         nvme_fc_ctrl_put(ctrl);
2426 }
2427
2428 /*
2429  * This routine is used by the transport when it needs to find active
2430  * io on a queue that is to be terminated. The transport uses
2431  * blk_mq_tagset_busy_itr() to find the busy requests, which then invoke
2432  * this routine to kill them on a 1 by 1 basis.
2433  *
2434  * As FC allocates FC exchange for each io, the transport must contact
2435  * the LLDD to terminate the exchange, thus releasing the FC exchange.
2436  * After terminating the exchange the LLDD will call the transport's
2437  * normal io done path for the request, but it will have an aborted
2438  * status. The done path will return the io request back to the block
2439  * layer with an error status.
2440  */
2441 static bool
2442 nvme_fc_terminate_exchange(struct request *req, void *data, bool reserved)
2443 {
2444         struct nvme_ctrl *nctrl = data;
2445         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
2446         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(req);
2447
2448         op->nreq.flags |= NVME_REQ_CANCELLED;
2449         __nvme_fc_abort_op(ctrl, op);
2450         return true;
2451 }
2452
2453 /*
2454  * This routine runs through all outstanding commands on the association
2455  * and aborts them.  This routine is typically be called by the
2456  * delete_association routine. It is also called due to an error during
2457  * reconnect. In that scenario, it is most likely a command that initializes
2458  * the controller, including fabric Connect commands on io queues, that
2459  * may have timed out or failed thus the io must be killed for the connect
2460  * thread to see the error.
2461  */
2462 static void
2463 __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues)
2464 {
2465         int q;
2466
2467         /*
2468          * if aborting io, the queues are no longer good, mark them
2469          * all as not live.
2470          */
2471         if (ctrl->ctrl.queue_count > 1) {
2472                 for (q = 1; q < ctrl->ctrl.queue_count; q++)
2473                         clear_bit(NVME_FC_Q_LIVE, &ctrl->queues[q].flags);
2474         }
2475         clear_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags);
2476
2477         /*
2478          * If io queues are present, stop them and terminate all outstanding
2479          * ios on them. As FC allocates FC exchange for each io, the
2480          * transport must contact the LLDD to terminate the exchange,
2481          * thus releasing the FC exchange. We use blk_mq_tagset_busy_itr()
2482          * to tell us what io's are busy and invoke a transport routine
2483          * to kill them with the LLDD.  After terminating the exchange
2484          * the LLDD will call the transport's normal io done path, but it
2485          * will have an aborted status. The done path will return the
2486          * io requests back to the block layer as part of normal completions
2487          * (but with error status).
2488          */
2489         if (ctrl->ctrl.queue_count > 1) {
2490                 nvme_stop_queues(&ctrl->ctrl);
2491                 nvme_sync_io_queues(&ctrl->ctrl);
2492                 blk_mq_tagset_busy_iter(&ctrl->tag_set,
2493                                 nvme_fc_terminate_exchange, &ctrl->ctrl);
2494                 blk_mq_tagset_wait_completed_request(&ctrl->tag_set);
2495                 if (start_queues)
2496                         nvme_start_queues(&ctrl->ctrl);
2497         }
2498
2499         /*
2500          * Other transports, which don't have link-level contexts bound
2501          * to sqe's, would try to gracefully shutdown the controller by
2502          * writing the registers for shutdown and polling (call
2503          * nvme_shutdown_ctrl()). Given a bunch of i/o was potentially
2504          * just aborted and we will wait on those contexts, and given
2505          * there was no indication of how live the controlelr is on the
2506          * link, don't send more io to create more contexts for the
2507          * shutdown. Let the controller fail via keepalive failure if
2508          * its still present.
2509          */
2510
2511         /*
2512          * clean up the admin queue. Same thing as above.
2513          */
2514         nvme_stop_admin_queue(&ctrl->ctrl);
2515         blk_sync_queue(ctrl->ctrl.admin_q);
2516         blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
2517                                 nvme_fc_terminate_exchange, &ctrl->ctrl);
2518         blk_mq_tagset_wait_completed_request(&ctrl->admin_tag_set);
2519 }
2520
2521 static void
2522 nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg)
2523 {
2524         /*
2525          * if an error (io timeout, etc) while (re)connecting, the remote
2526          * port requested terminating of the association (disconnect_ls)
2527          * or an error (timeout or abort) occurred on an io while creating
2528          * the controller.  Abort any ios on the association and let the
2529          * create_association error path resolve things.
2530          */
2531         if (ctrl->ctrl.state == NVME_CTRL_CONNECTING) {
2532                 __nvme_fc_abort_outstanding_ios(ctrl, true);
2533                 set_bit(ASSOC_FAILED, &ctrl->flags);
2534                 return;
2535         }
2536
2537         /* Otherwise, only proceed if in LIVE state - e.g. on first error */
2538         if (ctrl->ctrl.state != NVME_CTRL_LIVE)
2539                 return;
2540
2541         dev_warn(ctrl->ctrl.device,
2542                 "NVME-FC{%d}: transport association event: %s\n",
2543                 ctrl->cnum, errmsg);
2544         dev_warn(ctrl->ctrl.device,
2545                 "NVME-FC{%d}: resetting controller\n", ctrl->cnum);
2546
2547         nvme_reset_ctrl(&ctrl->ctrl);
2548 }
2549
2550 static enum blk_eh_timer_return
2551 nvme_fc_timeout(struct request *rq, bool reserved)
2552 {
2553         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2554         struct nvme_fc_ctrl *ctrl = op->ctrl;
2555         struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2556         struct nvme_command *sqe = &cmdiu->sqe;
2557
2558         /*
2559          * Attempt to abort the offending command. Command completion
2560          * will detect the aborted io and will fail the connection.
2561          */
2562         dev_info(ctrl->ctrl.device,
2563                 "NVME-FC{%d.%d}: io timeout: opcode %d fctype %d w10/11: "
2564                 "x%08x/x%08x\n",
2565                 ctrl->cnum, op->queue->qnum, sqe->common.opcode,
2566                 sqe->connect.fctype, sqe->common.cdw10, sqe->common.cdw11);
2567         if (__nvme_fc_abort_op(ctrl, op))
2568                 nvme_fc_error_recovery(ctrl, "io timeout abort failed");
2569
2570         /*
2571          * the io abort has been initiated. Have the reset timer
2572          * restarted and the abort completion will complete the io
2573          * shortly. Avoids a synchronous wait while the abort finishes.
2574          */
2575         return BLK_EH_RESET_TIMER;
2576 }
2577
2578 static int
2579 nvme_fc_map_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2580                 struct nvme_fc_fcp_op *op)
2581 {
2582         struct nvmefc_fcp_req *freq = &op->fcp_req;
2583         int ret;
2584
2585         freq->sg_cnt = 0;
2586
2587         if (!blk_rq_nr_phys_segments(rq))
2588                 return 0;
2589
2590         freq->sg_table.sgl = freq->first_sgl;
2591         ret = sg_alloc_table_chained(&freq->sg_table,
2592                         blk_rq_nr_phys_segments(rq), freq->sg_table.sgl,
2593                         NVME_INLINE_SG_CNT);
2594         if (ret)
2595                 return -ENOMEM;
2596
2597         op->nents = blk_rq_map_sg(rq->q, rq, freq->sg_table.sgl);
2598         WARN_ON(op->nents > blk_rq_nr_phys_segments(rq));
2599         freq->sg_cnt = fc_dma_map_sg(ctrl->lport->dev, freq->sg_table.sgl,
2600                                 op->nents, rq_dma_dir(rq));
2601         if (unlikely(freq->sg_cnt <= 0)) {
2602                 sg_free_table_chained(&freq->sg_table, NVME_INLINE_SG_CNT);
2603                 freq->sg_cnt = 0;
2604                 return -EFAULT;
2605         }
2606
2607         /*
2608          * TODO: blk_integrity_rq(rq)  for DIF
2609          */
2610         return 0;
2611 }
2612
2613 static void
2614 nvme_fc_unmap_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2615                 struct nvme_fc_fcp_op *op)
2616 {
2617         struct nvmefc_fcp_req *freq = &op->fcp_req;
2618
2619         if (!freq->sg_cnt)
2620                 return;
2621
2622         fc_dma_unmap_sg(ctrl->lport->dev, freq->sg_table.sgl, op->nents,
2623                         rq_dma_dir(rq));
2624
2625         sg_free_table_chained(&freq->sg_table, NVME_INLINE_SG_CNT);
2626
2627         freq->sg_cnt = 0;
2628 }
2629
2630 /*
2631  * In FC, the queue is a logical thing. At transport connect, the target
2632  * creates its "queue" and returns a handle that is to be given to the
2633  * target whenever it posts something to the corresponding SQ.  When an
2634  * SQE is sent on a SQ, FC effectively considers the SQE, or rather the
2635  * command contained within the SQE, an io, and assigns a FC exchange
2636  * to it. The SQE and the associated SQ handle are sent in the initial
2637  * CMD IU sents on the exchange. All transfers relative to the io occur
2638  * as part of the exchange.  The CQE is the last thing for the io,
2639  * which is transferred (explicitly or implicitly) with the RSP IU
2640  * sent on the exchange. After the CQE is received, the FC exchange is
2641  * terminaed and the Exchange may be used on a different io.
2642  *
2643  * The transport to LLDD api has the transport making a request for a
2644  * new fcp io request to the LLDD. The LLDD then allocates a FC exchange
2645  * resource and transfers the command. The LLDD will then process all
2646  * steps to complete the io. Upon completion, the transport done routine
2647  * is called.
2648  *
2649  * So - while the operation is outstanding to the LLDD, there is a link
2650  * level FC exchange resource that is also outstanding. This must be
2651  * considered in all cleanup operations.
2652  */
2653 static blk_status_t
2654 nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
2655         struct nvme_fc_fcp_op *op, u32 data_len,
2656         enum nvmefc_fcp_datadir io_dir)
2657 {
2658         struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2659         struct nvme_command *sqe = &cmdiu->sqe;
2660         int ret, opstate;
2661
2662         /*
2663          * before attempting to send the io, check to see if we believe
2664          * the target device is present
2665          */
2666         if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
2667                 return BLK_STS_RESOURCE;
2668
2669         if (!nvme_fc_ctrl_get(ctrl))
2670                 return BLK_STS_IOERR;
2671
2672         /* format the FC-NVME CMD IU and fcp_req */
2673         cmdiu->connection_id = cpu_to_be64(queue->connection_id);
2674         cmdiu->data_len = cpu_to_be32(data_len);
2675         switch (io_dir) {
2676         case NVMEFC_FCP_WRITE:
2677                 cmdiu->flags = FCNVME_CMD_FLAGS_WRITE;
2678                 break;
2679         case NVMEFC_FCP_READ:
2680                 cmdiu->flags = FCNVME_CMD_FLAGS_READ;
2681                 break;
2682         case NVMEFC_FCP_NODATA:
2683                 cmdiu->flags = 0;
2684                 break;
2685         }
2686         op->fcp_req.payload_length = data_len;
2687         op->fcp_req.io_dir = io_dir;
2688         op->fcp_req.transferred_length = 0;
2689         op->fcp_req.rcv_rsplen = 0;
2690         op->fcp_req.status = NVME_SC_SUCCESS;
2691         op->fcp_req.sqid = cpu_to_le16(queue->qnum);
2692
2693         /*
2694          * validate per fabric rules, set fields mandated by fabric spec
2695          * as well as those by FC-NVME spec.
2696          */
2697         WARN_ON_ONCE(sqe->common.metadata);
2698         sqe->common.flags |= NVME_CMD_SGL_METABUF;
2699
2700         /*
2701          * format SQE DPTR field per FC-NVME rules:
2702          *    type=0x5     Transport SGL Data Block Descriptor
2703          *    subtype=0xA  Transport-specific value
2704          *    address=0
2705          *    length=length of the data series
2706          */
2707         sqe->rw.dptr.sgl.type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2708                                         NVME_SGL_FMT_TRANSPORT_A;
2709         sqe->rw.dptr.sgl.length = cpu_to_le32(data_len);
2710         sqe->rw.dptr.sgl.addr = 0;
2711
2712         if (!(op->flags & FCOP_FLAGS_AEN)) {
2713                 ret = nvme_fc_map_data(ctrl, op->rq, op);
2714                 if (ret < 0) {
2715                         nvme_cleanup_cmd(op->rq);
2716                         nvme_fc_ctrl_put(ctrl);
2717                         if (ret == -ENOMEM || ret == -EAGAIN)
2718                                 return BLK_STS_RESOURCE;
2719                         return BLK_STS_IOERR;
2720                 }
2721         }
2722
2723         fc_dma_sync_single_for_device(ctrl->lport->dev, op->fcp_req.cmddma,
2724                                   sizeof(op->cmd_iu), DMA_TO_DEVICE);
2725
2726         atomic_set(&op->state, FCPOP_STATE_ACTIVE);
2727
2728         if (!(op->flags & FCOP_FLAGS_AEN))
2729                 blk_mq_start_request(op->rq);
2730
2731         cmdiu->csn = cpu_to_be32(atomic_inc_return(&queue->csn));
2732         ret = ctrl->lport->ops->fcp_io(&ctrl->lport->localport,
2733                                         &ctrl->rport->remoteport,
2734                                         queue->lldd_handle, &op->fcp_req);
2735
2736         if (ret) {
2737                 /*
2738                  * If the lld fails to send the command is there an issue with
2739                  * the csn value?  If the command that fails is the Connect,
2740                  * no - as the connection won't be live.  If it is a command
2741                  * post-connect, it's possible a gap in csn may be created.
2742                  * Does this matter?  As Linux initiators don't send fused
2743                  * commands, no.  The gap would exist, but as there's nothing
2744                  * that depends on csn order to be delivered on the target
2745                  * side, it shouldn't hurt.  It would be difficult for a
2746                  * target to even detect the csn gap as it has no idea when the
2747                  * cmd with the csn was supposed to arrive.
2748                  */
2749                 opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
2750                 __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2751
2752                 if (!(op->flags & FCOP_FLAGS_AEN)) {
2753                         nvme_fc_unmap_data(ctrl, op->rq, op);
2754                         nvme_cleanup_cmd(op->rq);
2755                 }
2756
2757                 nvme_fc_ctrl_put(ctrl);
2758
2759                 if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE &&
2760                                 ret != -EBUSY)
2761                         return BLK_STS_IOERR;
2762
2763                 return BLK_STS_RESOURCE;
2764         }
2765
2766         return BLK_STS_OK;
2767 }
2768
2769 static blk_status_t
2770 nvme_fc_queue_rq(struct blk_mq_hw_ctx *hctx,
2771                         const struct blk_mq_queue_data *bd)
2772 {
2773         struct nvme_ns *ns = hctx->queue->queuedata;
2774         struct nvme_fc_queue *queue = hctx->driver_data;
2775         struct nvme_fc_ctrl *ctrl = queue->ctrl;
2776         struct request *rq = bd->rq;
2777         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2778         enum nvmefc_fcp_datadir io_dir;
2779         bool queue_ready = test_bit(NVME_FC_Q_LIVE, &queue->flags);
2780         u32 data_len;
2781         blk_status_t ret;
2782
2783         if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE ||
2784             !nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2785                 return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq);
2786
2787         ret = nvme_setup_cmd(ns, rq);
2788         if (ret)
2789                 return ret;
2790
2791         /*
2792          * nvme core doesn't quite treat the rq opaquely. Commands such
2793          * as WRITE ZEROES will return a non-zero rq payload_bytes yet
2794          * there is no actual payload to be transferred.
2795          * To get it right, key data transmission on there being 1 or
2796          * more physical segments in the sg list. If there is no
2797          * physical segments, there is no payload.
2798          */
2799         if (blk_rq_nr_phys_segments(rq)) {
2800                 data_len = blk_rq_payload_bytes(rq);
2801                 io_dir = ((rq_data_dir(rq) == WRITE) ?
2802                                         NVMEFC_FCP_WRITE : NVMEFC_FCP_READ);
2803         } else {
2804                 data_len = 0;
2805                 io_dir = NVMEFC_FCP_NODATA;
2806         }
2807
2808
2809         return nvme_fc_start_fcp_op(ctrl, queue, op, data_len, io_dir);
2810 }
2811
2812 static void
2813 nvme_fc_submit_async_event(struct nvme_ctrl *arg)
2814 {
2815         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(arg);
2816         struct nvme_fc_fcp_op *aen_op;
2817         blk_status_t ret;
2818
2819         if (test_bit(FCCTRL_TERMIO, &ctrl->flags))
2820                 return;
2821
2822         aen_op = &ctrl->aen_ops[0];
2823
2824         ret = nvme_fc_start_fcp_op(ctrl, aen_op->queue, aen_op, 0,
2825                                         NVMEFC_FCP_NODATA);
2826         if (ret)
2827                 dev_err(ctrl->ctrl.device,
2828                         "failed async event work\n");
2829 }
2830
2831 static void
2832 nvme_fc_complete_rq(struct request *rq)
2833 {
2834         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2835         struct nvme_fc_ctrl *ctrl = op->ctrl;
2836
2837         atomic_set(&op->state, FCPOP_STATE_IDLE);
2838         op->flags &= ~FCOP_FLAGS_TERMIO;
2839
2840         nvme_fc_unmap_data(ctrl, rq, op);
2841         nvme_complete_rq(rq);
2842         nvme_fc_ctrl_put(ctrl);
2843 }
2844
2845 static int nvme_fc_map_queues(struct blk_mq_tag_set *set)
2846 {
2847         struct nvme_fc_ctrl *ctrl = set->driver_data;
2848         int i;
2849
2850         for (i = 0; i < set->nr_maps; i++) {
2851                 struct blk_mq_queue_map *map = &set->map[i];
2852
2853                 if (!map->nr_queues) {
2854                         WARN_ON(i == HCTX_TYPE_DEFAULT);
2855                         continue;
2856                 }
2857
2858                 /* Call LLDD map queue functionality if defined */
2859                 if (ctrl->lport->ops->map_queues)
2860                         ctrl->lport->ops->map_queues(&ctrl->lport->localport,
2861                                                      map);
2862                 else
2863                         blk_mq_map_queues(map);
2864         }
2865         return 0;
2866 }
2867
2868 static const struct blk_mq_ops nvme_fc_mq_ops = {
2869         .queue_rq       = nvme_fc_queue_rq,
2870         .complete       = nvme_fc_complete_rq,
2871         .init_request   = nvme_fc_init_request,
2872         .exit_request   = nvme_fc_exit_request,
2873         .init_hctx      = nvme_fc_init_hctx,
2874         .timeout        = nvme_fc_timeout,
2875         .map_queues     = nvme_fc_map_queues,
2876 };
2877
2878 static int
2879 nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl)
2880 {
2881         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2882         unsigned int nr_io_queues;
2883         int ret;
2884
2885         nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),
2886                                 ctrl->lport->ops->max_hw_queues);
2887         ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
2888         if (ret) {
2889                 dev_info(ctrl->ctrl.device,
2890                         "set_queue_count failed: %d\n", ret);
2891                 return ret;
2892         }
2893
2894         ctrl->ctrl.queue_count = nr_io_queues + 1;
2895         if (!nr_io_queues)
2896                 return 0;
2897
2898         nvme_fc_init_io_queues(ctrl);
2899
2900         memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set));
2901         ctrl->tag_set.ops = &nvme_fc_mq_ops;
2902         ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size;
2903         ctrl->tag_set.reserved_tags = NVMF_RESERVED_TAGS;
2904         ctrl->tag_set.numa_node = ctrl->ctrl.numa_node;
2905         ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
2906         ctrl->tag_set.cmd_size =
2907                 struct_size((struct nvme_fcp_op_w_sgl *)NULL, priv,
2908                             ctrl->lport->ops->fcprqst_priv_sz);
2909         ctrl->tag_set.driver_data = ctrl;
2910         ctrl->tag_set.nr_hw_queues = ctrl->ctrl.queue_count - 1;
2911         ctrl->tag_set.timeout = NVME_IO_TIMEOUT;
2912
2913         ret = blk_mq_alloc_tag_set(&ctrl->tag_set);
2914         if (ret)
2915                 return ret;
2916
2917         ctrl->ctrl.tagset = &ctrl->tag_set;
2918
2919         ret = nvme_ctrl_init_connect_q(&(ctrl->ctrl));
2920         if (ret)
2921                 goto out_free_tag_set;
2922
2923         ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2924         if (ret)
2925                 goto out_cleanup_blk_queue;
2926
2927         ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2928         if (ret)
2929                 goto out_delete_hw_queues;
2930
2931         ctrl->ioq_live = true;
2932
2933         return 0;
2934
2935 out_delete_hw_queues:
2936         nvme_fc_delete_hw_io_queues(ctrl);
2937 out_cleanup_blk_queue:
2938         blk_cleanup_queue(ctrl->ctrl.connect_q);
2939 out_free_tag_set:
2940         blk_mq_free_tag_set(&ctrl->tag_set);
2941         nvme_fc_free_io_queues(ctrl);
2942
2943         /* force put free routine to ignore io queues */
2944         ctrl->ctrl.tagset = NULL;
2945
2946         return ret;
2947 }
2948
2949 static int
2950 nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl)
2951 {
2952         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2953         u32 prior_ioq_cnt = ctrl->ctrl.queue_count - 1;
2954         unsigned int nr_io_queues;
2955         int ret;
2956
2957         nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),
2958                                 ctrl->lport->ops->max_hw_queues);
2959         ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
2960         if (ret) {
2961                 dev_info(ctrl->ctrl.device,
2962                         "set_queue_count failed: %d\n", ret);
2963                 return ret;
2964         }
2965
2966         if (!nr_io_queues && prior_ioq_cnt) {
2967                 dev_info(ctrl->ctrl.device,
2968                         "Fail Reconnect: At least 1 io queue "
2969                         "required (was %d)\n", prior_ioq_cnt);
2970                 return -ENOSPC;
2971         }
2972
2973         ctrl->ctrl.queue_count = nr_io_queues + 1;
2974         /* check for io queues existing */
2975         if (ctrl->ctrl.queue_count == 1)
2976                 return 0;
2977
2978         if (prior_ioq_cnt != nr_io_queues) {
2979                 dev_info(ctrl->ctrl.device,
2980                         "reconnect: revising io queue count from %d to %d\n",
2981                         prior_ioq_cnt, nr_io_queues);
2982                 blk_mq_update_nr_hw_queues(&ctrl->tag_set, nr_io_queues);
2983         }
2984
2985         ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2986         if (ret)
2987                 goto out_free_io_queues;
2988
2989         ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2990         if (ret)
2991                 goto out_delete_hw_queues;
2992
2993         return 0;
2994
2995 out_delete_hw_queues:
2996         nvme_fc_delete_hw_io_queues(ctrl);
2997 out_free_io_queues:
2998         nvme_fc_free_io_queues(ctrl);
2999         return ret;
3000 }
3001
3002 static void
3003 nvme_fc_rport_active_on_lport(struct nvme_fc_rport *rport)
3004 {
3005         struct nvme_fc_lport *lport = rport->lport;
3006
3007         atomic_inc(&lport->act_rport_cnt);
3008 }
3009
3010 static void
3011 nvme_fc_rport_inactive_on_lport(struct nvme_fc_rport *rport)
3012 {
3013         struct nvme_fc_lport *lport = rport->lport;
3014         u32 cnt;
3015
3016         cnt = atomic_dec_return(&lport->act_rport_cnt);
3017         if (cnt == 0 && lport->localport.port_state == FC_OBJSTATE_DELETED)
3018                 lport->ops->localport_delete(&lport->localport);
3019 }
3020
3021 static int
3022 nvme_fc_ctlr_active_on_rport(struct nvme_fc_ctrl *ctrl)
3023 {
3024         struct nvme_fc_rport *rport = ctrl->rport;
3025         u32 cnt;
3026
3027         if (test_and_set_bit(ASSOC_ACTIVE, &ctrl->flags))
3028                 return 1;
3029
3030         cnt = atomic_inc_return(&rport->act_ctrl_cnt);
3031         if (cnt == 1)
3032                 nvme_fc_rport_active_on_lport(rport);
3033
3034         return 0;
3035 }
3036
3037 static int
3038 nvme_fc_ctlr_inactive_on_rport(struct nvme_fc_ctrl *ctrl)
3039 {
3040         struct nvme_fc_rport *rport = ctrl->rport;
3041         struct nvme_fc_lport *lport = rport->lport;
3042         u32 cnt;
3043
3044         /* clearing of ctrl->flags ASSOC_ACTIVE bit is in association delete */
3045
3046         cnt = atomic_dec_return(&rport->act_ctrl_cnt);
3047         if (cnt == 0) {
3048                 if (rport->remoteport.port_state == FC_OBJSTATE_DELETED)
3049                         lport->ops->remoteport_delete(&rport->remoteport);
3050                 nvme_fc_rport_inactive_on_lport(rport);
3051         }
3052
3053         return 0;
3054 }
3055
3056 /*
3057  * This routine restarts the controller on the host side, and
3058  * on the link side, recreates the controller association.
3059  */
3060 static int
3061 nvme_fc_create_association(struct nvme_fc_ctrl *ctrl)
3062 {
3063         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
3064         struct nvmefc_ls_rcv_op *disls = NULL;
3065         unsigned long flags;
3066         int ret;
3067         bool changed;
3068
3069         ++ctrl->ctrl.nr_reconnects;
3070
3071         if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
3072                 return -ENODEV;
3073
3074         if (nvme_fc_ctlr_active_on_rport(ctrl))
3075                 return -ENOTUNIQ;
3076
3077         dev_info(ctrl->ctrl.device,
3078                 "NVME-FC{%d}: create association : host wwpn 0x%016llx "
3079                 " rport wwpn 0x%016llx: NQN \"%s\"\n",
3080                 ctrl->cnum, ctrl->lport->localport.port_name,
3081                 ctrl->rport->remoteport.port_name, ctrl->ctrl.opts->subsysnqn);
3082
3083         clear_bit(ASSOC_FAILED, &ctrl->flags);
3084
3085         /*
3086          * Create the admin queue
3087          */
3088
3089         ret = __nvme_fc_create_hw_queue(ctrl, &ctrl->queues[0], 0,
3090                                 NVME_AQ_DEPTH);
3091         if (ret)
3092                 goto out_free_queue;
3093
3094         ret = nvme_fc_connect_admin_queue(ctrl, &ctrl->queues[0],
3095                                 NVME_AQ_DEPTH, (NVME_AQ_DEPTH / 4));
3096         if (ret)
3097                 goto out_delete_hw_queue;
3098
3099         ret = nvmf_connect_admin_queue(&ctrl->ctrl);
3100         if (ret)
3101                 goto out_disconnect_admin_queue;
3102
3103         set_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags);
3104
3105         /*
3106          * Check controller capabilities
3107          *
3108          * todo:- add code to check if ctrl attributes changed from
3109          * prior connection values
3110          */
3111
3112         ret = nvme_enable_ctrl(&ctrl->ctrl);
3113         if (ret || test_bit(ASSOC_FAILED, &ctrl->flags))
3114                 goto out_disconnect_admin_queue;
3115
3116         ctrl->ctrl.max_segments = ctrl->lport->ops->max_sgl_segments;
3117         ctrl->ctrl.max_hw_sectors = ctrl->ctrl.max_segments <<
3118                                                 (ilog2(SZ_4K) - 9);
3119
3120         nvme_start_admin_queue(&ctrl->ctrl);
3121
3122         ret = nvme_init_ctrl_finish(&ctrl->ctrl);
3123         if (ret || test_bit(ASSOC_FAILED, &ctrl->flags))
3124                 goto out_disconnect_admin_queue;
3125
3126         /* sanity checks */
3127
3128         /* FC-NVME does not have other data in the capsule */
3129         if (ctrl->ctrl.icdoff) {
3130                 dev_err(ctrl->ctrl.device, "icdoff %d is not supported!\n",
3131                                 ctrl->ctrl.icdoff);
3132                 ret = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
3133                 goto out_disconnect_admin_queue;
3134         }
3135
3136         /* FC-NVME supports normal SGL Data Block Descriptors */
3137         if (!nvme_ctrl_sgl_supported(&ctrl->ctrl)) {
3138                 dev_err(ctrl->ctrl.device,
3139                         "Mandatory sgls are not supported!\n");
3140                 ret = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
3141                 goto out_disconnect_admin_queue;
3142         }
3143
3144         if (opts->queue_size > ctrl->ctrl.maxcmd) {
3145                 /* warn if maxcmd is lower than queue_size */
3146                 dev_warn(ctrl->ctrl.device,
3147                         "queue_size %zu > ctrl maxcmd %u, reducing "
3148                         "to maxcmd\n",
3149                         opts->queue_size, ctrl->ctrl.maxcmd);
3150                 opts->queue_size = ctrl->ctrl.maxcmd;
3151         }
3152
3153         if (opts->queue_size > ctrl->ctrl.sqsize + 1) {
3154                 /* warn if sqsize is lower than queue_size */
3155                 dev_warn(ctrl->ctrl.device,
3156                         "queue_size %zu > ctrl sqsize %u, reducing "
3157                         "to sqsize\n",
3158                         opts->queue_size, ctrl->ctrl.sqsize + 1);
3159                 opts->queue_size = ctrl->ctrl.sqsize + 1;
3160         }
3161
3162         ret = nvme_fc_init_aen_ops(ctrl);
3163         if (ret)
3164                 goto out_term_aen_ops;
3165
3166         /*
3167          * Create the io queues
3168          */
3169
3170         if (ctrl->ctrl.queue_count > 1) {
3171                 if (!ctrl->ioq_live)
3172                         ret = nvme_fc_create_io_queues(ctrl);
3173                 else
3174                         ret = nvme_fc_recreate_io_queues(ctrl);
3175         }
3176         if (ret || test_bit(ASSOC_FAILED, &ctrl->flags))
3177                 goto out_term_aen_ops;
3178
3179         changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
3180
3181         ctrl->ctrl.nr_reconnects = 0;
3182
3183         if (changed)
3184                 nvme_start_ctrl(&ctrl->ctrl);
3185
3186         return 0;       /* Success */
3187
3188 out_term_aen_ops:
3189         nvme_fc_term_aen_ops(ctrl);
3190 out_disconnect_admin_queue:
3191         /* send a Disconnect(association) LS to fc-nvme target */
3192         nvme_fc_xmt_disconnect_assoc(ctrl);
3193         spin_lock_irqsave(&ctrl->lock, flags);
3194         ctrl->association_id = 0;
3195         disls = ctrl->rcv_disconn;
3196         ctrl->rcv_disconn = NULL;
3197         spin_unlock_irqrestore(&ctrl->lock, flags);
3198         if (disls)
3199                 nvme_fc_xmt_ls_rsp(disls);
3200 out_delete_hw_queue:
3201         __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
3202 out_free_queue:
3203         nvme_fc_free_queue(&ctrl->queues[0]);
3204         clear_bit(ASSOC_ACTIVE, &ctrl->flags);
3205         nvme_fc_ctlr_inactive_on_rport(ctrl);
3206
3207         return ret;
3208 }
3209
3210
3211 /*
3212  * This routine stops operation of the controller on the host side.
3213  * On the host os stack side: Admin and IO queues are stopped,
3214  *   outstanding ios on them terminated via FC ABTS.
3215  * On the link side: the association is terminated.
3216  */
3217 static void
3218 nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl)
3219 {
3220         struct nvmefc_ls_rcv_op *disls = NULL;
3221         unsigned long flags;
3222
3223         if (!test_and_clear_bit(ASSOC_ACTIVE, &ctrl->flags))
3224                 return;
3225
3226         spin_lock_irqsave(&ctrl->lock, flags);
3227         set_bit(FCCTRL_TERMIO, &ctrl->flags);
3228         ctrl->iocnt = 0;
3229         spin_unlock_irqrestore(&ctrl->lock, flags);
3230
3231         __nvme_fc_abort_outstanding_ios(ctrl, false);
3232
3233         /* kill the aens as they are a separate path */
3234         nvme_fc_abort_aen_ops(ctrl);
3235
3236         /* wait for all io that had to be aborted */
3237         spin_lock_irq(&ctrl->lock);
3238         wait_event_lock_irq(ctrl->ioabort_wait, ctrl->iocnt == 0, ctrl->lock);
3239         clear_bit(FCCTRL_TERMIO, &ctrl->flags);
3240         spin_unlock_irq(&ctrl->lock);
3241
3242         nvme_fc_term_aen_ops(ctrl);
3243
3244         /*
3245          * send a Disconnect(association) LS to fc-nvme target
3246          * Note: could have been sent at top of process, but
3247          * cleaner on link traffic if after the aborts complete.
3248          * Note: if association doesn't exist, association_id will be 0
3249          */
3250         if (ctrl->association_id)
3251                 nvme_fc_xmt_disconnect_assoc(ctrl);
3252
3253         spin_lock_irqsave(&ctrl->lock, flags);
3254         ctrl->association_id = 0;
3255         disls = ctrl->rcv_disconn;
3256         ctrl->rcv_disconn = NULL;
3257         spin_unlock_irqrestore(&ctrl->lock, flags);
3258         if (disls)
3259                 /*
3260                  * if a Disconnect Request was waiting for a response, send
3261                  * now that all ABTS's have been issued (and are complete).
3262                  */
3263                 nvme_fc_xmt_ls_rsp(disls);
3264
3265         if (ctrl->ctrl.tagset) {
3266                 nvme_fc_delete_hw_io_queues(ctrl);
3267                 nvme_fc_free_io_queues(ctrl);
3268         }
3269
3270         __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
3271         nvme_fc_free_queue(&ctrl->queues[0]);
3272
3273         /* re-enable the admin_q so anything new can fast fail */
3274         nvme_start_admin_queue(&ctrl->ctrl);
3275
3276         /* resume the io queues so that things will fast fail */
3277         nvme_start_queues(&ctrl->ctrl);
3278
3279         nvme_fc_ctlr_inactive_on_rport(ctrl);
3280 }
3281
3282 static void
3283 nvme_fc_delete_ctrl(struct nvme_ctrl *nctrl)
3284 {
3285         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
3286
3287         cancel_work_sync(&ctrl->ioerr_work);
3288         cancel_delayed_work_sync(&ctrl->connect_work);
3289         /*
3290          * kill the association on the link side.  this will block
3291          * waiting for io to terminate
3292          */
3293         nvme_fc_delete_association(ctrl);
3294 }
3295
3296 static void
3297 nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status)
3298 {
3299         struct nvme_fc_rport *rport = ctrl->rport;
3300         struct nvme_fc_remote_port *portptr = &rport->remoteport;
3301         unsigned long recon_delay = ctrl->ctrl.opts->reconnect_delay * HZ;
3302         bool recon = true;
3303
3304         if (ctrl->ctrl.state != NVME_CTRL_CONNECTING)
3305                 return;
3306
3307         if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3308                 dev_info(ctrl->ctrl.device,
3309                         "NVME-FC{%d}: reset: Reconnect attempt failed (%d)\n",
3310                         ctrl->cnum, status);
3311                 if (status > 0 && (status & NVME_SC_DNR))
3312                         recon = false;
3313         } else if (time_after_eq(jiffies, rport->dev_loss_end))
3314                 recon = false;
3315
3316         if (recon && nvmf_should_reconnect(&ctrl->ctrl)) {
3317                 if (portptr->port_state == FC_OBJSTATE_ONLINE)
3318                         dev_info(ctrl->ctrl.device,
3319                                 "NVME-FC{%d}: Reconnect attempt in %ld "
3320                                 "seconds\n",
3321                                 ctrl->cnum, recon_delay / HZ);
3322                 else if (time_after(jiffies + recon_delay, rport->dev_loss_end))
3323                         recon_delay = rport->dev_loss_end - jiffies;
3324
3325                 queue_delayed_work(nvme_wq, &ctrl->connect_work, recon_delay);
3326         } else {
3327                 if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3328                         if (status > 0 && (status & NVME_SC_DNR))
3329                                 dev_warn(ctrl->ctrl.device,
3330                                          "NVME-FC{%d}: reconnect failure\n",
3331                                          ctrl->cnum);
3332                         else
3333                                 dev_warn(ctrl->ctrl.device,
3334                                          "NVME-FC{%d}: Max reconnect attempts "
3335                                          "(%d) reached.\n",
3336                                          ctrl->cnum, ctrl->ctrl.nr_reconnects);
3337                 } else
3338                         dev_warn(ctrl->ctrl.device,
3339                                 "NVME-FC{%d}: dev_loss_tmo (%d) expired "
3340                                 "while waiting for remoteport connectivity.\n",
3341                                 ctrl->cnum, min_t(int, portptr->dev_loss_tmo,
3342                                         (ctrl->ctrl.opts->max_reconnects *
3343                                          ctrl->ctrl.opts->reconnect_delay)));
3344                 WARN_ON(nvme_delete_ctrl(&ctrl->ctrl));
3345         }
3346 }
3347
3348 static void
3349 nvme_fc_reset_ctrl_work(struct work_struct *work)
3350 {
3351         struct nvme_fc_ctrl *ctrl =
3352                 container_of(work, struct nvme_fc_ctrl, ctrl.reset_work);
3353
3354         nvme_stop_ctrl(&ctrl->ctrl);
3355
3356         /* will block will waiting for io to terminate */
3357         nvme_fc_delete_association(ctrl);
3358
3359         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING))
3360                 dev_err(ctrl->ctrl.device,
3361                         "NVME-FC{%d}: error_recovery: Couldn't change state "
3362                         "to CONNECTING\n", ctrl->cnum);
3363
3364         if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
3365                 if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) {
3366                         dev_err(ctrl->ctrl.device,
3367                                 "NVME-FC{%d}: failed to schedule connect "
3368                                 "after reset\n", ctrl->cnum);
3369                 } else {
3370                         flush_delayed_work(&ctrl->connect_work);
3371                 }
3372         } else {
3373                 nvme_fc_reconnect_or_delete(ctrl, -ENOTCONN);
3374         }
3375 }
3376
3377
3378 static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = {
3379         .name                   = "fc",
3380         .module                 = THIS_MODULE,
3381         .flags                  = NVME_F_FABRICS,
3382         .reg_read32             = nvmf_reg_read32,
3383         .reg_read64             = nvmf_reg_read64,
3384         .reg_write32            = nvmf_reg_write32,
3385         .free_ctrl              = nvme_fc_nvme_ctrl_freed,
3386         .submit_async_event     = nvme_fc_submit_async_event,
3387         .delete_ctrl            = nvme_fc_delete_ctrl,
3388         .get_address            = nvmf_get_address,
3389 };
3390
3391 static void
3392 nvme_fc_connect_ctrl_work(struct work_struct *work)
3393 {
3394         int ret;
3395
3396         struct nvme_fc_ctrl *ctrl =
3397                         container_of(to_delayed_work(work),
3398                                 struct nvme_fc_ctrl, connect_work);
3399
3400         ret = nvme_fc_create_association(ctrl);
3401         if (ret)
3402                 nvme_fc_reconnect_or_delete(ctrl, ret);
3403         else
3404                 dev_info(ctrl->ctrl.device,
3405                         "NVME-FC{%d}: controller connect complete\n",
3406                         ctrl->cnum);
3407 }
3408
3409
3410 static const struct blk_mq_ops nvme_fc_admin_mq_ops = {
3411         .queue_rq       = nvme_fc_queue_rq,
3412         .complete       = nvme_fc_complete_rq,
3413         .init_request   = nvme_fc_init_request,
3414         .exit_request   = nvme_fc_exit_request,
3415         .init_hctx      = nvme_fc_init_admin_hctx,
3416         .timeout        = nvme_fc_timeout,
3417 };
3418
3419
3420 /*
3421  * Fails a controller request if it matches an existing controller
3422  * (association) with the same tuple:
3423  * <Host NQN, Host ID, local FC port, remote FC port, SUBSYS NQN>
3424  *
3425  * The ports don't need to be compared as they are intrinsically
3426  * already matched by the port pointers supplied.
3427  */
3428 static bool
3429 nvme_fc_existing_controller(struct nvme_fc_rport *rport,
3430                 struct nvmf_ctrl_options *opts)
3431 {
3432         struct nvme_fc_ctrl *ctrl;
3433         unsigned long flags;
3434         bool found = false;
3435
3436         spin_lock_irqsave(&rport->lock, flags);
3437         list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3438                 found = nvmf_ctlr_matches_baseopts(&ctrl->ctrl, opts);
3439                 if (found)
3440                         break;
3441         }
3442         spin_unlock_irqrestore(&rport->lock, flags);
3443
3444         return found;
3445 }
3446
3447 static struct nvme_ctrl *
3448 nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
3449         struct nvme_fc_lport *lport, struct nvme_fc_rport *rport)
3450 {
3451         struct nvme_fc_ctrl *ctrl;
3452         unsigned long flags;
3453         int ret, idx, ctrl_loss_tmo;
3454
3455         if (!(rport->remoteport.port_role &
3456             (FC_PORT_ROLE_NVME_DISCOVERY | FC_PORT_ROLE_NVME_TARGET))) {
3457                 ret = -EBADR;
3458                 goto out_fail;
3459         }
3460
3461         if (!opts->duplicate_connect &&
3462             nvme_fc_existing_controller(rport, opts)) {
3463                 ret = -EALREADY;
3464                 goto out_fail;
3465         }
3466
3467         ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
3468         if (!ctrl) {
3469                 ret = -ENOMEM;
3470                 goto out_fail;
3471         }
3472
3473         idx = ida_alloc(&nvme_fc_ctrl_cnt, GFP_KERNEL);
3474         if (idx < 0) {
3475                 ret = -ENOSPC;
3476                 goto out_free_ctrl;
3477         }
3478
3479         /*
3480          * if ctrl_loss_tmo is being enforced and the default reconnect delay
3481          * is being used, change to a shorter reconnect delay for FC.
3482          */
3483         if (opts->max_reconnects != -1 &&
3484             opts->reconnect_delay == NVMF_DEF_RECONNECT_DELAY &&
3485             opts->reconnect_delay > NVME_FC_DEFAULT_RECONNECT_TMO) {
3486                 ctrl_loss_tmo = opts->max_reconnects * opts->reconnect_delay;
3487                 opts->reconnect_delay = NVME_FC_DEFAULT_RECONNECT_TMO;
3488                 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3489                                                 opts->reconnect_delay);
3490         }
3491
3492         ctrl->ctrl.opts = opts;
3493         ctrl->ctrl.nr_reconnects = 0;
3494         if (lport->dev)
3495                 ctrl->ctrl.numa_node = dev_to_node(lport->dev);
3496         else
3497                 ctrl->ctrl.numa_node = NUMA_NO_NODE;
3498         INIT_LIST_HEAD(&ctrl->ctrl_list);
3499         ctrl->lport = lport;
3500         ctrl->rport = rport;
3501         ctrl->dev = lport->dev;
3502         ctrl->cnum = idx;
3503         ctrl->ioq_live = false;
3504         init_waitqueue_head(&ctrl->ioabort_wait);
3505
3506         get_device(ctrl->dev);
3507         kref_init(&ctrl->ref);
3508
3509         INIT_WORK(&ctrl->ctrl.reset_work, nvme_fc_reset_ctrl_work);
3510         INIT_DELAYED_WORK(&ctrl->connect_work, nvme_fc_connect_ctrl_work);
3511         INIT_WORK(&ctrl->ioerr_work, nvme_fc_ctrl_ioerr_work);
3512         spin_lock_init(&ctrl->lock);
3513
3514         /* io queue count */
3515         ctrl->ctrl.queue_count = min_t(unsigned int,
3516                                 opts->nr_io_queues,
3517                                 lport->ops->max_hw_queues);
3518         ctrl->ctrl.queue_count++;       /* +1 for admin queue */
3519
3520         ctrl->ctrl.sqsize = opts->queue_size - 1;
3521         ctrl->ctrl.kato = opts->kato;
3522         ctrl->ctrl.cntlid = 0xffff;
3523
3524         ret = -ENOMEM;
3525         ctrl->queues = kcalloc(ctrl->ctrl.queue_count,
3526                                 sizeof(struct nvme_fc_queue), GFP_KERNEL);
3527         if (!ctrl->queues)
3528                 goto out_free_ida;
3529
3530         nvme_fc_init_queue(ctrl, 0);
3531
3532         memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set));
3533         ctrl->admin_tag_set.ops = &nvme_fc_admin_mq_ops;
3534         ctrl->admin_tag_set.queue_depth = NVME_AQ_MQ_TAG_DEPTH;
3535         ctrl->admin_tag_set.reserved_tags = NVMF_RESERVED_TAGS;
3536         ctrl->admin_tag_set.numa_node = ctrl->ctrl.numa_node;
3537         ctrl->admin_tag_set.cmd_size =
3538                 struct_size((struct nvme_fcp_op_w_sgl *)NULL, priv,
3539                             ctrl->lport->ops->fcprqst_priv_sz);
3540         ctrl->admin_tag_set.driver_data = ctrl;
3541         ctrl->admin_tag_set.nr_hw_queues = 1;
3542         ctrl->admin_tag_set.timeout = NVME_ADMIN_TIMEOUT;
3543         ctrl->admin_tag_set.flags = BLK_MQ_F_NO_SCHED;
3544
3545         ret = blk_mq_alloc_tag_set(&ctrl->admin_tag_set);
3546         if (ret)
3547                 goto out_free_queues;
3548         ctrl->ctrl.admin_tagset = &ctrl->admin_tag_set;
3549
3550         ctrl->ctrl.fabrics_q = blk_mq_init_queue(&ctrl->admin_tag_set);
3551         if (IS_ERR(ctrl->ctrl.fabrics_q)) {
3552                 ret = PTR_ERR(ctrl->ctrl.fabrics_q);
3553                 goto out_free_admin_tag_set;
3554         }
3555
3556         ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
3557         if (IS_ERR(ctrl->ctrl.admin_q)) {
3558                 ret = PTR_ERR(ctrl->ctrl.admin_q);
3559                 goto out_cleanup_fabrics_q;
3560         }
3561
3562         /*
3563          * Would have been nice to init io queues tag set as well.
3564          * However, we require interaction from the controller
3565          * for max io queue count before we can do so.
3566          * Defer this to the connect path.
3567          */
3568
3569         ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_fc_ctrl_ops, 0);
3570         if (ret)
3571                 goto out_cleanup_admin_q;
3572
3573         /* at this point, teardown path changes to ref counting on nvme ctrl */
3574
3575         spin_lock_irqsave(&rport->lock, flags);
3576         list_add_tail(&ctrl->ctrl_list, &rport->ctrl_list);
3577         spin_unlock_irqrestore(&rport->lock, flags);
3578
3579         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING) ||
3580             !nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
3581                 dev_err(ctrl->ctrl.device,
3582                         "NVME-FC{%d}: failed to init ctrl state\n", ctrl->cnum);
3583                 goto fail_ctrl;
3584         }
3585
3586         if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) {
3587                 dev_err(ctrl->ctrl.device,
3588                         "NVME-FC{%d}: failed to schedule initial connect\n",
3589                         ctrl->cnum);
3590                 goto fail_ctrl;
3591         }
3592
3593         flush_delayed_work(&ctrl->connect_work);
3594
3595         dev_info(ctrl->ctrl.device,
3596                 "NVME-FC{%d}: new ctrl: NQN \"%s\"\n",
3597                 ctrl->cnum, nvmf_ctrl_subsysnqn(&ctrl->ctrl));
3598
3599         return &ctrl->ctrl;
3600
3601 fail_ctrl:
3602         nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING);
3603         cancel_work_sync(&ctrl->ioerr_work);
3604         cancel_work_sync(&ctrl->ctrl.reset_work);
3605         cancel_delayed_work_sync(&ctrl->connect_work);
3606
3607         ctrl->ctrl.opts = NULL;
3608
3609         /* initiate nvme ctrl ref counting teardown */
3610         nvme_uninit_ctrl(&ctrl->ctrl);
3611
3612         /* Remove core ctrl ref. */
3613         nvme_put_ctrl(&ctrl->ctrl);
3614
3615         /* as we're past the point where we transition to the ref
3616          * counting teardown path, if we return a bad pointer here,
3617          * the calling routine, thinking it's prior to the
3618          * transition, will do an rport put. Since the teardown
3619          * path also does a rport put, we do an extra get here to
3620          * so proper order/teardown happens.
3621          */
3622         nvme_fc_rport_get(rport);
3623
3624         return ERR_PTR(-EIO);
3625
3626 out_cleanup_admin_q:
3627         blk_cleanup_queue(ctrl->ctrl.admin_q);
3628 out_cleanup_fabrics_q:
3629         blk_cleanup_queue(ctrl->ctrl.fabrics_q);
3630 out_free_admin_tag_set:
3631         blk_mq_free_tag_set(&ctrl->admin_tag_set);
3632 out_free_queues:
3633         kfree(ctrl->queues);
3634 out_free_ida:
3635         put_device(ctrl->dev);
3636         ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum);
3637 out_free_ctrl:
3638         kfree(ctrl);
3639 out_fail:
3640         /* exit via here doesn't follow ctlr ref points */
3641         return ERR_PTR(ret);
3642 }
3643
3644
3645 struct nvmet_fc_traddr {
3646         u64     nn;
3647         u64     pn;
3648 };
3649
3650 static int
3651 __nvme_fc_parse_u64(substring_t *sstr, u64 *val)
3652 {
3653         u64 token64;
3654
3655         if (match_u64(sstr, &token64))
3656                 return -EINVAL;
3657         *val = token64;
3658
3659         return 0;
3660 }
3661
3662 /*
3663  * This routine validates and extracts the WWN's from the TRADDR string.
3664  * As kernel parsers need the 0x to determine number base, universally
3665  * build string to parse with 0x prefix before parsing name strings.
3666  */
3667 static int
3668 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
3669 {
3670         char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
3671         substring_t wwn = { name, &name[sizeof(name)-1] };
3672         int nnoffset, pnoffset;
3673
3674         /* validate if string is one of the 2 allowed formats */
3675         if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH &&
3676                         !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
3677                         !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
3678                                 "pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
3679                 nnoffset = NVME_FC_TRADDR_OXNNLEN;
3680                 pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
3681                                                 NVME_FC_TRADDR_OXNNLEN;
3682         } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH &&
3683                         !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
3684                         !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
3685                                 "pn-", NVME_FC_TRADDR_NNLEN))) {
3686                 nnoffset = NVME_FC_TRADDR_NNLEN;
3687                 pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
3688         } else
3689                 goto out_einval;
3690
3691         name[0] = '0';
3692         name[1] = 'x';
3693         name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
3694
3695         memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3696         if (__nvme_fc_parse_u64(&wwn, &traddr->nn))
3697                 goto out_einval;
3698
3699         memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3700         if (__nvme_fc_parse_u64(&wwn, &traddr->pn))
3701                 goto out_einval;
3702
3703         return 0;
3704
3705 out_einval:
3706         pr_warn("%s: bad traddr string\n", __func__);
3707         return -EINVAL;
3708 }
3709
3710 static struct nvme_ctrl *
3711 nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts)
3712 {
3713         struct nvme_fc_lport *lport;
3714         struct nvme_fc_rport *rport;
3715         struct nvme_ctrl *ctrl;
3716         struct nvmet_fc_traddr laddr = { 0L, 0L };
3717         struct nvmet_fc_traddr raddr = { 0L, 0L };
3718         unsigned long flags;
3719         int ret;
3720
3721         ret = nvme_fc_parse_traddr(&raddr, opts->traddr, NVMF_TRADDR_SIZE);
3722         if (ret || !raddr.nn || !raddr.pn)
3723                 return ERR_PTR(-EINVAL);
3724
3725         ret = nvme_fc_parse_traddr(&laddr, opts->host_traddr, NVMF_TRADDR_SIZE);
3726         if (ret || !laddr.nn || !laddr.pn)
3727                 return ERR_PTR(-EINVAL);
3728
3729         /* find the host and remote ports to connect together */
3730         spin_lock_irqsave(&nvme_fc_lock, flags);
3731         list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3732                 if (lport->localport.node_name != laddr.nn ||
3733                     lport->localport.port_name != laddr.pn ||
3734                     lport->localport.port_state != FC_OBJSTATE_ONLINE)
3735                         continue;
3736
3737                 list_for_each_entry(rport, &lport->endp_list, endp_list) {
3738                         if (rport->remoteport.node_name != raddr.nn ||
3739                             rport->remoteport.port_name != raddr.pn ||
3740                             rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
3741                                 continue;
3742
3743                         /* if fail to get reference fall through. Will error */
3744                         if (!nvme_fc_rport_get(rport))
3745                                 break;
3746
3747                         spin_unlock_irqrestore(&nvme_fc_lock, flags);
3748
3749                         ctrl = nvme_fc_init_ctrl(dev, opts, lport, rport);
3750                         if (IS_ERR(ctrl))
3751                                 nvme_fc_rport_put(rport);
3752                         return ctrl;
3753                 }
3754         }
3755         spin_unlock_irqrestore(&nvme_fc_lock, flags);
3756
3757         pr_warn("%s: %s - %s combination not found\n",
3758                 __func__, opts->traddr, opts->host_traddr);
3759         return ERR_PTR(-ENOENT);
3760 }
3761
3762
3763 static struct nvmf_transport_ops nvme_fc_transport = {
3764         .name           = "fc",
3765         .module         = THIS_MODULE,
3766         .required_opts  = NVMF_OPT_TRADDR | NVMF_OPT_HOST_TRADDR,
3767         .allowed_opts   = NVMF_OPT_RECONNECT_DELAY | NVMF_OPT_CTRL_LOSS_TMO,
3768         .create_ctrl    = nvme_fc_create_ctrl,
3769 };
3770
3771 /* Arbitrary successive failures max. With lots of subsystems could be high */
3772 #define DISCOVERY_MAX_FAIL      20
3773
3774 static ssize_t nvme_fc_nvme_discovery_store(struct device *dev,
3775                 struct device_attribute *attr, const char *buf, size_t count)
3776 {
3777         unsigned long flags;
3778         LIST_HEAD(local_disc_list);
3779         struct nvme_fc_lport *lport;
3780         struct nvme_fc_rport *rport;
3781         int failcnt = 0;
3782
3783         spin_lock_irqsave(&nvme_fc_lock, flags);
3784 restart:
3785         list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3786                 list_for_each_entry(rport, &lport->endp_list, endp_list) {
3787                         if (!nvme_fc_lport_get(lport))
3788                                 continue;
3789                         if (!nvme_fc_rport_get(rport)) {
3790                                 /*
3791                                  * This is a temporary condition. Upon restart
3792                                  * this rport will be gone from the list.
3793                                  *
3794                                  * Revert the lport put and retry.  Anything
3795                                  * added to the list already will be skipped (as
3796                                  * they are no longer list_empty).  Loops should
3797                                  * resume at rports that were not yet seen.
3798                                  */
3799                                 nvme_fc_lport_put(lport);
3800
3801                                 if (failcnt++ < DISCOVERY_MAX_FAIL)
3802                                         goto restart;
3803
3804                                 pr_err("nvme_discovery: too many reference "
3805                                        "failures\n");
3806                                 goto process_local_list;
3807                         }
3808                         if (list_empty(&rport->disc_list))
3809                                 list_add_tail(&rport->disc_list,
3810                                               &local_disc_list);
3811                 }
3812         }
3813
3814 process_local_list:
3815         while (!list_empty(&local_disc_list)) {
3816                 rport = list_first_entry(&local_disc_list,
3817                                          struct nvme_fc_rport, disc_list);
3818                 list_del_init(&rport->disc_list);
3819                 spin_unlock_irqrestore(&nvme_fc_lock, flags);
3820
3821                 lport = rport->lport;
3822                 /* signal discovery. Won't hurt if it repeats */
3823                 nvme_fc_signal_discovery_scan(lport, rport);
3824                 nvme_fc_rport_put(rport);
3825                 nvme_fc_lport_put(lport);
3826
3827                 spin_lock_irqsave(&nvme_fc_lock, flags);
3828         }
3829         spin_unlock_irqrestore(&nvme_fc_lock, flags);
3830
3831         return count;
3832 }
3833
3834 static DEVICE_ATTR(nvme_discovery, 0200, NULL, nvme_fc_nvme_discovery_store);
3835
3836 #ifdef CONFIG_BLK_CGROUP_FC_APPID
3837 /* Parse the cgroup id from a buf and return the length of cgrpid */
3838 static int fc_parse_cgrpid(const char *buf, u64 *id)
3839 {
3840         char cgrp_id[16+1];
3841         int cgrpid_len, j;
3842
3843         memset(cgrp_id, 0x0, sizeof(cgrp_id));
3844         for (cgrpid_len = 0, j = 0; cgrpid_len < 17; cgrpid_len++) {
3845                 if (buf[cgrpid_len] != ':')
3846                         cgrp_id[cgrpid_len] = buf[cgrpid_len];
3847                 else {
3848                         j = 1;
3849                         break;
3850                 }
3851         }
3852         if (!j)
3853                 return -EINVAL;
3854         if (kstrtou64(cgrp_id, 16, id) < 0)
3855                 return -EINVAL;
3856         return cgrpid_len;
3857 }
3858
3859 /*
3860  * Parse and update the appid in the blkcg associated with the cgroupid.
3861  */
3862 static ssize_t fc_appid_store(struct device *dev,
3863                 struct device_attribute *attr, const char *buf, size_t count)
3864 {
3865         u64 cgrp_id;
3866         int appid_len = 0;
3867         int cgrpid_len = 0;
3868         char app_id[FC_APPID_LEN];
3869         int ret = 0;
3870
3871         if (buf[count-1] == '\n')
3872                 count--;
3873
3874         if ((count > (16+1+FC_APPID_LEN)) || (!strchr(buf, ':')))
3875                 return -EINVAL;
3876
3877         cgrpid_len = fc_parse_cgrpid(buf, &cgrp_id);
3878         if (cgrpid_len < 0)
3879                 return -EINVAL;
3880         appid_len = count - cgrpid_len - 1;
3881         if (appid_len > FC_APPID_LEN)
3882                 return -EINVAL;
3883
3884         memset(app_id, 0x0, sizeof(app_id));
3885         memcpy(app_id, &buf[cgrpid_len+1], appid_len);
3886         ret = blkcg_set_fc_appid(app_id, cgrp_id, sizeof(app_id));
3887         if (ret < 0)
3888                 return ret;
3889         return count;
3890 }
3891 static DEVICE_ATTR(appid_store, 0200, NULL, fc_appid_store);
3892 #endif /* CONFIG_BLK_CGROUP_FC_APPID */
3893
3894 static struct attribute *nvme_fc_attrs[] = {
3895         &dev_attr_nvme_discovery.attr,
3896 #ifdef CONFIG_BLK_CGROUP_FC_APPID
3897         &dev_attr_appid_store.attr,
3898 #endif
3899         NULL
3900 };
3901
3902 static const struct attribute_group nvme_fc_attr_group = {
3903         .attrs = nvme_fc_attrs,
3904 };
3905
3906 static const struct attribute_group *nvme_fc_attr_groups[] = {
3907         &nvme_fc_attr_group,
3908         NULL
3909 };
3910
3911 static struct class fc_class = {
3912         .name = "fc",
3913         .dev_groups = nvme_fc_attr_groups,
3914         .owner = THIS_MODULE,
3915 };
3916
3917 static int __init nvme_fc_init_module(void)
3918 {
3919         int ret;
3920
3921         nvme_fc_wq = alloc_workqueue("nvme_fc_wq", WQ_MEM_RECLAIM, 0);
3922         if (!nvme_fc_wq)
3923                 return -ENOMEM;
3924
3925         /*
3926          * NOTE:
3927          * It is expected that in the future the kernel will combine
3928          * the FC-isms that are currently under scsi and now being
3929          * added to by NVME into a new standalone FC class. The SCSI
3930          * and NVME protocols and their devices would be under this
3931          * new FC class.
3932          *
3933          * As we need something to post FC-specific udev events to,
3934          * specifically for nvme probe events, start by creating the
3935          * new device class.  When the new standalone FC class is
3936          * put in place, this code will move to a more generic
3937          * location for the class.
3938          */
3939         ret = class_register(&fc_class);
3940         if (ret) {
3941                 pr_err("couldn't register class fc\n");
3942                 goto out_destroy_wq;
3943         }
3944
3945         /*
3946          * Create a device for the FC-centric udev events
3947          */
3948         fc_udev_device = device_create(&fc_class, NULL, MKDEV(0, 0), NULL,
3949                                 "fc_udev_device");
3950         if (IS_ERR(fc_udev_device)) {
3951                 pr_err("couldn't create fc_udev device!\n");
3952                 ret = PTR_ERR(fc_udev_device);
3953                 goto out_destroy_class;
3954         }
3955
3956         ret = nvmf_register_transport(&nvme_fc_transport);
3957         if (ret)
3958                 goto out_destroy_device;
3959
3960         return 0;
3961
3962 out_destroy_device:
3963         device_destroy(&fc_class, MKDEV(0, 0));
3964 out_destroy_class:
3965         class_unregister(&fc_class);
3966 out_destroy_wq:
3967         destroy_workqueue(nvme_fc_wq);
3968
3969         return ret;
3970 }
3971
3972 static void
3973 nvme_fc_delete_controllers(struct nvme_fc_rport *rport)
3974 {
3975         struct nvme_fc_ctrl *ctrl;
3976
3977         spin_lock(&rport->lock);
3978         list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3979                 dev_warn(ctrl->ctrl.device,
3980                         "NVME-FC{%d}: transport unloading: deleting ctrl\n",
3981                         ctrl->cnum);
3982                 nvme_delete_ctrl(&ctrl->ctrl);
3983         }
3984         spin_unlock(&rport->lock);
3985 }
3986
3987 static void
3988 nvme_fc_cleanup_for_unload(void)
3989 {
3990         struct nvme_fc_lport *lport;
3991         struct nvme_fc_rport *rport;
3992
3993         list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3994                 list_for_each_entry(rport, &lport->endp_list, endp_list) {
3995                         nvme_fc_delete_controllers(rport);
3996                 }
3997         }
3998 }
3999
4000 static void __exit nvme_fc_exit_module(void)
4001 {
4002         unsigned long flags;
4003         bool need_cleanup = false;
4004
4005         spin_lock_irqsave(&nvme_fc_lock, flags);
4006         nvme_fc_waiting_to_unload = true;
4007         if (!list_empty(&nvme_fc_lport_list)) {
4008                 need_cleanup = true;
4009                 nvme_fc_cleanup_for_unload();
4010         }
4011         spin_unlock_irqrestore(&nvme_fc_lock, flags);
4012         if (need_cleanup) {
4013                 pr_info("%s: waiting for ctlr deletes\n", __func__);
4014                 wait_for_completion(&nvme_fc_unload_proceed);
4015                 pr_info("%s: ctrl deletes complete\n", __func__);
4016         }
4017
4018         nvmf_unregister_transport(&nvme_fc_transport);
4019
4020         ida_destroy(&nvme_fc_local_port_cnt);
4021         ida_destroy(&nvme_fc_ctrl_cnt);
4022
4023         device_destroy(&fc_class, MKDEV(0, 0));
4024         class_unregister(&fc_class);
4025         destroy_workqueue(nvme_fc_wq);
4026 }
4027
4028 module_init(nvme_fc_init_module);
4029 module_exit(nvme_fc_exit_module);
4030
4031 MODULE_LICENSE("GPL v2");