drivers/scsi/lpfc/lpfc_nvmet.c

   1 /*******************************************************************
   2  * This file is part of the Emulex Linux Device Driver for         *
   3  * Fibre Channsel Host Bus Adapters.                               *
   4  * Copyright (C) 2017 Broadcom. All Rights Reserved. The term      *
   5  * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
   6  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
   7  * EMULEX and SLI are trademarks of Emulex.                        *
   8  * www.broadcom.com                                                *
   9  * Portions Copyright (C) 2004-2005 Christoph Hellwig              *
  10  *                                                                 *
  11  * This program is free software; you can redistribute it and/or   *
  12  * modify it under the terms of version 2 of the GNU General       *
  13  * Public License as published by the Free Software Foundation.    *
  14  * This program is distributed in the hope that it will be useful. *
  15  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
  16  * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
  17  * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
  18  * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
  19  * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
  20  * more details, a copy of which can be found in the file COPYING  *
  21  * included with this package.                                     *
  22  ********************************************************************/
  23 #include <linux/pci.h>
  24 #include <linux/slab.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/delay.h>
  27 #include <asm/unaligned.h>
  28 #include <linux/crc-t10dif.h>
  29 #include <net/checksum.h>
  30
  31 #include <scsi/scsi.h>
  32 #include <scsi/scsi_device.h>
  33 #include <scsi/scsi_eh.h>
  34 #include <scsi/scsi_host.h>
  35 #include <scsi/scsi_tcq.h>
  36 #include <scsi/scsi_transport_fc.h>
  37 #include <scsi/fc/fc_fs.h>
  38
  39 #include <../drivers/nvme/host/nvme.h>
  40 #include <linux/nvme-fc-driver.h>
  41
  42 #include "lpfc_version.h"
  43 #include "lpfc_hw4.h"
  44 #include "lpfc_hw.h"
  45 #include "lpfc_sli.h"
  46 #include "lpfc_sli4.h"
  47 #include "lpfc_nl.h"
  48 #include "lpfc_disc.h"
  49 #include "lpfc.h"
  50 #include "lpfc_scsi.h"
  51 #include "lpfc_nvme.h"
  52 #include "lpfc_nvmet.h"
  53 #include "lpfc_logmsg.h"
  54 #include "lpfc_crtn.h"
  55 #include "lpfc_vport.h"
  56 #include "lpfc_debugfs.h"
  57
  58 static struct lpfc_iocbq *lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *,
  59                                                  struct lpfc_nvmet_rcv_ctx *,
  60                                                  dma_addr_t rspbuf,
  61                                                  uint16_t rspsize);
  62 static struct lpfc_iocbq *lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *,
  63                                                   struct lpfc_nvmet_rcv_ctx *);
  64 static int lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *,
  65                                           struct lpfc_nvmet_rcv_ctx *,
  66                                           uint32_t, uint16_t);
  67 static int lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *,
  68                                             struct lpfc_nvmet_rcv_ctx *,
  69                                             uint32_t, uint16_t);
  70 static int lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *,
  71                                            struct lpfc_nvmet_rcv_ctx *,
  72                                            uint32_t, uint16_t);
  73
  74 void
  75 lpfc_nvmet_defer_release(struct lpfc_hba *phba, struct lpfc_nvmet_rcv_ctx *ctxp)
  76 {
  77         unsigned long iflag;
  78
  79         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
  80                         "6313 NVMET Defer ctx release xri x%x flg x%x\n",
  81                         ctxp->oxid, ctxp->flag);
  82
  83         spin_lock_irqsave(&phba->sli4_hba.abts_nvme_buf_list_lock, iflag);
  84         if (ctxp->flag & LPFC_NVMET_CTX_RLS) {
  85                 spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock,
  86                                        iflag);
  87                 return;
  88         }
  89         ctxp->flag |= LPFC_NVMET_CTX_RLS;
  90         list_add_tail(&ctxp->list, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list);
  91         spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock, iflag);
  92 }
  93
  94 /**
  95  * lpfc_nvmet_xmt_ls_rsp_cmp - Completion handler for LS Response
  96  * @phba: Pointer to HBA context object.
  97  * @cmdwqe: Pointer to driver command WQE object.
  98  * @wcqe: Pointer to driver response CQE object.
  99  *
 100  * The function is called from SLI ring event handler with no
 101  * lock held. This function is the completion handler for NVME LS commands
 102  * The function frees memory resources used for the NVME commands.
 103  **/
 104 static void
 105 lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
 106                           struct lpfc_wcqe_complete *wcqe)
 107 {
 108         struct lpfc_nvmet_tgtport *tgtp;
 109         struct nvmefc_tgt_ls_req *rsp;
 110         struct lpfc_nvmet_rcv_ctx *ctxp;
 111         uint32_t status, result;
 112
 113         status = bf_get(lpfc_wcqe_c_status, wcqe);
 114         result = wcqe->parameter;
 115         if (!phba->targetport)
 116                 goto out;
 117
 118         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
 119
 120         if (status)
 121                 atomic_inc(&tgtp->xmt_ls_rsp_error);
 122         else
 123                 atomic_inc(&tgtp->xmt_ls_rsp_cmpl);
 124
 125 out:
 126         ctxp = cmdwqe->context2;
 127         rsp = &ctxp->ctx.ls_req;
 128
 129         lpfc_nvmeio_data(phba, "NVMET LS  CMPL: xri x%x stat x%x result x%x\n",
 130                          ctxp->oxid, status, result);
 131
 132         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
 133                         "6038 %s: Entrypoint: ctx %p status %x/%x\n", __func__,
 134                         ctxp, status, result);
 135
 136         lpfc_nlp_put(cmdwqe->context1);
 137         cmdwqe->context2 = NULL;
 138         cmdwqe->context3 = NULL;
 139         lpfc_sli_release_iocbq(phba, cmdwqe);
 140         rsp->done(rsp);
 141         kfree(ctxp);
 142 }
 143
 144 /**
 145  * lpfc_nvmet_rq_post - Repost a NVMET RQ DMA buffer and clean up context
 146  * @phba: HBA buffer is associated with
 147  * @ctxp: context to clean up
 148  * @mp: Buffer to free
 149  *
 150  * Description: Frees the given DMA buffer in the appropriate way given by
 151  * reposting it to its associated RQ so it can be reused.
 152  *
 153  * Notes: Takes phba->hbalock.  Can be called with or without other locks held.
 154  *
 155  * Returns: None
 156  **/
 157 void
 158 lpfc_nvmet_rq_post(struct lpfc_hba *phba, struct lpfc_nvmet_rcv_ctx *ctxp,
 159                    struct lpfc_dmabuf *mp)
 160 {
 161         if (ctxp) {
 162                 if (ctxp->flag)
 163                         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
 164                                 "6314 rq_post ctx xri x%x flag x%x\n",
 165                                 ctxp->oxid, ctxp->flag);
 166
 167                 if (ctxp->txrdy) {
 168                         pci_pool_free(phba->txrdy_payload_pool, ctxp->txrdy,
 169                                       ctxp->txrdy_phys);
 170                         ctxp->txrdy = NULL;
 171                         ctxp->txrdy_phys = 0;
 172                 }
 173                 ctxp->state = LPFC_NVMET_STE_FREE;
 174         }
 175         lpfc_rq_buf_free(phba, mp);
 176 }
 177
 178 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 179 static void
 180 lpfc_nvmet_ktime(struct lpfc_hba *phba,
 181                  struct lpfc_nvmet_rcv_ctx *ctxp)
 182 {
 183         uint64_t seg1, seg2, seg3, seg4, seg5;
 184         uint64_t seg6, seg7, seg8, seg9, seg10;
 185
 186         if (!phba->ktime_on)
 187                 return;
 188
 189         if (!ctxp->ts_isr_cmd || !ctxp->ts_cmd_nvme ||
 190             !ctxp->ts_nvme_data || !ctxp->ts_data_wqput ||
 191             !ctxp->ts_isr_data || !ctxp->ts_data_nvme ||
 192             !ctxp->ts_nvme_status || !ctxp->ts_status_wqput ||
 193             !ctxp->ts_isr_status || !ctxp->ts_status_nvme)
 194                 return;
 195
 196         if (ctxp->ts_isr_cmd  > ctxp->ts_cmd_nvme)
 197                 return;
 198         if (ctxp->ts_cmd_nvme > ctxp->ts_nvme_data)
 199                 return;
 200         if (ctxp->ts_nvme_data > ctxp->ts_data_wqput)
 201                 return;
 202         if (ctxp->ts_data_wqput > ctxp->ts_isr_data)
 203                 return;
 204         if (ctxp->ts_isr_data > ctxp->ts_data_nvme)
 205                 return;
 206         if (ctxp->ts_data_nvme > ctxp->ts_nvme_status)
 207                 return;
 208         if (ctxp->ts_nvme_status > ctxp->ts_status_wqput)
 209                 return;
 210         if (ctxp->ts_status_wqput > ctxp->ts_isr_status)
 211                 return;
 212         if (ctxp->ts_isr_status > ctxp->ts_status_nvme)
 213                 return;
 214         /*
 215          * Segment 1 - Time from FCP command received by MSI-X ISR
 216          * to FCP command is passed to NVME Layer.
 217          * Segment 2 - Time from FCP command payload handed
 218          * off to NVME Layer to Driver receives a Command op
 219          * from NVME Layer.
 220          * Segment 3 - Time from Driver receives a Command op
 221          * from NVME Layer to Command is put on WQ.
 222          * Segment 4 - Time from Driver WQ put is done
 223          * to MSI-X ISR for Command cmpl.
 224          * Segment 5 - Time from MSI-X ISR for Command cmpl to
 225          * Command cmpl is passed to NVME Layer.
 226          * Segment 6 - Time from Command cmpl is passed to NVME
 227          * Layer to Driver receives a RSP op from NVME Layer.
 228          * Segment 7 - Time from Driver receives a RSP op from
 229          * NVME Layer to WQ put is done on TRSP FCP Status.
 230          * Segment 8 - Time from Driver WQ put is done on TRSP
 231          * FCP Status to MSI-X ISR for TRSP cmpl.
 232          * Segment 9 - Time from MSI-X ISR for TRSP cmpl to
 233          * TRSP cmpl is passed to NVME Layer.
 234          * Segment 10 - Time from FCP command received by
 235          * MSI-X ISR to command is completed on wire.
 236          * (Segments 1 thru 8) for READDATA / WRITEDATA
 237          * (Segments 1 thru 4) for READDATA_RSP
 238          */
 239         seg1 = ctxp->ts_cmd_nvme - ctxp->ts_isr_cmd;
 240         seg2 = (ctxp->ts_nvme_data - ctxp->ts_isr_cmd) - seg1;
 241         seg3 = (ctxp->ts_data_wqput - ctxp->ts_isr_cmd) -
 242                 seg1 - seg2;
 243         seg4 = (ctxp->ts_isr_data - ctxp->ts_isr_cmd) -
 244                 seg1 - seg2 - seg3;
 245         seg5 = (ctxp->ts_data_nvme - ctxp->ts_isr_cmd) -
 246                 seg1 - seg2 - seg3 - seg4;
 247
 248         /* For auto rsp commands seg6 thru seg10 will be 0 */
 249         if (ctxp->ts_nvme_status > ctxp->ts_data_nvme) {
 250                 seg6 = (ctxp->ts_nvme_status -
 251                         ctxp->ts_isr_cmd) -
 252                         seg1 - seg2 - seg3 - seg4 - seg5;
 253                 seg7 = (ctxp->ts_status_wqput -
 254                         ctxp->ts_isr_cmd) -
 255                         seg1 - seg2 - seg3 -
 256                         seg4 - seg5 - seg6;
 257                 seg8 = (ctxp->ts_isr_status -
 258                         ctxp->ts_isr_cmd) -
 259                         seg1 - seg2 - seg3 - seg4 -
 260                         seg5 - seg6 - seg7;
 261                 seg9 = (ctxp->ts_status_nvme -
 262                         ctxp->ts_isr_cmd) -
 263                         seg1 - seg2 - seg3 - seg4 -
 264                         seg5 - seg6 - seg7 - seg8;
 265                 seg10 = (ctxp->ts_isr_status -
 266                         ctxp->ts_isr_cmd);
 267         } else {
 268                 seg6 =  0;
 269                 seg7 =  0;
 270                 seg8 =  0;
 271                 seg9 =  0;
 272                 seg10 = (ctxp->ts_isr_data - ctxp->ts_isr_cmd);
 273         }
 274
 275         phba->ktime_seg1_total += seg1;
 276         if (seg1 < phba->ktime_seg1_min)
 277                 phba->ktime_seg1_min = seg1;
 278         else if (seg1 > phba->ktime_seg1_max)
 279                 phba->ktime_seg1_max = seg1;
 280
 281         phba->ktime_seg2_total += seg2;
 282         if (seg2 < phba->ktime_seg2_min)
 283                 phba->ktime_seg2_min = seg2;
 284         else if (seg2 > phba->ktime_seg2_max)
 285                 phba->ktime_seg2_max = seg2;
 286
 287         phba->ktime_seg3_total += seg3;
 288         if (seg3 < phba->ktime_seg3_min)
 289                 phba->ktime_seg3_min = seg3;
 290         else if (seg3 > phba->ktime_seg3_max)
 291                 phba->ktime_seg3_max = seg3;
 292
 293         phba->ktime_seg4_total += seg4;
 294         if (seg4 < phba->ktime_seg4_min)
 295                 phba->ktime_seg4_min = seg4;
 296         else if (seg4 > phba->ktime_seg4_max)
 297                 phba->ktime_seg4_max = seg4;
 298
 299         phba->ktime_seg5_total += seg5;
 300         if (seg5 < phba->ktime_seg5_min)
 301                 phba->ktime_seg5_min = seg5;
 302         else if (seg5 > phba->ktime_seg5_max)
 303                 phba->ktime_seg5_max = seg5;
 304
 305         phba->ktime_data_samples++;
 306         if (!seg6)
 307                 goto out;
 308
 309         phba->ktime_seg6_total += seg6;
 310         if (seg6 < phba->ktime_seg6_min)
 311                 phba->ktime_seg6_min = seg6;
 312         else if (seg6 > phba->ktime_seg6_max)
 313                 phba->ktime_seg6_max = seg6;
 314
 315         phba->ktime_seg7_total += seg7;
 316         if (seg7 < phba->ktime_seg7_min)
 317                 phba->ktime_seg7_min = seg7;
 318         else if (seg7 > phba->ktime_seg7_max)
 319                 phba->ktime_seg7_max = seg7;
 320
 321         phba->ktime_seg8_total += seg8;
 322         if (seg8 < phba->ktime_seg8_min)
 323                 phba->ktime_seg8_min = seg8;
 324         else if (seg8 > phba->ktime_seg8_max)
 325                 phba->ktime_seg8_max = seg8;
 326
 327         phba->ktime_seg9_total += seg9;
 328         if (seg9 < phba->ktime_seg9_min)
 329                 phba->ktime_seg9_min = seg9;
 330         else if (seg9 > phba->ktime_seg9_max)
 331                 phba->ktime_seg9_max = seg9;
 332 out:
 333         phba->ktime_seg10_total += seg10;
 334         if (seg10 < phba->ktime_seg10_min)
 335                 phba->ktime_seg10_min = seg10;
 336         else if (seg10 > phba->ktime_seg10_max)
 337                 phba->ktime_seg10_max = seg10;
 338         phba->ktime_status_samples++;
 339 }
 340 #endif
 341
 342 /**
 343  * lpfc_nvmet_xmt_fcp_op_cmp - Completion handler for FCP Response
 344  * @phba: Pointer to HBA context object.
 345  * @cmdwqe: Pointer to driver command WQE object.
 346  * @wcqe: Pointer to driver response CQE object.
 347  *
 348  * The function is called from SLI ring event handler with no
 349  * lock held. This function is the completion handler for NVME FCP commands
 350  * The function frees memory resources used for the NVME commands.
 351  **/
 352 static void
 353 lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
 354                           struct lpfc_wcqe_complete *wcqe)
 355 {
 356         struct lpfc_nvmet_tgtport *tgtp;
 357         struct nvmefc_tgt_fcp_req *rsp;
 358         struct lpfc_nvmet_rcv_ctx *ctxp;
 359         uint32_t status, result, op, start_clean;
 360 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 361         uint32_t id;
 362 #endif
 363
 364         ctxp = cmdwqe->context2;
 365         ctxp->flag &= ~LPFC_NVMET_IO_INP;
 366
 367         rsp = &ctxp->ctx.fcp_req;
 368         op = rsp->op;
 369
 370         status = bf_get(lpfc_wcqe_c_status, wcqe);
 371         result = wcqe->parameter;
 372
 373         if (phba->targetport)
 374                 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
 375         else
 376                 tgtp = NULL;
 377
 378         lpfc_nvmeio_data(phba, "NVMET FCP CMPL: xri x%x op x%x status x%x\n",
 379                          ctxp->oxid, op, status);
 380
 381         if (status) {
 382                 rsp->fcp_error = NVME_SC_DATA_XFER_ERROR;
 383                 rsp->transferred_length = 0;
 384                 if (tgtp)
 385                         atomic_inc(&tgtp->xmt_fcp_rsp_error);
 386
 387                 /* pick up SLI4 exhange busy condition */
 388                 if (bf_get(lpfc_wcqe_c_xb, wcqe)) {
 389                         ctxp->flag |= LPFC_NVMET_XBUSY;
 390
 391                         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
 392                                         "6315 IO Cmpl XBUSY: xri x%x: %x/%x\n",
 393                                         ctxp->oxid, status, result);
 394                 } else {
 395                         ctxp->flag &= ~LPFC_NVMET_XBUSY;
 396                 }
 397
 398         } else {
 399                 rsp->fcp_error = NVME_SC_SUCCESS;
 400                 if (op == NVMET_FCOP_RSP)
 401                         rsp->transferred_length = rsp->rsplen;
 402                 else
 403                         rsp->transferred_length = rsp->transfer_length;
 404                 if (tgtp)
 405                         atomic_inc(&tgtp->xmt_fcp_rsp_cmpl);
 406         }
 407
 408         if ((op == NVMET_FCOP_READDATA_RSP) ||
 409             (op == NVMET_FCOP_RSP)) {
 410                 /* Sanity check */
 411                 ctxp->state = LPFC_NVMET_STE_DONE;
 412                 ctxp->entry_cnt++;
 413
 414 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 415                 if (phba->ktime_on) {
 416                         if (rsp->op == NVMET_FCOP_READDATA_RSP) {
 417                                 ctxp->ts_isr_data =
 418                                         cmdwqe->isr_timestamp;
 419                                 ctxp->ts_data_nvme =
 420                                         ktime_get_ns();
 421                                 ctxp->ts_nvme_status =
 422                                         ctxp->ts_data_nvme;
 423                                 ctxp->ts_status_wqput =
 424                                         ctxp->ts_data_nvme;
 425                                 ctxp->ts_isr_status =
 426                                         ctxp->ts_data_nvme;
 427                                 ctxp->ts_status_nvme =
 428                                         ctxp->ts_data_nvme;
 429                         } else {
 430                                 ctxp->ts_isr_status =
 431                                         cmdwqe->isr_timestamp;
 432                                 ctxp->ts_status_nvme =
 433                                         ktime_get_ns();
 434                         }
 435                 }
 436                 if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) {
 437                         id = smp_processor_id();
 438                         if (ctxp->cpu != id)
 439                                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 440                                                 "6703 CPU Check cmpl: "
 441                                                 "cpu %d expect %d\n",
 442                                                 id, ctxp->cpu);
 443                         if (ctxp->cpu < LPFC_CHECK_CPU_CNT)
 444                                 phba->cpucheck_cmpl_io[id]++;
 445                 }
 446 #endif
 447                 rsp->done(rsp);
 448 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 449                 if (phba->ktime_on)
 450                         lpfc_nvmet_ktime(phba, ctxp);
 451 #endif
 452                 /* lpfc_nvmet_xmt_fcp_release() will recycle the context */
 453         } else {
 454                 ctxp->entry_cnt++;
 455                 start_clean = offsetof(struct lpfc_iocbq, wqe);
 456                 memset(((char *)cmdwqe) + start_clean, 0,
 457                        (sizeof(struct lpfc_iocbq) - start_clean));
 458 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 459                 if (phba->ktime_on) {
 460                         ctxp->ts_isr_data = cmdwqe->isr_timestamp;
 461                         ctxp->ts_data_nvme = ktime_get_ns();
 462                 }
 463                 if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) {
 464                         id = smp_processor_id();
 465                         if (ctxp->cpu != id)
 466                                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 467                                                 "6704 CPU Check cmdcmpl: "
 468                                                 "cpu %d expect %d\n",
 469                                                 id, ctxp->cpu);
 470                         if (ctxp->cpu < LPFC_CHECK_CPU_CNT)
 471                                 phba->cpucheck_ccmpl_io[id]++;
 472                 }
 473 #endif
 474                 rsp->done(rsp);
 475         }
 476 }
 477
 478 static int
 479 lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport,
 480                       struct nvmefc_tgt_ls_req *rsp)
 481 {
 482         struct lpfc_nvmet_rcv_ctx *ctxp =
 483                 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.ls_req);
 484         struct lpfc_hba *phba = ctxp->phba;
 485         struct hbq_dmabuf *nvmebuf =
 486                 (struct hbq_dmabuf *)ctxp->rqb_buffer;
 487         struct lpfc_iocbq *nvmewqeq;
 488         struct lpfc_nvmet_tgtport *nvmep = tgtport->private;
 489         struct lpfc_dmabuf dmabuf;
 490         struct ulp_bde64 bpl;
 491         int rc;
 492
 493         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
 494                         "6023 %s: Entrypoint ctx %p %p\n", __func__,
 495                         ctxp, tgtport);
 496
 497         nvmewqeq = lpfc_nvmet_prep_ls_wqe(phba, ctxp, rsp->rspdma,
 498                                       rsp->rsplen);
 499         if (nvmewqeq == NULL) {
 500                 atomic_inc(&nvmep->xmt_ls_drop);
 501                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 502                                 "6150 LS Drop IO x%x: Prep\n",
 503                                 ctxp->oxid);
 504                 lpfc_in_buf_free(phba, &nvmebuf->dbuf);
 505                 lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp,
 506                                                 ctxp->sid, ctxp->oxid);
 507                 return -ENOMEM;
 508         }
 509
 510         /* Save numBdes for bpl2sgl */
 511         nvmewqeq->rsvd2 = 1;
 512         nvmewqeq->hba_wqidx = 0;
 513         nvmewqeq->context3 = &dmabuf;
 514         dmabuf.virt = &bpl;
 515         bpl.addrLow = nvmewqeq->wqe.xmit_sequence.bde.addrLow;
 516         bpl.addrHigh = nvmewqeq->wqe.xmit_sequence.bde.addrHigh;
 517         bpl.tus.f.bdeSize = rsp->rsplen;
 518         bpl.tus.f.bdeFlags = 0;
 519         bpl.tus.w = le32_to_cpu(bpl.tus.w);
 520
 521         nvmewqeq->wqe_cmpl = lpfc_nvmet_xmt_ls_rsp_cmp;
 522         nvmewqeq->iocb_cmpl = NULL;
 523         nvmewqeq->context2 = ctxp;
 524
 525         lpfc_nvmeio_data(phba, "NVMET LS  RESP: xri x%x wqidx x%x len x%x\n",
 526                          ctxp->oxid, nvmewqeq->hba_wqidx, rsp->rsplen);
 527
 528         rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, nvmewqeq);
 529         if (rc == WQE_SUCCESS) {
 530                 /*
 531                  * Okay to repost buffer here, but wait till cmpl
 532                  * before freeing ctxp and iocbq.
 533                  */
 534                 lpfc_in_buf_free(phba, &nvmebuf->dbuf);
 535                 ctxp->rqb_buffer = 0;
 536                 atomic_inc(&nvmep->xmt_ls_rsp);
 537                 return 0;
 538         }
 539         /* Give back resources */
 540         atomic_inc(&nvmep->xmt_ls_drop);
 541         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 542                         "6151 LS Drop IO x%x: Issue %d\n",
 543                         ctxp->oxid, rc);
 544
 545         lpfc_nlp_put(nvmewqeq->context1);
 546
 547         lpfc_in_buf_free(phba, &nvmebuf->dbuf);
 548         lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid);
 549         return -ENXIO;
 550 }
 551
 552 static int
 553 lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport,
 554                       struct nvmefc_tgt_fcp_req *rsp)
 555 {
 556         struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
 557         struct lpfc_nvmet_rcv_ctx *ctxp =
 558                 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req);
 559         struct lpfc_hba *phba = ctxp->phba;
 560         struct lpfc_iocbq *nvmewqeq;
 561         int rc;
 562
 563 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 564         if (phba->ktime_on) {
 565                 if (rsp->op == NVMET_FCOP_RSP)
 566                         ctxp->ts_nvme_status = ktime_get_ns();
 567                 else
 568                         ctxp->ts_nvme_data = ktime_get_ns();
 569         }
 570         if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) {
 571                 int id = smp_processor_id();
 572                 ctxp->cpu = id;
 573                 if (id < LPFC_CHECK_CPU_CNT)
 574                         phba->cpucheck_xmt_io[id]++;
 575                 if (rsp->hwqid != id) {
 576                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 577                                         "6705 CPU Check OP: "
 578                                         "cpu %d expect %d\n",
 579                                         id, rsp->hwqid);
 580                         ctxp->cpu = rsp->hwqid;
 581                 }
 582         }
 583 #endif
 584
 585         /* Sanity check */
 586         if ((ctxp->flag & LPFC_NVMET_ABTS_RCV) ||
 587             (ctxp->state == LPFC_NVMET_STE_ABORT)) {
 588                 atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
 589                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 590                                 "6102 IO xri x%x aborted\n",
 591                                 ctxp->oxid);
 592                 rc = -ENXIO;
 593                 goto aerr;
 594         }
 595
 596         nvmewqeq = lpfc_nvmet_prep_fcp_wqe(phba, ctxp);
 597         if (nvmewqeq == NULL) {
 598                 atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
 599                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 600                                 "6152 FCP Drop IO x%x: Prep\n",
 601                                 ctxp->oxid);
 602                 rc = -ENXIO;
 603                 goto aerr;
 604         }
 605
 606         nvmewqeq->wqe_cmpl = lpfc_nvmet_xmt_fcp_op_cmp;
 607         nvmewqeq->iocb_cmpl = NULL;
 608         nvmewqeq->context2 = ctxp;
 609         nvmewqeq->iocb_flag |=  LPFC_IO_NVMET;
 610         ctxp->wqeq->hba_wqidx = rsp->hwqid;
 611
 612         lpfc_nvmeio_data(phba, "NVMET FCP CMND: xri x%x op x%x len x%x\n",
 613                          ctxp->oxid, rsp->op, rsp->rsplen);
 614
 615         rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, nvmewqeq);
 616         if (rc == WQE_SUCCESS) {
 617                 ctxp->flag |= LPFC_NVMET_IO_INP;
 618 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 619                 if (!phba->ktime_on)
 620                         return 0;
 621                 if (rsp->op == NVMET_FCOP_RSP)
 622                         ctxp->ts_status_wqput = ktime_get_ns();
 623                 else
 624                         ctxp->ts_data_wqput = ktime_get_ns();
 625 #endif
 626                 return 0;
 627         }
 628
 629         /* Give back resources */
 630         atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
 631         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 632                         "6153 FCP Drop IO x%x: Issue: %d\n",
 633                         ctxp->oxid, rc);
 634
 635         ctxp->wqeq->hba_wqidx = 0;
 636         nvmewqeq->context2 = NULL;
 637         nvmewqeq->context3 = NULL;
 638         rc = -EBUSY;
 639 aerr:
 640         return rc;
 641 }
 642
 643 static void
 644 lpfc_nvmet_targetport_delete(struct nvmet_fc_target_port *targetport)
 645 {
 646         struct lpfc_nvmet_tgtport *tport = targetport->private;
 647
 648         /* release any threads waiting for the unreg to complete */
 649         complete(&tport->tport_unreg_done);
 650 }
 651
 652 static void
 653 lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport,
 654                          struct nvmefc_tgt_fcp_req *req)
 655 {
 656         struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
 657         struct lpfc_nvmet_rcv_ctx *ctxp =
 658                 container_of(req, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req);
 659         struct lpfc_hba *phba = ctxp->phba;
 660         unsigned long flags;
 661
 662         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
 663                         "6103 Abort op: oxri x%x flg x%x cnt %d\n",
 664                         ctxp->oxid, ctxp->flag, ctxp->entry_cnt);
 665
 666         lpfc_nvmeio_data(phba, "NVMET FCP ABRT: "
 667                          "xri x%x flg x%x cnt x%x\n",
 668                          ctxp->oxid, ctxp->flag, ctxp->entry_cnt);
 669
 670         atomic_inc(&lpfc_nvmep->xmt_fcp_abort);
 671         ctxp->entry_cnt++;
 672         spin_lock_irqsave(&ctxp->ctxlock, flags);
 673
 674         /* Since iaab/iaar are NOT set, we need to check
 675          * if the firmware is in process of aborting IO
 676          */
 677         if (ctxp->flag & LPFC_NVMET_XBUSY) {
 678                 spin_unlock_irqrestore(&ctxp->ctxlock, flags);
 679                 return;
 680         }
 681         ctxp->flag |= LPFC_NVMET_ABORT_OP;
 682         if (ctxp->flag & LPFC_NVMET_IO_INP)
 683                 lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid,
 684                                                ctxp->oxid);
 685         else
 686                 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid,
 687                                                  ctxp->oxid);
 688         spin_unlock_irqrestore(&ctxp->ctxlock, flags);
 689 }
 690
 691 static void
 692 lpfc_nvmet_xmt_fcp_release(struct nvmet_fc_target_port *tgtport,
 693                            struct nvmefc_tgt_fcp_req *rsp)
 694 {
 695         struct lpfc_nvmet_rcv_ctx *ctxp =
 696                 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req);
 697         struct lpfc_hba *phba = ctxp->phba;
 698         unsigned long flags;
 699         bool aborting = false;
 700
 701         spin_lock_irqsave(&ctxp->ctxlock, flags);
 702         if ((ctxp->flag & LPFC_NVMET_ABORT_OP) ||
 703             (ctxp->flag & LPFC_NVMET_XBUSY)) {
 704                 aborting = true;
 705                 /* let the abort path do the real release */
 706                 lpfc_nvmet_defer_release(phba, ctxp);
 707         }
 708         spin_unlock_irqrestore(&ctxp->ctxlock, flags);
 709
 710         lpfc_nvmeio_data(phba, "NVMET FCP FREE: xri x%x ste %d\n", ctxp->oxid,
 711                          ctxp->state, 0);
 712
 713         if (aborting)
 714                 return;
 715
 716         lpfc_nvmet_rq_post(phba, ctxp, &ctxp->rqb_buffer->hbuf);
 717 }
 718
 719 static struct nvmet_fc_target_template lpfc_tgttemplate = {
 720         .targetport_delete = lpfc_nvmet_targetport_delete,
 721         .xmt_ls_rsp     = lpfc_nvmet_xmt_ls_rsp,
 722         .fcp_op         = lpfc_nvmet_xmt_fcp_op,
 723         .fcp_abort      = lpfc_nvmet_xmt_fcp_abort,
 724         .fcp_req_release = lpfc_nvmet_xmt_fcp_release,
 725
 726         .max_hw_queues  = 1,
 727         .max_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,
 728         .max_dif_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,
 729         .dma_boundary = 0xFFFFFFFF,
 730
 731         /* optional features */
 732         .target_features = 0,
 733         /* sizes of additional private data for data structures */
 734         .target_priv_sz = sizeof(struct lpfc_nvmet_tgtport),
 735 };
 736
 737 int
 738 lpfc_nvmet_create_targetport(struct lpfc_hba *phba)
 739 {
 740         struct lpfc_vport  *vport = phba->pport;
 741         struct lpfc_nvmet_tgtport *tgtp;
 742         struct nvmet_fc_port_info pinfo;
 743         int error = 0;
 744
 745         if (phba->targetport)
 746                 return 0;
 747
 748         memset(&pinfo, 0, sizeof(struct nvmet_fc_port_info));
 749         pinfo.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);
 750         pinfo.port_name = wwn_to_u64(vport->fc_portname.u.wwn);
 751         pinfo.port_id = vport->fc_myDID;
 752
 753         /* Limit to LPFC_MAX_NVME_SEG_CNT.
 754          * For now need + 1 to get around NVME transport logic.
 755          */
 756         if (phba->cfg_sg_seg_cnt > LPFC_MAX_NVME_SEG_CNT) {
 757                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_INIT,
 758                                 "6400 Reducing sg segment cnt to %d\n",
 759                                 LPFC_MAX_NVME_SEG_CNT);
 760                 phba->cfg_nvme_seg_cnt = LPFC_MAX_NVME_SEG_CNT;
 761         } else {
 762                 phba->cfg_nvme_seg_cnt = phba->cfg_sg_seg_cnt;
 763         }
 764         lpfc_tgttemplate.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
 765         lpfc_tgttemplate.max_hw_queues = phba->cfg_nvme_io_channel;
 766         lpfc_tgttemplate.target_features = NVMET_FCTGTFEAT_READDATA_RSP |
 767                                            NVMET_FCTGTFEAT_CMD_IN_ISR |
 768                                            NVMET_FCTGTFEAT_OPDONE_IN_ISR;
 769
 770 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
 771         error = nvmet_fc_register_targetport(&pinfo, &lpfc_tgttemplate,
 772                                              &phba->pcidev->dev,
 773                                              &phba->targetport);
 774 #else
 775         error = -ENOMEM;
 776 #endif
 777         if (error) {
 778                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
 779                                 "6025 Cannot register NVME targetport "
 780                                 "x%x\n", error);
 781                 phba->targetport = NULL;
 782         } else {
 783                 tgtp = (struct lpfc_nvmet_tgtport *)
 784                         phba->targetport->private;
 785                 tgtp->phba = phba;
 786
 787                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
 788                                 "6026 Registered NVME "
 789                                 "targetport: %p, private %p "
 790                                 "portnm %llx nodenm %llx\n",
 791                                 phba->targetport, tgtp,
 792                                 pinfo.port_name, pinfo.node_name);
 793
 794                 atomic_set(&tgtp->rcv_ls_req_in, 0);
 795                 atomic_set(&tgtp->rcv_ls_req_out, 0);
 796                 atomic_set(&tgtp->rcv_ls_req_drop, 0);
 797                 atomic_set(&tgtp->xmt_ls_abort, 0);
 798                 atomic_set(&tgtp->xmt_ls_rsp, 0);
 799                 atomic_set(&tgtp->xmt_ls_drop, 0);
 800                 atomic_set(&tgtp->xmt_ls_rsp_error, 0);
 801                 atomic_set(&tgtp->xmt_ls_rsp_cmpl, 0);
 802                 atomic_set(&tgtp->rcv_fcp_cmd_in, 0);
 803                 atomic_set(&tgtp->rcv_fcp_cmd_out, 0);
 804                 atomic_set(&tgtp->rcv_fcp_cmd_drop, 0);
 805                 atomic_set(&tgtp->xmt_fcp_abort, 0);
 806                 atomic_set(&tgtp->xmt_fcp_drop, 0);
 807                 atomic_set(&tgtp->xmt_fcp_read_rsp, 0);
 808                 atomic_set(&tgtp->xmt_fcp_read, 0);
 809                 atomic_set(&tgtp->xmt_fcp_write, 0);
 810                 atomic_set(&tgtp->xmt_fcp_rsp, 0);
 811                 atomic_set(&tgtp->xmt_fcp_rsp_cmpl, 0);
 812                 atomic_set(&tgtp->xmt_fcp_rsp_error, 0);
 813                 atomic_set(&tgtp->xmt_fcp_rsp_drop, 0);
 814                 atomic_set(&tgtp->xmt_abort_rsp, 0);
 815                 atomic_set(&tgtp->xmt_abort_rsp_error, 0);
 816                 atomic_set(&tgtp->xmt_abort_cmpl, 0);
 817         }
 818         return error;
 819 }
 820
 821 int
 822 lpfc_nvmet_update_targetport(struct lpfc_hba *phba)
 823 {
 824         struct lpfc_vport  *vport = phba->pport;
 825
 826         if (!phba->targetport)
 827                 return 0;
 828
 829         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
 830                          "6007 Update NVMET port %p did x%x\n",
 831                          phba->targetport, vport->fc_myDID);
 832
 833         phba->targetport->port_id = vport->fc_myDID;
 834         return 0;
 835 }
 836
 837 /**
 838  * lpfc_sli4_nvmet_xri_aborted - Fast-path process of nvmet xri abort
 839  * @phba: pointer to lpfc hba data structure.
 840  * @axri: pointer to the nvmet xri abort wcqe structure.
 841  *
 842  * This routine is invoked by the worker thread to process a SLI4 fast-path
 843  * NVMET aborted xri.
 844  **/
 845 void
 846 lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba,
 847                             struct sli4_wcqe_xri_aborted *axri)
 848 {
 849         uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);
 850         uint16_t rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri);
 851         struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp;
 852         struct lpfc_nodelist *ndlp;
 853         unsigned long iflag = 0;
 854         int rrq_empty = 0;
 855         bool released = false;
 856
 857         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
 858                         "6317 XB aborted xri x%x rxid x%x\n", xri, rxid);
 859
 860         if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME))
 861                 return;
 862         spin_lock_irqsave(&phba->hbalock, iflag);
 863         spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock);
 864         list_for_each_entry_safe(ctxp, next_ctxp,
 865                                  &phba->sli4_hba.lpfc_abts_nvmet_ctx_list,
 866                                  list) {
 867                 if (ctxp->rqb_buffer->sglq->sli4_xritag != xri)
 868                         continue;
 869
 870                 /* Check if we already received a free context call
 871                  * and we have completed processing an abort situation.
 872                  */
 873                 if (ctxp->flag & LPFC_NVMET_CTX_RLS &&
 874                     !(ctxp->flag & LPFC_NVMET_ABORT_OP)) {
 875                         list_del(&ctxp->list);
 876                         released = true;
 877                 }
 878                 ctxp->flag &= ~LPFC_NVMET_XBUSY;
 879                 spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
 880
 881                 rrq_empty = list_empty(&phba->active_rrq_list);
 882                 spin_unlock_irqrestore(&phba->hbalock, iflag);
 883                 ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
 884                 if (ndlp && NLP_CHK_NODE_ACT(ndlp) &&
 885                     (ndlp->nlp_state == NLP_STE_UNMAPPED_NODE ||
 886                      ndlp->nlp_state == NLP_STE_MAPPED_NODE)) {
 887                         lpfc_set_rrq_active(phba, ndlp,
 888                                 ctxp->rqb_buffer->sglq->sli4_lxritag,
 889                                 rxid, 1);
 890                         lpfc_sli4_abts_err_handler(phba, ndlp, axri);
 891                 }
 892
 893                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
 894                                 "6318 XB aborted %x flg x%x (%x)\n",
 895                                 ctxp->oxid, ctxp->flag, released);
 896                 if (released)
 897                         lpfc_nvmet_rq_post(phba, ctxp,
 898                                            &ctxp->rqb_buffer->hbuf);
 899                 if (rrq_empty)
 900                         lpfc_worker_wake_up(phba);
 901                 return;
 902         }
 903         spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
 904         spin_unlock_irqrestore(&phba->hbalock, iflag);
 905 }
 906
 907 int
 908 lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport,
 909                            struct fc_frame_header *fc_hdr)
 910
 911 {
 912 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
 913         struct lpfc_hba *phba = vport->phba;
 914         struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp;
 915         struct nvmefc_tgt_fcp_req *rsp;
 916         uint16_t xri;
 917         unsigned long iflag = 0;
 918
 919         xri = be16_to_cpu(fc_hdr->fh_ox_id);
 920
 921         spin_lock_irqsave(&phba->hbalock, iflag);
 922         spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock);
 923         list_for_each_entry_safe(ctxp, next_ctxp,
 924                                  &phba->sli4_hba.lpfc_abts_nvmet_ctx_list,
 925                                  list) {
 926                 if (ctxp->rqb_buffer->sglq->sli4_xritag != xri)
 927                         continue;
 928
 929                 spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
 930                 spin_unlock_irqrestore(&phba->hbalock, iflag);
 931
 932                 spin_lock_irqsave(&ctxp->ctxlock, iflag);
 933                 ctxp->flag |= LPFC_NVMET_ABTS_RCV;
 934                 spin_unlock_irqrestore(&ctxp->ctxlock, iflag);
 935
 936                 lpfc_nvmeio_data(phba,
 937                         "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",
 938                         xri, smp_processor_id(), 0);
 939
 940                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
 941                                 "6319 NVMET Rcv ABTS:acc xri x%x\n", xri);
 942
 943                 rsp = &ctxp->ctx.fcp_req;
 944                 nvmet_fc_rcv_fcp_abort(phba->targetport, rsp);
 945
 946                 /* Respond with BA_ACC accordingly */
 947                 lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 1);
 948                 return 0;
 949         }
 950         spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
 951         spin_unlock_irqrestore(&phba->hbalock, iflag);
 952
 953         lpfc_nvmeio_data(phba, "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",
 954                          xri, smp_processor_id(), 1);
 955
 956         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
 957                         "6320 NVMET Rcv ABTS:rjt xri x%x\n", xri);
 958
 959         /* Respond with BA_RJT accordingly */
 960         lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 0);
 961 #endif
 962         return 0;
 963 }
 964
 965 void
 966 lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba)
 967 {
 968 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
 969         struct lpfc_nvmet_tgtport *tgtp;
 970
 971         if (phba->nvmet_support == 0)
 972                 return;
 973         if (phba->targetport) {
 974                 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
 975                 init_completion(&tgtp->tport_unreg_done);
 976                 nvmet_fc_unregister_targetport(phba->targetport);
 977                 wait_for_completion_timeout(&tgtp->tport_unreg_done, 5);
 978         }
 979         phba->targetport = NULL;
 980 #endif
 981 }
 982
 983 /**
 984  * lpfc_nvmet_unsol_ls_buffer - Process an unsolicited event data buffer
 985  * @phba: pointer to lpfc hba data structure.
 986  * @pring: pointer to a SLI ring.
 987  * @nvmebuf: pointer to lpfc nvme command HBQ data structure.
 988  *
 989  * This routine is used for processing the WQE associated with a unsolicited
 990  * event. It first determines whether there is an existing ndlp that matches
 991  * the DID from the unsolicited WQE. If not, it will create a new one with
 992  * the DID from the unsolicited WQE. The ELS command from the unsolicited
 993  * WQE is then used to invoke the proper routine and to set up proper state
 994  * of the discovery state machine.
 995  **/
 996 static void
 997 lpfc_nvmet_unsol_ls_buffer(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 998                            struct hbq_dmabuf *nvmebuf)
 999 {
1000 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1001         struct lpfc_nvmet_tgtport *tgtp;
1002         struct fc_frame_header *fc_hdr;
1003         struct lpfc_nvmet_rcv_ctx *ctxp;
1004         uint32_t *payload;
1005         uint32_t size, oxid, sid, rc;
1006
1007         if (!nvmebuf || !phba->targetport) {
1008                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1009                                 "6154 LS Drop IO\n");
1010                 oxid = 0;
1011                 size = 0;
1012                 sid = 0;
1013                 goto dropit;
1014         }
1015
1016         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1017         payload = (uint32_t *)(nvmebuf->dbuf.virt);
1018         fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
1019         size = bf_get(lpfc_rcqe_length,  &nvmebuf->cq_event.cqe.rcqe_cmpl);
1020         oxid = be16_to_cpu(fc_hdr->fh_ox_id);
1021         sid = sli4_sid_from_fc_hdr(fc_hdr);
1022
1023         ctxp = kzalloc(sizeof(struct lpfc_nvmet_rcv_ctx), GFP_ATOMIC);
1024         if (ctxp == NULL) {
1025                 atomic_inc(&tgtp->rcv_ls_req_drop);
1026                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1027                                 "6155 LS Drop IO x%x: Alloc\n",
1028                                 oxid);
1029 dropit:
1030                 lpfc_nvmeio_data(phba, "NVMET LS  DROP: "
1031                                  "xri x%x sz %d from %06x\n",
1032                                  oxid, size, sid);
1033                 if (nvmebuf)
1034                         lpfc_in_buf_free(phba, &nvmebuf->dbuf);
1035                 return;
1036         }
1037         ctxp->phba = phba;
1038         ctxp->size = size;
1039         ctxp->oxid = oxid;
1040         ctxp->sid = sid;
1041         ctxp->wqeq = NULL;
1042         ctxp->state = LPFC_NVMET_STE_RCV;
1043         ctxp->rqb_buffer = (void *)nvmebuf;
1044
1045         lpfc_nvmeio_data(phba, "NVMET LS   RCV: xri x%x sz %d from %06x\n",
1046                          oxid, size, sid);
1047         /*
1048          * The calling sequence should be:
1049          * nvmet_fc_rcv_ls_req -> lpfc_nvmet_xmt_ls_rsp/cmp ->_req->done
1050          * lpfc_nvmet_xmt_ls_rsp_cmp should free the allocated ctxp.
1051          */
1052         atomic_inc(&tgtp->rcv_ls_req_in);
1053         rc = nvmet_fc_rcv_ls_req(phba->targetport, &ctxp->ctx.ls_req,
1054                                  payload, size);
1055
1056         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
1057                         "6037 %s: ctx %p sz %d rc %d: %08x %08x %08x "
1058                         "%08x %08x %08x\n", __func__, ctxp, size, rc,
1059                         *payload, *(payload+1), *(payload+2),
1060                         *(payload+3), *(payload+4), *(payload+5));
1061
1062         if (rc == 0) {
1063                 atomic_inc(&tgtp->rcv_ls_req_out);
1064                 return;
1065         }
1066
1067         lpfc_nvmeio_data(phba, "NVMET LS  DROP: xri x%x sz %d from %06x\n",
1068                          oxid, size, sid);
1069
1070         atomic_inc(&tgtp->rcv_ls_req_drop);
1071         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1072                         "6156 LS Drop IO x%x: nvmet_fc_rcv_ls_req %d\n",
1073                         ctxp->oxid, rc);
1074
1075         /* We assume a rcv'ed cmd ALWAYs fits into 1 buffer */
1076         if (nvmebuf)
1077                 lpfc_in_buf_free(phba, &nvmebuf->dbuf);
1078
1079         atomic_inc(&tgtp->xmt_ls_abort);
1080         lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, sid, oxid);
1081 #endif
1082 }
1083
1084 /**
1085  * lpfc_nvmet_unsol_fcp_buffer - Process an unsolicited event data buffer
1086  * @phba: pointer to lpfc hba data structure.
1087  * @pring: pointer to a SLI ring.
1088  * @nvmebuf: pointer to lpfc nvme command HBQ data structure.
1089  *
1090  * This routine is used for processing the WQE associated with a unsolicited
1091  * event. It first determines whether there is an existing ndlp that matches
1092  * the DID from the unsolicited WQE. If not, it will create a new one with
1093  * the DID from the unsolicited WQE. The ELS command from the unsolicited
1094  * WQE is then used to invoke the proper routine and to set up proper state
1095  * of the discovery state machine.
1096  **/
1097 static void
1098 lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba,
1099                             struct lpfc_sli_ring *pring,
1100                             struct rqb_dmabuf *nvmebuf,
1101                             uint64_t isr_timestamp)
1102 {
1103 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1104         struct lpfc_nvmet_rcv_ctx *ctxp;
1105         struct lpfc_nvmet_tgtport *tgtp;
1106         struct fc_frame_header *fc_hdr;
1107         uint32_t *payload;
1108         uint32_t size, oxid, sid, rc;
1109 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1110         uint32_t id;
1111 #endif
1112
1113         if (!nvmebuf || !phba->targetport) {
1114                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1115                                 "6157 FCP Drop IO\n");
1116                 oxid = 0;
1117                 size = 0;
1118                 sid = 0;
1119                 goto dropit;
1120         }
1121
1122
1123         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1124         payload = (uint32_t *)(nvmebuf->dbuf.virt);
1125         fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
1126         size = nvmebuf->bytes_recv;
1127         oxid = be16_to_cpu(fc_hdr->fh_ox_id);
1128         sid = sli4_sid_from_fc_hdr(fc_hdr);
1129
1130         ctxp = (struct lpfc_nvmet_rcv_ctx *)nvmebuf->context;
1131         if (ctxp == NULL) {
1132                 atomic_inc(&tgtp->rcv_fcp_cmd_drop);
1133                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1134                                 "6158 FCP Drop IO x%x: Alloc\n",
1135                                 oxid);
1136                 lpfc_nvmet_rq_post(phba, NULL, &nvmebuf->hbuf);
1137                 /* Cannot send ABTS without context */
1138                 return;
1139         }
1140         memset(ctxp, 0, sizeof(ctxp->ctx));
1141         ctxp->wqeq = NULL;
1142         ctxp->txrdy = NULL;
1143         ctxp->offset = 0;
1144         ctxp->phba = phba;
1145         ctxp->size = size;
1146         ctxp->oxid = oxid;
1147         ctxp->sid = sid;
1148         ctxp->state = LPFC_NVMET_STE_RCV;
1149         ctxp->rqb_buffer = nvmebuf;
1150         ctxp->entry_cnt = 1;
1151         ctxp->flag = 0;
1152         spin_lock_init(&ctxp->ctxlock);
1153
1154 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1155         if (phba->ktime_on) {
1156                 ctxp->ts_isr_cmd = isr_timestamp;
1157                 ctxp->ts_cmd_nvme = ktime_get_ns();
1158                 ctxp->ts_nvme_data = 0;
1159                 ctxp->ts_data_wqput = 0;
1160                 ctxp->ts_isr_data = 0;
1161                 ctxp->ts_data_nvme = 0;
1162                 ctxp->ts_nvme_status = 0;
1163                 ctxp->ts_status_wqput = 0;
1164                 ctxp->ts_isr_status = 0;
1165                 ctxp->ts_status_nvme = 0;
1166         }
1167
1168         if (phba->cpucheck_on & LPFC_CHECK_NVMET_RCV) {
1169                 id = smp_processor_id();
1170                 if (id < LPFC_CHECK_CPU_CNT)
1171                         phba->cpucheck_rcv_io[id]++;
1172         }
1173 #endif
1174
1175         lpfc_nvmeio_data(phba, "NVMET FCP  RCV: xri x%x sz %d CPU %02x\n",
1176                          oxid, size, smp_processor_id());
1177
1178         atomic_inc(&tgtp->rcv_fcp_cmd_in);
1179         /*
1180          * The calling sequence should be:
1181          * nvmet_fc_rcv_fcp_req -> lpfc_nvmet_xmt_fcp_op/cmp -> req->done
1182          * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp.
1183          */
1184         rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req,
1185                                   payload, size);
1186
1187         /* Process FCP command */
1188         if (rc == 0) {
1189                 atomic_inc(&tgtp->rcv_fcp_cmd_out);
1190                 return;
1191         }
1192
1193         atomic_inc(&tgtp->rcv_fcp_cmd_drop);
1194         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1195                         "6159 FCP Drop IO x%x: err x%x\n",
1196                         ctxp->oxid, rc);
1197 dropit:
1198         lpfc_nvmeio_data(phba, "NVMET FCP DROP: xri x%x sz %d from %06x\n",
1199                          oxid, size, sid);
1200         if (oxid) {
1201                 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid);
1202                 return;
1203         }
1204
1205         if (nvmebuf) {
1206                 nvmebuf->iocbq->hba_wqidx = 0;
1207                 /* We assume a rcv'ed cmd ALWAYs fits into 1 buffer */
1208                 lpfc_nvmet_rq_post(phba, NULL, &nvmebuf->hbuf);
1209         }
1210 #endif
1211 }
1212
1213 /**
1214  * lpfc_nvmet_unsol_ls_event - Process an unsolicited event from an nvme nport
1215  * @phba: pointer to lpfc hba data structure.
1216  * @pring: pointer to a SLI ring.
1217  * @nvmebuf: pointer to received nvme data structure.
1218  *
1219  * This routine is used to process an unsolicited event received from a SLI
1220  * (Service Level Interface) ring. The actual processing of the data buffer
1221  * associated with the unsolicited event is done by invoking the routine
1222  * lpfc_nvmet_unsol_ls_buffer() after properly set up the buffer from the
1223  * SLI RQ on which the unsolicited event was received.
1224  **/
1225 void
1226 lpfc_nvmet_unsol_ls_event(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
1227                           struct lpfc_iocbq *piocb)
1228 {
1229         struct lpfc_dmabuf *d_buf;
1230         struct hbq_dmabuf *nvmebuf;
1231
1232         d_buf = piocb->context2;
1233         nvmebuf = container_of(d_buf, struct hbq_dmabuf, dbuf);
1234
1235         if (phba->nvmet_support == 0) {
1236                 lpfc_in_buf_free(phba, &nvmebuf->dbuf);
1237                 return;
1238         }
1239         lpfc_nvmet_unsol_ls_buffer(phba, pring, nvmebuf);
1240 }
1241
1242 /**
1243  * lpfc_nvmet_unsol_fcp_event - Process an unsolicited event from an nvme nport
1244  * @phba: pointer to lpfc hba data structure.
1245  * @pring: pointer to a SLI ring.
1246  * @nvmebuf: pointer to received nvme data structure.
1247  *
1248  * This routine is used to process an unsolicited event received from a SLI
1249  * (Service Level Interface) ring. The actual processing of the data buffer
1250  * associated with the unsolicited event is done by invoking the routine
1251  * lpfc_nvmet_unsol_fcp_buffer() after properly set up the buffer from the
1252  * SLI RQ on which the unsolicited event was received.
1253  **/
1254 void
1255 lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba,
1256                            struct lpfc_sli_ring *pring,
1257                            struct rqb_dmabuf *nvmebuf,
1258                            uint64_t isr_timestamp)
1259 {
1260         if (phba->nvmet_support == 0) {
1261                 lpfc_nvmet_rq_post(phba, NULL, &nvmebuf->hbuf);
1262                 return;
1263         }
1264         lpfc_nvmet_unsol_fcp_buffer(phba, pring, nvmebuf,
1265                                     isr_timestamp);
1266 }
1267
1268 /**
1269  * lpfc_nvmet_prep_ls_wqe - Allocate and prepare a lpfc wqe data structure
1270  * @phba: pointer to a host N_Port data structure.
1271  * @ctxp: Context info for NVME LS Request
1272  * @rspbuf: DMA buffer of NVME command.
1273  * @rspsize: size of the NVME command.
1274  *
1275  * This routine is used for allocating a lpfc-WQE data structure from
1276  * the driver lpfc-WQE free-list and prepare the WQE with the parameters
1277  * passed into the routine for discovery state machine to issue an Extended
1278  * Link Service (NVME) commands. It is a generic lpfc-WQE allocation
1279  * and preparation routine that is used by all the discovery state machine
1280  * routines and the NVME command-specific fields will be later set up by
1281  * the individual discovery machine routines after calling this routine
1282  * allocating and preparing a generic WQE data structure. It fills in the
1283  * Buffer Descriptor Entries (BDEs), allocates buffers for both command
1284  * payload and response payload (if expected). The reference count on the
1285  * ndlp is incremented by 1 and the reference to the ndlp is put into
1286  * context1 of the WQE data structure for this WQE to hold the ndlp
1287  * reference for the command's callback function to access later.
1288  *
1289  * Return code
1290  *   Pointer to the newly allocated/prepared nvme wqe data structure
1291  *   NULL - when nvme wqe data structure allocation/preparation failed
1292  **/
1293 static struct lpfc_iocbq *
1294 lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *phba,
1295                        struct lpfc_nvmet_rcv_ctx *ctxp,
1296                        dma_addr_t rspbuf, uint16_t rspsize)
1297 {
1298         struct lpfc_nodelist *ndlp;
1299         struct lpfc_iocbq *nvmewqe;
1300         union lpfc_wqe *wqe;
1301
1302         if (!lpfc_is_link_up(phba)) {
1303                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
1304                                 "6104 lpfc_nvmet_prep_ls_wqe: link err: "
1305                                 "NPORT x%x oxid:x%x\n",
1306                                 ctxp->sid, ctxp->oxid);
1307                 return NULL;
1308         }
1309
1310         /* Allocate buffer for  command wqe */
1311         nvmewqe = lpfc_sli_get_iocbq(phba);
1312         if (nvmewqe == NULL) {
1313                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
1314                                 "6105 lpfc_nvmet_prep_ls_wqe: No WQE: "
1315                                 "NPORT x%x oxid:x%x\n",
1316                                 ctxp->sid, ctxp->oxid);
1317                 return NULL;
1318         }
1319
1320         ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
1321         if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
1322             ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
1323             (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
1324                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
1325                                 "6106 lpfc_nvmet_prep_ls_wqe: No ndlp: "
1326                                 "NPORT x%x oxid:x%x\n",
1327                                 ctxp->sid, ctxp->oxid);
1328                 goto nvme_wqe_free_wqeq_exit;
1329         }
1330         ctxp->wqeq = nvmewqe;
1331
1332         /* prevent preparing wqe with NULL ndlp reference */
1333         nvmewqe->context1 = lpfc_nlp_get(ndlp);
1334         if (nvmewqe->context1 == NULL)
1335                 goto nvme_wqe_free_wqeq_exit;
1336         nvmewqe->context2 = ctxp;
1337
1338         wqe = &nvmewqe->wqe;
1339         memset(wqe, 0, sizeof(union lpfc_wqe));
1340
1341         /* Words 0 - 2 */
1342         wqe->xmit_sequence.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1343         wqe->xmit_sequence.bde.tus.f.bdeSize = rspsize;
1344         wqe->xmit_sequence.bde.addrLow = le32_to_cpu(putPaddrLow(rspbuf));
1345         wqe->xmit_sequence.bde.addrHigh = le32_to_cpu(putPaddrHigh(rspbuf));
1346
1347         /* Word 3 */
1348
1349         /* Word 4 */
1350
1351         /* Word 5 */
1352         bf_set(wqe_dfctl, &wqe->xmit_sequence.wge_ctl, 0);
1353         bf_set(wqe_ls, &wqe->xmit_sequence.wge_ctl, 1);
1354         bf_set(wqe_la, &wqe->xmit_sequence.wge_ctl, 0);
1355         bf_set(wqe_rctl, &wqe->xmit_sequence.wge_ctl, FC_RCTL_ELS4_REP);
1356         bf_set(wqe_type, &wqe->xmit_sequence.wge_ctl, FC_TYPE_NVME);
1357
1358         /* Word 6 */
1359         bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com,
1360                phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
1361         bf_set(wqe_xri_tag, &wqe->xmit_sequence.wqe_com, nvmewqe->sli4_xritag);
1362
1363         /* Word 7 */
1364         bf_set(wqe_cmnd, &wqe->xmit_sequence.wqe_com,
1365                CMD_XMIT_SEQUENCE64_WQE);
1366         bf_set(wqe_ct, &wqe->xmit_sequence.wqe_com, SLI4_CT_RPI);
1367         bf_set(wqe_class, &wqe->xmit_sequence.wqe_com, CLASS3);
1368         bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0);
1369
1370         /* Word 8 */
1371         wqe->xmit_sequence.wqe_com.abort_tag = nvmewqe->iotag;
1372
1373         /* Word 9 */
1374         bf_set(wqe_reqtag, &wqe->xmit_sequence.wqe_com, nvmewqe->iotag);
1375         /* Needs to be set by caller */
1376         bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com, ctxp->oxid);
1377
1378         /* Word 10 */
1379         bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1);
1380         bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE);
1381         bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com,
1382                LPFC_WQE_LENLOC_WORD12);
1383         bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0);
1384
1385         /* Word 11 */
1386         bf_set(wqe_cqid, &wqe->xmit_sequence.wqe_com,
1387                LPFC_WQE_CQ_ID_DEFAULT);
1388         bf_set(wqe_cmd_type, &wqe->xmit_sequence.wqe_com,
1389                OTHER_COMMAND);
1390
1391         /* Word 12 */
1392         wqe->xmit_sequence.xmit_len = rspsize;
1393
1394         nvmewqe->retry = 1;
1395         nvmewqe->vport = phba->pport;
1396         nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT;
1397         nvmewqe->iocb_flag |= LPFC_IO_NVME_LS;
1398
1399         /* Xmit NVME response to remote NPORT <did> */
1400         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
1401                         "6039 Xmit NVME LS response to remote "
1402                         "NPORT x%x iotag:x%x oxid:x%x size:x%x\n",
1403                         ndlp->nlp_DID, nvmewqe->iotag, ctxp->oxid,
1404                         rspsize);
1405         return nvmewqe;
1406
1407 nvme_wqe_free_wqeq_exit:
1408         nvmewqe->context2 = NULL;
1409         nvmewqe->context3 = NULL;
1410         lpfc_sli_release_iocbq(phba, nvmewqe);
1411         return NULL;
1412 }
1413
1414
1415 static struct lpfc_iocbq *
1416 lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
1417                         struct lpfc_nvmet_rcv_ctx *ctxp)
1418 {
1419         struct nvmefc_tgt_fcp_req *rsp = &ctxp->ctx.fcp_req;
1420         struct lpfc_nvmet_tgtport *tgtp;
1421         struct sli4_sge *sgl;
1422         struct lpfc_nodelist *ndlp;
1423         struct lpfc_iocbq *nvmewqe;
1424         struct scatterlist *sgel;
1425         union lpfc_wqe128 *wqe;
1426         uint32_t *txrdy;
1427         dma_addr_t physaddr;
1428         int i, cnt;
1429         int xc = 1;
1430
1431         if (!lpfc_is_link_up(phba)) {
1432                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1433                                 "6107 lpfc_nvmet_prep_fcp_wqe: link err:"
1434                                 "NPORT x%x oxid:x%x\n", ctxp->sid,
1435                                 ctxp->oxid);
1436                 return NULL;
1437         }
1438
1439         ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
1440         if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
1441             ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
1442              (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
1443                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1444                                 "6108 lpfc_nvmet_prep_fcp_wqe: no ndlp: "
1445                                 "NPORT x%x oxid:x%x\n",
1446                                 ctxp->sid, ctxp->oxid);
1447                 return NULL;
1448         }
1449
1450         if (rsp->sg_cnt > phba->cfg_nvme_seg_cnt) {
1451                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1452                                 "6109 lpfc_nvmet_prep_fcp_wqe: seg cnt err: "
1453                                 "NPORT x%x oxid:x%x cnt %d\n",
1454                                 ctxp->sid, ctxp->oxid, phba->cfg_nvme_seg_cnt);
1455                 return NULL;
1456         }
1457
1458         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1459         nvmewqe = ctxp->wqeq;
1460         if (nvmewqe == NULL) {
1461                 /* Allocate buffer for  command wqe */
1462                 nvmewqe = ctxp->rqb_buffer->iocbq;
1463                 if (nvmewqe == NULL) {
1464                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1465                                         "6110 lpfc_nvmet_prep_fcp_wqe: No "
1466                                         "WQE: NPORT x%x oxid:x%x\n",
1467                                         ctxp->sid, ctxp->oxid);
1468                         return NULL;
1469                 }
1470                 ctxp->wqeq = nvmewqe;
1471                 xc = 0; /* create new XRI */
1472                 nvmewqe->sli4_lxritag = NO_XRI;
1473                 nvmewqe->sli4_xritag = NO_XRI;
1474         }
1475
1476         /* Sanity check */
1477         if (((ctxp->state == LPFC_NVMET_STE_RCV) &&
1478             (ctxp->entry_cnt == 1)) ||
1479             ((ctxp->state == LPFC_NVMET_STE_DATA) &&
1480             (ctxp->entry_cnt > 1))) {
1481                 wqe = (union lpfc_wqe128 *)&nvmewqe->wqe;
1482         } else {
1483                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1484                                 "6111 Wrong state %s: %d  cnt %d\n",
1485                                 __func__, ctxp->state, ctxp->entry_cnt);
1486                 return NULL;
1487         }
1488
1489         sgl  = (struct sli4_sge *)ctxp->rqb_buffer->sglq->sgl;
1490         switch (rsp->op) {
1491         case NVMET_FCOP_READDATA:
1492         case NVMET_FCOP_READDATA_RSP:
1493                 /* Words 0 - 2 : The first sg segment */
1494                 sgel = &rsp->sg[0];
1495                 physaddr = sg_dma_address(sgel);
1496                 wqe->fcp_tsend.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1497                 wqe->fcp_tsend.bde.tus.f.bdeSize = sg_dma_len(sgel);
1498                 wqe->fcp_tsend.bde.addrLow = cpu_to_le32(putPaddrLow(physaddr));
1499                 wqe->fcp_tsend.bde.addrHigh =
1500                         cpu_to_le32(putPaddrHigh(physaddr));
1501
1502                 /* Word 3 */
1503                 wqe->fcp_tsend.payload_offset_len = 0;
1504
1505                 /* Word 4 */
1506                 wqe->fcp_tsend.relative_offset = ctxp->offset;
1507
1508                 /* Word 5 */
1509
1510                 /* Word 6 */
1511                 bf_set(wqe_ctxt_tag, &wqe->fcp_tsend.wqe_com,
1512                        phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
1513                 bf_set(wqe_xri_tag, &wqe->fcp_tsend.wqe_com,
1514                        nvmewqe->sli4_xritag);
1515
1516                 /* Word 7 */
1517                 bf_set(wqe_cmnd, &wqe->fcp_tsend.wqe_com, CMD_FCP_TSEND64_WQE);
1518
1519                 /* Word 8 */
1520                 wqe->fcp_tsend.wqe_com.abort_tag = nvmewqe->iotag;
1521
1522                 /* Word 9 */
1523                 bf_set(wqe_reqtag, &wqe->fcp_tsend.wqe_com, nvmewqe->iotag);
1524                 bf_set(wqe_rcvoxid, &wqe->fcp_tsend.wqe_com, ctxp->oxid);
1525
1526                 /* Word 10 */
1527                 bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1);
1528                 bf_set(wqe_dbde, &wqe->fcp_tsend.wqe_com, 1);
1529                 bf_set(wqe_iod, &wqe->fcp_tsend.wqe_com, LPFC_WQE_IOD_WRITE);
1530                 bf_set(wqe_lenloc, &wqe->fcp_tsend.wqe_com,
1531                        LPFC_WQE_LENLOC_WORD12);
1532                 bf_set(wqe_ebde_cnt, &wqe->fcp_tsend.wqe_com, 0);
1533                 bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, xc);
1534                 bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1);
1535                 if (phba->cfg_nvme_oas)
1536                         bf_set(wqe_oas, &wqe->fcp_tsend.wqe_com, 1);
1537
1538                 /* Word 11 */
1539                 bf_set(wqe_cqid, &wqe->fcp_tsend.wqe_com,
1540                        LPFC_WQE_CQ_ID_DEFAULT);
1541                 bf_set(wqe_cmd_type, &wqe->fcp_tsend.wqe_com,
1542                        FCP_COMMAND_TSEND);
1543
1544                 /* Word 12 */
1545                 wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;
1546
1547                 /* Setup 2 SKIP SGEs */
1548                 sgl->addr_hi = 0;
1549                 sgl->addr_lo = 0;
1550                 sgl->word2 = 0;
1551                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
1552                 sgl->word2 = cpu_to_le32(sgl->word2);
1553                 sgl->sge_len = 0;
1554                 sgl++;
1555                 sgl->addr_hi = 0;
1556                 sgl->addr_lo = 0;
1557                 sgl->word2 = 0;
1558                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
1559                 sgl->word2 = cpu_to_le32(sgl->word2);
1560                 sgl->sge_len = 0;
1561                 sgl++;
1562                 if (rsp->op == NVMET_FCOP_READDATA_RSP) {
1563                         atomic_inc(&tgtp->xmt_fcp_read_rsp);
1564                         bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 1);
1565                         if ((ndlp->nlp_flag & NLP_SUPPRESS_RSP) &&
1566                             (rsp->rsplen == 12)) {
1567                                 bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 1);
1568                                 bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0);
1569                                 bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0);
1570                                 bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0);
1571                         } else {
1572                                 bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
1573                                 bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 1);
1574                                 bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 1);
1575                                 bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com,
1576                                        ((rsp->rsplen >> 2) - 1));
1577                                 memcpy(&wqe->words[16], rsp->rspaddr,
1578                                        rsp->rsplen);
1579                         }
1580                 } else {
1581                         atomic_inc(&tgtp->xmt_fcp_read);
1582
1583                         bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
1584                         bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0);
1585                         bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0);
1586                         bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 0);
1587                         bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0);
1588                 }
1589                 ctxp->state = LPFC_NVMET_STE_DATA;
1590                 break;
1591
1592         case NVMET_FCOP_WRITEDATA:
1593                 /* Words 0 - 2 : The first sg segment */
1594                 txrdy = pci_pool_alloc(phba->txrdy_payload_pool,
1595                                        GFP_KERNEL, &physaddr);
1596                 if (!txrdy) {
1597                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1598                                         "6041 Bad txrdy buffer: oxid x%x\n",
1599                                         ctxp->oxid);
1600                         return NULL;
1601                 }
1602                 ctxp->txrdy = txrdy;
1603                 ctxp->txrdy_phys = physaddr;
1604                 wqe->fcp_treceive.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1605                 wqe->fcp_treceive.bde.tus.f.bdeSize = TXRDY_PAYLOAD_LEN;
1606                 wqe->fcp_treceive.bde.addrLow =
1607                         cpu_to_le32(putPaddrLow(physaddr));
1608                 wqe->fcp_treceive.bde.addrHigh =
1609                         cpu_to_le32(putPaddrHigh(physaddr));
1610
1611                 /* Word 3 */
1612                 wqe->fcp_treceive.payload_offset_len = TXRDY_PAYLOAD_LEN;
1613
1614                 /* Word 4 */
1615                 wqe->fcp_treceive.relative_offset = ctxp->offset;
1616
1617                 /* Word 5 */
1618
1619                 /* Word 6 */
1620                 bf_set(wqe_ctxt_tag, &wqe->fcp_treceive.wqe_com,
1621                        phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
1622                 bf_set(wqe_xri_tag, &wqe->fcp_treceive.wqe_com,
1623                        nvmewqe->sli4_xritag);
1624
1625                 /* Word 7 */
1626                 bf_set(wqe_ar, &wqe->fcp_treceive.wqe_com, 0);
1627                 bf_set(wqe_cmnd, &wqe->fcp_treceive.wqe_com,
1628                        CMD_FCP_TRECEIVE64_WQE);
1629
1630                 /* Word 8 */
1631                 wqe->fcp_treceive.wqe_com.abort_tag = nvmewqe->iotag;
1632
1633                 /* Word 9 */
1634                 bf_set(wqe_reqtag, &wqe->fcp_treceive.wqe_com, nvmewqe->iotag);
1635                 bf_set(wqe_rcvoxid, &wqe->fcp_treceive.wqe_com, ctxp->oxid);
1636
1637                 /* Word 10 */
1638                 bf_set(wqe_nvme, &wqe->fcp_treceive.wqe_com, 1);
1639                 bf_set(wqe_dbde, &wqe->fcp_treceive.wqe_com, 1);
1640                 bf_set(wqe_iod, &wqe->fcp_treceive.wqe_com, LPFC_WQE_IOD_READ);
1641                 bf_set(wqe_lenloc, &wqe->fcp_treceive.wqe_com,
1642                        LPFC_WQE_LENLOC_WORD12);
1643                 bf_set(wqe_xc, &wqe->fcp_treceive.wqe_com, xc);
1644                 bf_set(wqe_wqes, &wqe->fcp_treceive.wqe_com, 0);
1645                 bf_set(wqe_irsp, &wqe->fcp_treceive.wqe_com, 0);
1646                 bf_set(wqe_irsplen, &wqe->fcp_treceive.wqe_com, 0);
1647                 bf_set(wqe_nvme, &wqe->fcp_treceive.wqe_com, 1);
1648                 if (phba->cfg_nvme_oas)
1649                         bf_set(wqe_oas, &wqe->fcp_treceive.wqe_com, 1);
1650
1651                 /* Word 11 */
1652                 bf_set(wqe_cqid, &wqe->fcp_treceive.wqe_com,
1653                        LPFC_WQE_CQ_ID_DEFAULT);
1654                 bf_set(wqe_cmd_type, &wqe->fcp_treceive.wqe_com,
1655                        FCP_COMMAND_TRECEIVE);
1656                 bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
1657
1658                 /* Word 12 */
1659                 wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;
1660
1661                 /* Setup 1 TXRDY and 1 SKIP SGE */
1662                 txrdy[0] = 0;
1663                 txrdy[1] = cpu_to_be32(rsp->transfer_length);
1664                 txrdy[2] = 0;
1665
1666                 sgl->addr_hi = putPaddrHigh(physaddr);
1667                 sgl->addr_lo = putPaddrLow(physaddr);
1668                 sgl->word2 = 0;
1669                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA);
1670                 sgl->word2 = cpu_to_le32(sgl->word2);
1671                 sgl->sge_len = cpu_to_le32(TXRDY_PAYLOAD_LEN);
1672                 sgl++;
1673                 sgl->addr_hi = 0;
1674                 sgl->addr_lo = 0;
1675                 sgl->word2 = 0;
1676                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
1677                 sgl->word2 = cpu_to_le32(sgl->word2);
1678                 sgl->sge_len = 0;
1679                 sgl++;
1680                 ctxp->state = LPFC_NVMET_STE_DATA;
1681                 atomic_inc(&tgtp->xmt_fcp_write);
1682                 break;
1683
1684         case NVMET_FCOP_RSP:
1685                 /* Words 0 - 2 */
1686                 physaddr = rsp->rspdma;
1687                 wqe->fcp_trsp.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1688                 wqe->fcp_trsp.bde.tus.f.bdeSize = rsp->rsplen;
1689                 wqe->fcp_trsp.bde.addrLow =
1690                         cpu_to_le32(putPaddrLow(physaddr));
1691                 wqe->fcp_trsp.bde.addrHigh =
1692                         cpu_to_le32(putPaddrHigh(physaddr));
1693
1694                 /* Word 3 */
1695                 wqe->fcp_trsp.response_len = rsp->rsplen;
1696
1697                 /* Word 4 */
1698                 wqe->fcp_trsp.rsvd_4_5[0] = 0;
1699
1700
1701                 /* Word 5 */
1702
1703                 /* Word 6 */
1704                 bf_set(wqe_ctxt_tag, &wqe->fcp_trsp.wqe_com,
1705                        phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
1706                 bf_set(wqe_xri_tag, &wqe->fcp_trsp.wqe_com,
1707                        nvmewqe->sli4_xritag);
1708
1709                 /* Word 7 */
1710                 bf_set(wqe_ag, &wqe->fcp_trsp.wqe_com, 1);
1711                 bf_set(wqe_cmnd, &wqe->fcp_trsp.wqe_com, CMD_FCP_TRSP64_WQE);
1712
1713                 /* Word 8 */
1714                 wqe->fcp_trsp.wqe_com.abort_tag = nvmewqe->iotag;
1715
1716                 /* Word 9 */
1717                 bf_set(wqe_reqtag, &wqe->fcp_trsp.wqe_com, nvmewqe->iotag);
1718                 bf_set(wqe_rcvoxid, &wqe->fcp_trsp.wqe_com, ctxp->oxid);
1719
1720                 /* Word 10 */
1721                 bf_set(wqe_nvme, &wqe->fcp_trsp.wqe_com, 1);
1722                 bf_set(wqe_dbde, &wqe->fcp_trsp.wqe_com, 0);
1723                 bf_set(wqe_iod, &wqe->fcp_trsp.wqe_com, LPFC_WQE_IOD_WRITE);
1724                 bf_set(wqe_lenloc, &wqe->fcp_trsp.wqe_com,
1725                        LPFC_WQE_LENLOC_WORD3);
1726                 bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, xc);
1727                 bf_set(wqe_nvme, &wqe->fcp_trsp.wqe_com, 1);
1728                 if (phba->cfg_nvme_oas)
1729                         bf_set(wqe_oas, &wqe->fcp_trsp.wqe_com, 1);
1730
1731                 /* Word 11 */
1732                 bf_set(wqe_cqid, &wqe->fcp_trsp.wqe_com,
1733                        LPFC_WQE_CQ_ID_DEFAULT);
1734                 bf_set(wqe_cmd_type, &wqe->fcp_trsp.wqe_com,
1735                        FCP_COMMAND_TRSP);
1736                 bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
1737                 ctxp->state = LPFC_NVMET_STE_RSP;
1738
1739                 if (rsp->rsplen == LPFC_NVMET_SUCCESS_LEN) {
1740                         /* Good response - all zero's on wire */
1741                         bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 0);
1742                         bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 0);
1743                         bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, 0);
1744                 } else {
1745                         bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 1);
1746                         bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 1);
1747                         bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com,
1748                                ((rsp->rsplen >> 2) - 1));
1749                         memcpy(&wqe->words[16], rsp->rspaddr, rsp->rsplen);
1750                 }
1751
1752                 /* Use rspbuf, NOT sg list */
1753                 rsp->sg_cnt = 0;
1754                 sgl->word2 = 0;
1755                 atomic_inc(&tgtp->xmt_fcp_rsp);
1756                 break;
1757
1758         default:
1759                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
1760                                 "6064 Unknown Rsp Op %d\n",
1761                                 rsp->op);
1762                 return NULL;
1763         }
1764
1765         nvmewqe->retry = 1;
1766         nvmewqe->vport = phba->pport;
1767         nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT;
1768         nvmewqe->context1 = ndlp;
1769
1770         for (i = 0; i < rsp->sg_cnt; i++) {
1771                 sgel = &rsp->sg[i];
1772                 physaddr = sg_dma_address(sgel);
1773                 cnt = sg_dma_len(sgel);
1774                 sgl->addr_hi = putPaddrHigh(physaddr);
1775                 sgl->addr_lo = putPaddrLow(physaddr);
1776                 sgl->word2 = 0;
1777                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA);
1778                 bf_set(lpfc_sli4_sge_offset, sgl, ctxp->offset);
1779                 if ((i+1) == rsp->sg_cnt)
1780                         bf_set(lpfc_sli4_sge_last, sgl, 1);
1781                 sgl->word2 = cpu_to_le32(sgl->word2);
1782                 sgl->sge_len = cpu_to_le32(cnt);
1783                 sgl++;
1784                 ctxp->offset += cnt;
1785         }
1786         return nvmewqe;
1787 }
1788
1789 /**
1790  * lpfc_nvmet_sol_fcp_abort_cmp - Completion handler for ABTS
1791  * @phba: Pointer to HBA context object.
1792  * @cmdwqe: Pointer to driver command WQE object.
1793  * @wcqe: Pointer to driver response CQE object.
1794  *
1795  * The function is called from SLI ring event handler with no
1796  * lock held. This function is the completion handler for NVME ABTS for FCP cmds
1797  * The function frees memory resources used for the NVME commands.
1798  **/
1799 static void
1800 lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
1801                              struct lpfc_wcqe_complete *wcqe)
1802 {
1803         struct lpfc_nvmet_rcv_ctx *ctxp;
1804         struct lpfc_nvmet_tgtport *tgtp;
1805         uint32_t status, result;
1806         unsigned long flags;
1807         bool released = false;
1808
1809         ctxp = cmdwqe->context2;
1810         status = bf_get(lpfc_wcqe_c_status, wcqe);
1811         result = wcqe->parameter;
1812
1813         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1814         atomic_inc(&tgtp->xmt_abort_cmpl);
1815
1816         ctxp->state = LPFC_NVMET_STE_DONE;
1817
1818         /* Check if we already received a free context call
1819          * and we have completed processing an abort situation.
1820          */
1821         spin_lock_irqsave(&ctxp->ctxlock, flags);
1822         if ((ctxp->flag & LPFC_NVMET_CTX_RLS) &&
1823             !(ctxp->flag & LPFC_NVMET_XBUSY)) {
1824                 list_del(&ctxp->list);
1825                 released = true;
1826         }
1827         ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
1828         spin_unlock_irqrestore(&ctxp->ctxlock, flags);
1829
1830         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
1831                         "6165 ABORT cmpl: xri x%x flg x%x (%d) "
1832                         "WCQE: %08x %08x %08x %08x\n",
1833                         ctxp->oxid, ctxp->flag, released,
1834                         wcqe->word0, wcqe->total_data_placed,
1835                         result, wcqe->word3);
1836
1837         /*
1838          * if transport has released ctx, then can reuse it. Otherwise,
1839          * will be recycled by transport release call.
1840          */
1841         if (released)
1842                 lpfc_nvmet_rq_post(phba, ctxp, &ctxp->rqb_buffer->hbuf);
1843
1844         cmdwqe->context2 = NULL;
1845         cmdwqe->context3 = NULL;
1846         lpfc_sli_release_iocbq(phba, cmdwqe);
1847
1848         /* Since iaab/iaar are NOT set, there is no work left.
1849          * For LPFC_NVMET_XBUSY, lpfc_sli4_nvmet_xri_aborted
1850          * should have been called already.
1851          */
1852 }
1853
1854 /**
1855  * lpfc_nvmet_unsol_fcp_abort_cmp - Completion handler for ABTS
1856  * @phba: Pointer to HBA context object.
1857  * @cmdwqe: Pointer to driver command WQE object.
1858  * @wcqe: Pointer to driver response CQE object.
1859  *
1860  * The function is called from SLI ring event handler with no
1861  * lock held. This function is the completion handler for NVME ABTS for FCP cmds
1862  * The function frees memory resources used for the NVME commands.
1863  **/
1864 static void
1865 lpfc_nvmet_unsol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
1866                                struct lpfc_wcqe_complete *wcqe)
1867 {
1868         struct lpfc_nvmet_rcv_ctx *ctxp;
1869         struct lpfc_nvmet_tgtport *tgtp;
1870         unsigned long flags;
1871         uint32_t status, result;
1872         bool released = false;
1873
1874         ctxp = cmdwqe->context2;
1875         status = bf_get(lpfc_wcqe_c_status, wcqe);
1876         result = wcqe->parameter;
1877
1878         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1879         atomic_inc(&tgtp->xmt_abort_cmpl);
1880
1881         if (!ctxp) {
1882                 /* if context is clear, related io alrady complete */
1883                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1884                                 "6070 ABTS cmpl: WCQE: %08x %08x %08x %08x\n",
1885                                 wcqe->word0, wcqe->total_data_placed,
1886                                 result, wcqe->word3);
1887                 return;
1888         }
1889
1890         /* Sanity check */
1891         if (ctxp->state != LPFC_NVMET_STE_ABORT) {
1892                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
1893                                 "6112 ABTS Wrong state:%d oxid x%x\n",
1894                                 ctxp->state, ctxp->oxid);
1895         }
1896
1897         /* Check if we already received a free context call
1898          * and we have completed processing an abort situation.
1899          */
1900         ctxp->state = LPFC_NVMET_STE_DONE;
1901         spin_lock_irqsave(&ctxp->ctxlock, flags);
1902         if ((ctxp->flag & LPFC_NVMET_CTX_RLS) &&
1903             !(ctxp->flag & LPFC_NVMET_XBUSY)) {
1904                 list_del(&ctxp->list);
1905                 released = true;
1906         }
1907         ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
1908         spin_unlock_irqrestore(&ctxp->ctxlock, flags);
1909
1910         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1911                         "6316 ABTS cmpl xri x%x flg x%x (%x) "
1912                         "WCQE: %08x %08x %08x %08x\n",
1913                         ctxp->oxid, ctxp->flag, released,
1914                         wcqe->word0, wcqe->total_data_placed,
1915                         result, wcqe->word3);
1916         /*
1917          * if transport has released ctx, then can reuse it. Otherwise,
1918          * will be recycled by transport release call.
1919          */
1920         if (released)
1921                 lpfc_nvmet_rq_post(phba, ctxp, &ctxp->rqb_buffer->hbuf);
1922
1923         cmdwqe->context2 = NULL;
1924         cmdwqe->context3 = NULL;
1925
1926         /* Since iaab/iaar are NOT set, there is no work left.
1927          * For LPFC_NVMET_XBUSY, lpfc_sli4_nvmet_xri_aborted
1928          * should have been called already.
1929          */
1930 }
1931
1932 /**
1933  * lpfc_nvmet_xmt_ls_abort_cmp - Completion handler for ABTS
1934  * @phba: Pointer to HBA context object.
1935  * @cmdwqe: Pointer to driver command WQE object.
1936  * @wcqe: Pointer to driver response CQE object.
1937  *
1938  * The function is called from SLI ring event handler with no
1939  * lock held. This function is the completion handler for NVME ABTS for LS cmds
1940  * The function frees memory resources used for the NVME commands.
1941  **/
1942 static void
1943 lpfc_nvmet_xmt_ls_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
1944                             struct lpfc_wcqe_complete *wcqe)
1945 {
1946         struct lpfc_nvmet_rcv_ctx *ctxp;
1947         struct lpfc_nvmet_tgtport *tgtp;
1948         uint32_t status, result;
1949
1950         ctxp = cmdwqe->context2;
1951         status = bf_get(lpfc_wcqe_c_status, wcqe);
1952         result = wcqe->parameter;
1953
1954         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1955         atomic_inc(&tgtp->xmt_abort_cmpl);
1956
1957         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1958                         "6083 Abort cmpl: ctx %p WCQE: %08x %08x %08x %08x\n",
1959                         ctxp, wcqe->word0, wcqe->total_data_placed,
1960                         result, wcqe->word3);
1961
1962         if (ctxp) {
1963                 cmdwqe->context2 = NULL;
1964                 cmdwqe->context3 = NULL;
1965                 lpfc_sli_release_iocbq(phba, cmdwqe);
1966                 kfree(ctxp);
1967         } else
1968                 lpfc_sli_release_iocbq(phba, cmdwqe);
1969 }
1970
1971 static int
1972 lpfc_nvmet_unsol_issue_abort(struct lpfc_hba *phba,
1973                              struct lpfc_nvmet_rcv_ctx *ctxp,
1974                              uint32_t sid, uint16_t xri)
1975 {
1976         struct lpfc_nvmet_tgtport *tgtp;
1977         struct lpfc_iocbq *abts_wqeq;
1978         union lpfc_wqe *wqe_abts;
1979         struct lpfc_nodelist *ndlp;
1980
1981         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1982                         "6067 ABTS: sid %x xri x%x/x%x\n",
1983                         sid, xri, ctxp->wqeq->sli4_xritag);
1984
1985         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1986         if (!ctxp->wqeq) {
1987                 ctxp->wqeq = ctxp->rqb_buffer->iocbq;
1988                 ctxp->wqeq->hba_wqidx = 0;
1989         }
1990
1991         ndlp = lpfc_findnode_did(phba->pport, sid);
1992         if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
1993             ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
1994             (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
1995                 atomic_inc(&tgtp->xmt_abort_rsp_error);
1996                 lpfc_printf_log(phba, KERN_WARNING, LOG_NVME_ABTS,
1997                                 "6134 Drop ABTS - wrong NDLP state x%x.\n",
1998                                 (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE);
1999
2000                 /* No failure to an ABTS request. */
2001                 return 0;
2002         }
2003
2004         abts_wqeq = ctxp->wqeq;
2005         wqe_abts = &abts_wqeq->wqe;
2006         ctxp->state = LPFC_NVMET_STE_ABORT;
2007
2008         /*
2009          * Since we zero the whole WQE, we need to ensure we set the WQE fields
2010          * that were initialized in lpfc_sli4_nvmet_alloc.
2011          */
2012         memset(wqe_abts, 0, sizeof(union lpfc_wqe));
2013
2014         /* Word 5 */
2015         bf_set(wqe_dfctl, &wqe_abts->xmit_sequence.wge_ctl, 0);
2016         bf_set(wqe_ls, &wqe_abts->xmit_sequence.wge_ctl, 1);
2017         bf_set(wqe_la, &wqe_abts->xmit_sequence.wge_ctl, 0);
2018         bf_set(wqe_rctl, &wqe_abts->xmit_sequence.wge_ctl, FC_RCTL_BA_ABTS);
2019         bf_set(wqe_type, &wqe_abts->xmit_sequence.wge_ctl, FC_TYPE_BLS);
2020
2021         /* Word 6 */
2022         bf_set(wqe_ctxt_tag, &wqe_abts->xmit_sequence.wqe_com,
2023                phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
2024         bf_set(wqe_xri_tag, &wqe_abts->xmit_sequence.wqe_com,
2025                abts_wqeq->sli4_xritag);
2026
2027         /* Word 7 */
2028         bf_set(wqe_cmnd, &wqe_abts->xmit_sequence.wqe_com,
2029                CMD_XMIT_SEQUENCE64_WQE);
2030         bf_set(wqe_ct, &wqe_abts->xmit_sequence.wqe_com, SLI4_CT_RPI);
2031         bf_set(wqe_class, &wqe_abts->xmit_sequence.wqe_com, CLASS3);
2032         bf_set(wqe_pu, &wqe_abts->xmit_sequence.wqe_com, 0);
2033
2034         /* Word 8 */
2035         wqe_abts->xmit_sequence.wqe_com.abort_tag = abts_wqeq->iotag;
2036
2037         /* Word 9 */
2038         bf_set(wqe_reqtag, &wqe_abts->xmit_sequence.wqe_com, abts_wqeq->iotag);
2039         /* Needs to be set by caller */
2040         bf_set(wqe_rcvoxid, &wqe_abts->xmit_sequence.wqe_com, xri);
2041
2042         /* Word 10 */
2043         bf_set(wqe_dbde, &wqe_abts->xmit_sequence.wqe_com, 1);
2044         bf_set(wqe_iod, &wqe_abts->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE);
2045         bf_set(wqe_lenloc, &wqe_abts->xmit_sequence.wqe_com,
2046                LPFC_WQE_LENLOC_WORD12);
2047         bf_set(wqe_ebde_cnt, &wqe_abts->xmit_sequence.wqe_com, 0);
2048         bf_set(wqe_qosd, &wqe_abts->xmit_sequence.wqe_com, 0);
2049
2050         /* Word 11 */
2051         bf_set(wqe_cqid, &wqe_abts->xmit_sequence.wqe_com,
2052                LPFC_WQE_CQ_ID_DEFAULT);
2053         bf_set(wqe_cmd_type, &wqe_abts->xmit_sequence.wqe_com,
2054                OTHER_COMMAND);
2055
2056         abts_wqeq->vport = phba->pport;
2057         abts_wqeq->context1 = ndlp;
2058         abts_wqeq->context2 = ctxp;
2059         abts_wqeq->context3 = NULL;
2060         abts_wqeq->rsvd2 = 0;
2061         /* hba_wqidx should already be setup from command we are aborting */
2062         abts_wqeq->iocb.ulpCommand = CMD_XMIT_SEQUENCE64_CR;
2063         abts_wqeq->iocb.ulpLe = 1;
2064
2065         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2066                         "6069 Issue ABTS to xri x%x reqtag x%x\n",
2067                         xri, abts_wqeq->iotag);
2068         return 1;
2069 }
2070
2071 static int
2072 lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba,
2073                                struct lpfc_nvmet_rcv_ctx *ctxp,
2074                                uint32_t sid, uint16_t xri)
2075 {
2076         struct lpfc_nvmet_tgtport *tgtp;
2077         struct lpfc_iocbq *abts_wqeq;
2078         union lpfc_wqe *abts_wqe;
2079         struct lpfc_nodelist *ndlp;
2080         unsigned long flags;
2081         int rc;
2082
2083         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2084         if (!ctxp->wqeq) {
2085                 ctxp->wqeq = ctxp->rqb_buffer->iocbq;
2086                 ctxp->wqeq->hba_wqidx = 0;
2087         }
2088
2089         ndlp = lpfc_findnode_did(phba->pport, sid);
2090         if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
2091             ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
2092             (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
2093                 atomic_inc(&tgtp->xmt_abort_rsp_error);
2094                 lpfc_printf_log(phba, KERN_WARNING, LOG_NVME_ABTS,
2095                                 "6160 Drop ABORT - wrong NDLP state x%x.\n",
2096                                 (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE);
2097
2098                 /* No failure to an ABTS request. */
2099                 ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2100                 return 0;
2101         }
2102
2103         /* Issue ABTS for this WQE based on iotag */
2104         ctxp->abort_wqeq = lpfc_sli_get_iocbq(phba);
2105         if (!ctxp->abort_wqeq) {
2106                 lpfc_printf_log(phba, KERN_WARNING, LOG_NVME_ABTS,
2107                                 "6161 ABORT failed: No wqeqs: "
2108                                 "xri: x%x\n", ctxp->oxid);
2109                 /* No failure to an ABTS request. */
2110                 ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2111                 return 0;
2112         }
2113         abts_wqeq = ctxp->abort_wqeq;
2114         abts_wqe = &abts_wqeq->wqe;
2115         ctxp->state = LPFC_NVMET_STE_ABORT;
2116
2117         /* Announce entry to new IO submit field. */
2118         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2119                         "6162 ABORT Request to rport DID x%06x "
2120                         "for xri x%x x%x\n",
2121                         ctxp->sid, ctxp->oxid, ctxp->wqeq->sli4_xritag);
2122
2123         /* If the hba is getting reset, this flag is set.  It is
2124          * cleared when the reset is complete and rings reestablished.
2125          */
2126         spin_lock_irqsave(&phba->hbalock, flags);
2127         /* driver queued commands are in process of being flushed */
2128         if (phba->hba_flag & HBA_NVME_IOQ_FLUSH) {
2129                 spin_unlock_irqrestore(&phba->hbalock, flags);
2130                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
2131                                 "6163 Driver in reset cleanup - flushing "
2132                                 "NVME Req now. hba_flag x%x oxid x%x\n",
2133                                 phba->hba_flag, ctxp->oxid);
2134                 lpfc_sli_release_iocbq(phba, abts_wqeq);
2135                 ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2136                 return 0;
2137         }
2138
2139         /* Outstanding abort is in progress */
2140         if (abts_wqeq->iocb_flag & LPFC_DRIVER_ABORTED) {
2141                 spin_unlock_irqrestore(&phba->hbalock, flags);
2142                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
2143                                 "6164 Outstanding NVME I/O Abort Request "
2144                                 "still pending on oxid x%x\n",
2145                                 ctxp->oxid);
2146                 lpfc_sli_release_iocbq(phba, abts_wqeq);
2147                 ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2148                 return 0;
2149         }
2150
2151         /* Ready - mark outstanding as aborted by driver. */
2152         abts_wqeq->iocb_flag |= LPFC_DRIVER_ABORTED;
2153
2154         /* WQEs are reused.  Clear stale data and set key fields to
2155          * zero like ia, iaab, iaar, xri_tag, and ctxt_tag.
2156          */
2157         memset(abts_wqe, 0, sizeof(union lpfc_wqe));
2158
2159         /* word 3 */
2160         bf_set(abort_cmd_criteria, &abts_wqe->abort_cmd, T_XRI_TAG);
2161
2162         /* word 7 */
2163         bf_set(wqe_ct, &abts_wqe->abort_cmd.wqe_com, 0);
2164         bf_set(wqe_cmnd, &abts_wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX);
2165
2166         /* word 8 - tell the FW to abort the IO associated with this
2167          * outstanding exchange ID.
2168          */
2169         abts_wqe->abort_cmd.wqe_com.abort_tag = ctxp->wqeq->sli4_xritag;
2170
2171         /* word 9 - this is the iotag for the abts_wqe completion. */
2172         bf_set(wqe_reqtag, &abts_wqe->abort_cmd.wqe_com,
2173                abts_wqeq->iotag);
2174
2175         /* word 10 */
2176         bf_set(wqe_qosd, &abts_wqe->abort_cmd.wqe_com, 1);
2177         bf_set(wqe_lenloc, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE);
2178
2179         /* word 11 */
2180         bf_set(wqe_cmd_type, &abts_wqe->abort_cmd.wqe_com, OTHER_COMMAND);
2181         bf_set(wqe_wqec, &abts_wqe->abort_cmd.wqe_com, 1);
2182         bf_set(wqe_cqid, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
2183
2184         /* ABTS WQE must go to the same WQ as the WQE to be aborted */
2185         abts_wqeq->hba_wqidx = ctxp->wqeq->hba_wqidx;
2186         abts_wqeq->wqe_cmpl = lpfc_nvmet_sol_fcp_abort_cmp;
2187         abts_wqeq->iocb_cmpl = 0;
2188         abts_wqeq->iocb_flag |= LPFC_IO_NVME;
2189         abts_wqeq->context2 = ctxp;
2190         rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_wqeq);
2191         spin_unlock_irqrestore(&phba->hbalock, flags);
2192         if (rc == WQE_SUCCESS)
2193                 return 0;
2194
2195         ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2196         lpfc_sli_release_iocbq(phba, abts_wqeq);
2197         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2198                         "6166 Failed ABORT issue_wqe with status x%x "
2199                         "for oxid x%x.\n",
2200                         rc, ctxp->oxid);
2201         return 1;
2202 }
2203
2204
2205 static int
2206 lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba,
2207                                  struct lpfc_nvmet_rcv_ctx *ctxp,
2208                                  uint32_t sid, uint16_t xri)
2209 {
2210         struct lpfc_nvmet_tgtport *tgtp;
2211         struct lpfc_iocbq *abts_wqeq;
2212         unsigned long flags;
2213         int rc;
2214
2215         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2216         if (!ctxp->wqeq) {
2217                 ctxp->wqeq = ctxp->rqb_buffer->iocbq;
2218                 ctxp->wqeq->hba_wqidx = 0;
2219         }
2220
2221         rc = lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri);
2222         if (rc == 0)
2223                 goto aerr;
2224
2225         spin_lock_irqsave(&phba->hbalock, flags);
2226         abts_wqeq = ctxp->wqeq;
2227         abts_wqeq->wqe_cmpl = lpfc_nvmet_unsol_fcp_abort_cmp;
2228         abts_wqeq->iocb_cmpl = NULL;
2229         abts_wqeq->iocb_flag |= LPFC_IO_NVMET;
2230         rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_wqeq);
2231         spin_unlock_irqrestore(&phba->hbalock, flags);
2232         if (rc == WQE_SUCCESS) {
2233                 atomic_inc(&tgtp->xmt_abort_rsp);
2234                 return 0;
2235         }
2236
2237 aerr:
2238         ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2239         atomic_inc(&tgtp->xmt_abort_rsp_error);
2240         lpfc_printf_log(phba, KERN_WARNING, LOG_NVME_ABTS,
2241                         "6135 Failed to Issue ABTS for oxid x%x. Status x%x\n",
2242                         ctxp->oxid, rc);
2243         return 1;
2244 }
2245
2246 static int
2247 lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *phba,
2248                                 struct lpfc_nvmet_rcv_ctx *ctxp,
2249                                 uint32_t sid, uint16_t xri)
2250 {
2251         struct lpfc_nvmet_tgtport *tgtp;
2252         struct lpfc_iocbq *abts_wqeq;
2253         union lpfc_wqe *wqe_abts;
2254         unsigned long flags;
2255         int rc;
2256
2257         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2258         if (!ctxp->wqeq) {
2259                 /* Issue ABTS for this WQE based on iotag */
2260                 ctxp->wqeq = lpfc_sli_get_iocbq(phba);
2261                 if (!ctxp->wqeq) {
2262                         lpfc_printf_log(phba, KERN_WARNING, LOG_NVME_ABTS,
2263                                         "6068 Abort failed: No wqeqs: "
2264                                         "xri: x%x\n", xri);
2265                         /* No failure to an ABTS request. */
2266                         kfree(ctxp);
2267                         return 0;
2268                 }
2269         }
2270         abts_wqeq = ctxp->wqeq;
2271         wqe_abts = &abts_wqeq->wqe;
2272         lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri);
2273
2274         spin_lock_irqsave(&phba->hbalock, flags);
2275         abts_wqeq->wqe_cmpl = lpfc_nvmet_xmt_ls_abort_cmp;
2276         abts_wqeq->iocb_cmpl = 0;
2277         abts_wqeq->iocb_flag |=  LPFC_IO_NVME_LS;
2278         rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, abts_wqeq);
2279         spin_unlock_irqrestore(&phba->hbalock, flags);
2280         if (rc == WQE_SUCCESS) {
2281                 atomic_inc(&tgtp->xmt_abort_rsp);
2282                 return 0;
2283         }
2284
2285         atomic_inc(&tgtp->xmt_abort_rsp_error);
2286         abts_wqeq->context2 = NULL;
2287         abts_wqeq->context3 = NULL;
2288         lpfc_sli_release_iocbq(phba, abts_wqeq);
2289         kfree(ctxp);
2290         lpfc_printf_log(phba, KERN_WARNING, LOG_NVME_ABTS,
2291                         "6056 Failed to Issue ABTS. Status x%x\n", rc);
2292         return 0;
2293 }