drivers/scsi/lpfc/lpfc_scsi.c

   1 /*******************************************************************
   2  * This file is part of the Emulex Linux Device Driver for         *
   3  * Fibre Channel Host Bus Adapters.                                *
   4  * Copyright (C) 2017-2024 Broadcom. All Rights Reserved. The term *
   5  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
   6  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
   7  * EMULEX and SLI are trademarks of Emulex.                        *
   8  * www.broadcom.com                                                *
   9  * Portions Copyright (C) 2004-2005 Christoph Hellwig              *
  10  *                                                                 *
  11  * This program is free software; you can redistribute it and/or   *
  12  * modify it under the terms of version 2 of the GNU General       *
  13  * Public License as published by the Free Software Foundation.    *
  14  * This program is distributed in the hope that it will be useful. *
  15  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
  16  * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
  17  * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
  18  * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
  19  * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
  20  * more details, a copy of which can be found in the file COPYING  *
  21  * included with this package.                                     *
  22  *******************************************************************/
  23 #include <linux/pci.h>
  24 #include <linux/slab.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/export.h>
  27 #include <linux/delay.h>
  28 #include <asm/unaligned.h>
  29 #include <linux/t10-pi.h>
  30 #include <linux/crc-t10dif.h>
  31 #include <linux/blk-cgroup.h>
  32 #include <net/checksum.h>
  33
  34 #include <scsi/scsi.h>
  35 #include <scsi/scsi_device.h>
  36 #include <scsi/scsi_eh.h>
  37 #include <scsi/scsi_host.h>
  38 #include <scsi/scsi_tcq.h>
  39 #include <scsi/scsi_transport_fc.h>
  40
  41 #include "lpfc_version.h"
  42 #include "lpfc_hw4.h"
  43 #include "lpfc_hw.h"
  44 #include "lpfc_sli.h"
  45 #include "lpfc_sli4.h"
  46 #include "lpfc_nl.h"
  47 #include "lpfc_disc.h"
  48 #include "lpfc.h"
  49 #include "lpfc_scsi.h"
  50 #include "lpfc_logmsg.h"
  51 #include "lpfc_crtn.h"
  52 #include "lpfc_vport.h"
  53
  54 #define LPFC_RESET_WAIT  2
  55 #define LPFC_ABORT_WAIT  2
  56
  57 static char *dif_op_str[] = {
  58         "PROT_NORMAL",
  59         "PROT_READ_INSERT",
  60         "PROT_WRITE_STRIP",
  61         "PROT_READ_STRIP",
  62         "PROT_WRITE_INSERT",
  63         "PROT_READ_PASS",
  64         "PROT_WRITE_PASS",
  65 };
  66
  67 struct scsi_dif_tuple {
  68         __be16 guard_tag;       /* Checksum */
  69         __be16 app_tag;         /* Opaque storage */
  70         __be32 ref_tag;         /* Target LBA or indirect LBA */
  71 };
  72
  73 static struct lpfc_rport_data *
  74 lpfc_rport_data_from_scsi_device(struct scsi_device *sdev)
  75 {
  76         struct lpfc_vport *vport = (struct lpfc_vport *)sdev->host->hostdata;
  77
  78         if (vport->phba->cfg_fof)
  79                 return ((struct lpfc_device_data *)sdev->hostdata)->rport_data;
  80         else
  81                 return (struct lpfc_rport_data *)sdev->hostdata;
  82 }
  83
  84 static void
  85 lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_io_buf *psb);
  86 static void
  87 lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_io_buf *psb);
  88 static int
  89 lpfc_prot_group_type(struct lpfc_hba *phba, struct scsi_cmnd *sc);
  90
  91 /**
  92  * lpfc_sli4_set_rsp_sgl_last - Set the last bit in the response sge.
  93  * @phba: Pointer to HBA object.
  94  * @lpfc_cmd: lpfc scsi command object pointer.
  95  *
  96  * This function is called from the lpfc_prep_task_mgmt_cmd function to
  97  * set the last bit in the response sge entry.
  98  **/
  99 static void
 100 lpfc_sli4_set_rsp_sgl_last(struct lpfc_hba *phba,
 101                                 struct lpfc_io_buf *lpfc_cmd)
 102 {
 103         struct sli4_sge *sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl;
 104         if (sgl) {
 105                 sgl += 1;
 106                 sgl->word2 = le32_to_cpu(sgl->word2);
 107                 bf_set(lpfc_sli4_sge_last, sgl, 1);
 108                 sgl->word2 = cpu_to_le32(sgl->word2);
 109         }
 110 }
 111
 112 /**
 113  * lpfc_rampdown_queue_depth - Post RAMP_DOWN_QUEUE event to worker thread
 114  * @phba: The Hba for which this call is being executed.
 115  *
 116  * This routine is called when there is resource error in driver or firmware.
 117  * This routine posts WORKER_RAMP_DOWN_QUEUE event for @phba. This routine
 118  * posts at most 1 event each second. This routine wakes up worker thread of
 119  * @phba to process WORKER_RAM_DOWN_EVENT event.
 120  *
 121  * This routine should be called with no lock held.
 122  **/
 123 void
 124 lpfc_rampdown_queue_depth(struct lpfc_hba *phba)
 125 {
 126         unsigned long flags;
 127         uint32_t evt_posted;
 128         unsigned long expires;
 129
 130         spin_lock_irqsave(&phba->hbalock, flags);
 131         atomic_inc(&phba->num_rsrc_err);
 132         phba->last_rsrc_error_time = jiffies;
 133
 134         expires = phba->last_ramp_down_time + QUEUE_RAMP_DOWN_INTERVAL;
 135         if (time_after(expires, jiffies)) {
 136                 spin_unlock_irqrestore(&phba->hbalock, flags);
 137                 return;
 138         }
 139
 140         phba->last_ramp_down_time = jiffies;
 141
 142         spin_unlock_irqrestore(&phba->hbalock, flags);
 143
 144         spin_lock_irqsave(&phba->pport->work_port_lock, flags);
 145         evt_posted = phba->pport->work_port_events & WORKER_RAMP_DOWN_QUEUE;
 146         if (!evt_posted)
 147                 phba->pport->work_port_events |= WORKER_RAMP_DOWN_QUEUE;
 148         spin_unlock_irqrestore(&phba->pport->work_port_lock, flags);
 149
 150         if (!evt_posted)
 151                 lpfc_worker_wake_up(phba);
 152         return;
 153 }
 154
 155 /**
 156  * lpfc_ramp_down_queue_handler - WORKER_RAMP_DOWN_QUEUE event handler
 157  * @phba: The Hba for which this call is being executed.
 158  *
 159  * This routine is called to  process WORKER_RAMP_DOWN_QUEUE event for worker
 160  * thread.This routine reduces queue depth for all scsi device on each vport
 161  * associated with @phba.
 162  **/
 163 void
 164 lpfc_ramp_down_queue_handler(struct lpfc_hba *phba)
 165 {
 166         struct lpfc_vport **vports;
 167         struct Scsi_Host  *shost;
 168         struct scsi_device *sdev;
 169         unsigned long new_queue_depth;
 170         unsigned long num_rsrc_err;
 171         int i;
 172
 173         num_rsrc_err = atomic_read(&phba->num_rsrc_err);
 174
 175         /*
 176          * The error and success command counters are global per
 177          * driver instance.  If another handler has already
 178          * operated on this error event, just exit.
 179          */
 180         if (num_rsrc_err == 0)
 181                 return;
 182
 183         vports = lpfc_create_vport_work_array(phba);
 184         if (vports != NULL)
 185                 for (i = 0; i <= phba->max_vports && vports[i] != NULL; i++) {
 186                         shost = lpfc_shost_from_vport(vports[i]);
 187                         shost_for_each_device(sdev, shost) {
 188                                 if (num_rsrc_err >= sdev->queue_depth)
 189                                         new_queue_depth = 1;
 190                                 else
 191                                         new_queue_depth = sdev->queue_depth -
 192                                                 num_rsrc_err;
 193                                 scsi_change_queue_depth(sdev, new_queue_depth);
 194                         }
 195                 }
 196         lpfc_destroy_vport_work_array(phba, vports);
 197         atomic_set(&phba->num_rsrc_err, 0);
 198 }
 199
 200 /**
 201  * lpfc_scsi_dev_block - set all scsi hosts to block state
 202  * @phba: Pointer to HBA context object.
 203  *
 204  * This function walks vport list and set each SCSI host to block state
 205  * by invoking fc_remote_port_delete() routine. This function is invoked
 206  * with EEH when device's PCI slot has been permanently disabled.
 207  **/
 208 void
 209 lpfc_scsi_dev_block(struct lpfc_hba *phba)
 210 {
 211         struct lpfc_vport **vports;
 212         struct Scsi_Host  *shost;
 213         struct scsi_device *sdev;
 214         struct fc_rport *rport;
 215         int i;
 216
 217         vports = lpfc_create_vport_work_array(phba);
 218         if (vports != NULL)
 219                 for (i = 0; i <= phba->max_vports && vports[i] != NULL; i++) {
 220                         shost = lpfc_shost_from_vport(vports[i]);
 221                         shost_for_each_device(sdev, shost) {
 222                                 rport = starget_to_rport(scsi_target(sdev));
 223                                 fc_remote_port_delete(rport);
 224                         }
 225                 }
 226         lpfc_destroy_vport_work_array(phba, vports);
 227 }
 228
 229 /**
 230  * lpfc_new_scsi_buf_s3 - Scsi buffer allocator for HBA with SLI3 IF spec
 231  * @vport: The virtual port for which this call being executed.
 232  * @num_to_alloc: The requested number of buffers to allocate.
 233  *
 234  * This routine allocates a scsi buffer for device with SLI-3 interface spec,
 235  * the scsi buffer contains all the necessary information needed to initiate
 236  * a SCSI I/O. The non-DMAable buffer region contains information to build
 237  * the IOCB. The DMAable region contains memory for the FCP CMND, FCP RSP,
 238  * and the initial BPL. In addition to allocating memory, the FCP CMND and
 239  * FCP RSP BDEs are setup in the BPL and the BPL BDE is setup in the IOCB.
 240  *
 241  * Return codes:
 242  *   int - number of scsi buffers that were allocated.
 243  *   0 = failure, less than num_to_alloc is a partial failure.
 244  **/
 245 static int
 246 lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc)
 247 {
 248         struct lpfc_hba *phba = vport->phba;
 249         struct lpfc_io_buf *psb;
 250         struct ulp_bde64 *bpl;
 251         IOCB_t *iocb;
 252         dma_addr_t pdma_phys_fcp_cmd;
 253         dma_addr_t pdma_phys_fcp_rsp;
 254         dma_addr_t pdma_phys_sgl;
 255         uint16_t iotag;
 256         int bcnt, bpl_size;
 257
 258         bpl_size = phba->cfg_sg_dma_buf_size -
 259                 (sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp));
 260
 261         lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
 262                          "9067 ALLOC %d scsi_bufs: %d (%d + %d + %d)\n",
 263                          num_to_alloc, phba->cfg_sg_dma_buf_size,
 264                          (int)sizeof(struct fcp_cmnd),
 265                          (int)sizeof(struct fcp_rsp), bpl_size);
 266
 267         for (bcnt = 0; bcnt < num_to_alloc; bcnt++) {
 268                 psb = kzalloc(sizeof(struct lpfc_io_buf), GFP_KERNEL);
 269                 if (!psb)
 270                         break;
 271
 272                 /*
 273                  * Get memory from the pci pool to map the virt space to pci
 274                  * bus space for an I/O.  The DMA buffer includes space for the
 275                  * struct fcp_cmnd, struct fcp_rsp and the number of bde's
 276                  * necessary to support the sg_tablesize.
 277                  */
 278                 psb->data = dma_pool_zalloc(phba->lpfc_sg_dma_buf_pool,
 279                                         GFP_KERNEL, &psb->dma_handle);
 280                 if (!psb->data) {
 281                         kfree(psb);
 282                         break;
 283                 }
 284
 285
 286                 /* Allocate iotag for psb->cur_iocbq. */
 287                 iotag = lpfc_sli_next_iotag(phba, &psb->cur_iocbq);
 288                 if (iotag == 0) {
 289                         dma_pool_free(phba->lpfc_sg_dma_buf_pool,
 290                                       psb->data, psb->dma_handle);
 291                         kfree(psb);
 292                         break;
 293                 }
 294                 psb->cur_iocbq.cmd_flag |= LPFC_IO_FCP;
 295
 296                 psb->fcp_cmnd = psb->data;
 297                 psb->fcp_rsp = psb->data + sizeof(struct fcp_cmnd);
 298                 psb->dma_sgl = psb->data + sizeof(struct fcp_cmnd) +
 299                         sizeof(struct fcp_rsp);
 300
 301                 /* Initialize local short-hand pointers. */
 302                 bpl = (struct ulp_bde64 *)psb->dma_sgl;
 303                 pdma_phys_fcp_cmd = psb->dma_handle;
 304                 pdma_phys_fcp_rsp = psb->dma_handle + sizeof(struct fcp_cmnd);
 305                 pdma_phys_sgl = psb->dma_handle + sizeof(struct fcp_cmnd) +
 306                         sizeof(struct fcp_rsp);
 307
 308                 /*
 309                  * The first two bdes are the FCP_CMD and FCP_RSP. The balance
 310                  * are sg list bdes.  Initialize the first two and leave the
 311                  * rest for queuecommand.
 312                  */
 313                 bpl[0].addrHigh = le32_to_cpu(putPaddrHigh(pdma_phys_fcp_cmd));
 314                 bpl[0].addrLow = le32_to_cpu(putPaddrLow(pdma_phys_fcp_cmd));
 315                 bpl[0].tus.f.bdeSize = sizeof(struct fcp_cmnd);
 316                 bpl[0].tus.f.bdeFlags = BUFF_TYPE_BDE_64;
 317                 bpl[0].tus.w = le32_to_cpu(bpl[0].tus.w);
 318
 319                 /* Setup the physical region for the FCP RSP */
 320                 bpl[1].addrHigh = le32_to_cpu(putPaddrHigh(pdma_phys_fcp_rsp));
 321                 bpl[1].addrLow = le32_to_cpu(putPaddrLow(pdma_phys_fcp_rsp));
 322                 bpl[1].tus.f.bdeSize = sizeof(struct fcp_rsp);
 323                 bpl[1].tus.f.bdeFlags = BUFF_TYPE_BDE_64;
 324                 bpl[1].tus.w = le32_to_cpu(bpl[1].tus.w);
 325
 326                 /*
 327                  * Since the IOCB for the FCP I/O is built into this
 328                  * lpfc_scsi_buf, initialize it with all known data now.
 329                  */
 330                 iocb = &psb->cur_iocbq.iocb;
 331                 iocb->un.fcpi64.bdl.ulpIoTag32 = 0;
 332                 if ((phba->sli_rev == 3) &&
 333                                 !(phba->sli3_options & LPFC_SLI3_BG_ENABLED)) {
 334                         /* fill in immediate fcp command BDE */
 335                         iocb->un.fcpi64.bdl.bdeFlags = BUFF_TYPE_BDE_IMMED;
 336                         iocb->un.fcpi64.bdl.bdeSize = sizeof(struct fcp_cmnd);
 337                         iocb->un.fcpi64.bdl.addrLow = offsetof(IOCB_t,
 338                                         unsli3.fcp_ext.icd);
 339                         iocb->un.fcpi64.bdl.addrHigh = 0;
 340                         iocb->ulpBdeCount = 0;
 341                         iocb->ulpLe = 0;
 342                         /* fill in response BDE */
 343                         iocb->unsli3.fcp_ext.rbde.tus.f.bdeFlags =
 344                                                         BUFF_TYPE_BDE_64;
 345                         iocb->unsli3.fcp_ext.rbde.tus.f.bdeSize =
 346                                 sizeof(struct fcp_rsp);
 347                         iocb->unsli3.fcp_ext.rbde.addrLow =
 348                                 putPaddrLow(pdma_phys_fcp_rsp);
 349                         iocb->unsli3.fcp_ext.rbde.addrHigh =
 350                                 putPaddrHigh(pdma_phys_fcp_rsp);
 351                 } else {
 352                         iocb->un.fcpi64.bdl.bdeFlags = BUFF_TYPE_BLP_64;
 353                         iocb->un.fcpi64.bdl.bdeSize =
 354                                         (2 * sizeof(struct ulp_bde64));
 355                         iocb->un.fcpi64.bdl.addrLow =
 356                                         putPaddrLow(pdma_phys_sgl);
 357                         iocb->un.fcpi64.bdl.addrHigh =
 358                                         putPaddrHigh(pdma_phys_sgl);
 359                         iocb->ulpBdeCount = 1;
 360                         iocb->ulpLe = 1;
 361                 }
 362                 iocb->ulpClass = CLASS3;
 363                 psb->status = IOSTAT_SUCCESS;
 364                 /* Put it back into the SCSI buffer list */
 365                 psb->cur_iocbq.io_buf = psb;
 366                 spin_lock_init(&psb->buf_lock);
 367                 lpfc_release_scsi_buf_s3(phba, psb);
 368
 369         }
 370
 371         return bcnt;
 372 }
 373
 374 /**
 375  * lpfc_sli4_vport_delete_fcp_xri_aborted -Remove all ndlp references for vport
 376  * @vport: pointer to lpfc vport data structure.
 377  *
 378  * This routine is invoked by the vport cleanup for deletions and the cleanup
 379  * for an ndlp on removal.
 380  **/
 381 void
 382 lpfc_sli4_vport_delete_fcp_xri_aborted(struct lpfc_vport *vport)
 383 {
 384         struct lpfc_hba *phba = vport->phba;
 385         struct lpfc_io_buf *psb, *next_psb;
 386         struct lpfc_sli4_hdw_queue *qp;
 387         unsigned long iflag = 0;
 388         int idx;
 389
 390         if (!(vport->cfg_enable_fc4_type & LPFC_ENABLE_FCP))
 391                 return;
 392
 393         spin_lock_irqsave(&phba->hbalock, iflag);
 394         for (idx = 0; idx < phba->cfg_hdw_queue; idx++) {
 395                 qp = &phba->sli4_hba.hdwq[idx];
 396
 397                 spin_lock(&qp->abts_io_buf_list_lock);
 398                 list_for_each_entry_safe(psb, next_psb,
 399                                          &qp->lpfc_abts_io_buf_list, list) {
 400                         if (psb->cur_iocbq.cmd_flag & LPFC_IO_NVME)
 401                                 continue;
 402
 403                         if (psb->rdata && psb->rdata->pnode &&
 404                             psb->rdata->pnode->vport == vport)
 405                                 psb->rdata = NULL;
 406                 }
 407                 spin_unlock(&qp->abts_io_buf_list_lock);
 408         }
 409         spin_unlock_irqrestore(&phba->hbalock, iflag);
 410 }
 411
 412 /**
 413  * lpfc_sli4_io_xri_aborted - Fast-path process of fcp xri abort
 414  * @phba: pointer to lpfc hba data structure.
 415  * @axri: pointer to the fcp xri abort wcqe structure.
 416  * @idx: index into hdwq
 417  *
 418  * This routine is invoked by the worker thread to process a SLI4 fast-path
 419  * FCP or NVME aborted xri.
 420  **/
 421 void
 422 lpfc_sli4_io_xri_aborted(struct lpfc_hba *phba,
 423                          struct sli4_wcqe_xri_aborted *axri, int idx)
 424 {
 425         u16 xri = 0;
 426         u16 rxid = 0;
 427         struct lpfc_io_buf *psb, *next_psb;
 428         struct lpfc_sli4_hdw_queue *qp;
 429         unsigned long iflag = 0;
 430         struct lpfc_iocbq *iocbq;
 431         int i;
 432         struct lpfc_nodelist *ndlp;
 433         int rrq_empty = 0;
 434         struct lpfc_sli_ring *pring = phba->sli4_hba.els_wq->pring;
 435         struct scsi_cmnd *cmd;
 436         int offline = 0;
 437
 438         if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP))
 439                 return;
 440         offline = pci_channel_offline(phba->pcidev);
 441         if (!offline) {
 442                 xri = bf_get(lpfc_wcqe_xa_xri, axri);
 443                 rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri);
 444         }
 445         qp = &phba->sli4_hba.hdwq[idx];
 446         spin_lock_irqsave(&phba->hbalock, iflag);
 447         spin_lock(&qp->abts_io_buf_list_lock);
 448         list_for_each_entry_safe(psb, next_psb,
 449                 &qp->lpfc_abts_io_buf_list, list) {
 450                 if (offline)
 451                         xri = psb->cur_iocbq.sli4_xritag;
 452                 if (psb->cur_iocbq.sli4_xritag == xri) {
 453                         list_del_init(&psb->list);
 454                         psb->flags &= ~LPFC_SBUF_XBUSY;
 455                         psb->status = IOSTAT_SUCCESS;
 456                         if (psb->cur_iocbq.cmd_flag & LPFC_IO_NVME) {
 457                                 qp->abts_nvme_io_bufs--;
 458                                 spin_unlock(&qp->abts_io_buf_list_lock);
 459                                 spin_unlock_irqrestore(&phba->hbalock, iflag);
 460                                 if (!offline) {
 461                                         lpfc_sli4_nvme_xri_aborted(phba, axri,
 462                                                                    psb);
 463                                         return;
 464                                 }
 465                                 lpfc_sli4_nvme_pci_offline_aborted(phba, psb);
 466                                 spin_lock_irqsave(&phba->hbalock, iflag);
 467                                 spin_lock(&qp->abts_io_buf_list_lock);
 468                                 continue;
 469                         }
 470                         qp->abts_scsi_io_bufs--;
 471                         spin_unlock(&qp->abts_io_buf_list_lock);
 472
 473                         if (psb->rdata && psb->rdata->pnode)
 474                                 ndlp = psb->rdata->pnode;
 475                         else
 476                                 ndlp = NULL;
 477
 478                         rrq_empty = list_empty(&phba->active_rrq_list);
 479                         spin_unlock_irqrestore(&phba->hbalock, iflag);
 480                         if (ndlp && !offline) {
 481                                 lpfc_set_rrq_active(phba, ndlp,
 482                                         psb->cur_iocbq.sli4_lxritag, rxid, 1);
 483                                 lpfc_sli4_abts_err_handler(phba, ndlp, axri);
 484                         }
 485
 486                         if (phba->cfg_fcp_wait_abts_rsp || offline) {
 487                                 spin_lock_irqsave(&psb->buf_lock, iflag);
 488                                 cmd = psb->pCmd;
 489                                 psb->pCmd = NULL;
 490                                 spin_unlock_irqrestore(&psb->buf_lock, iflag);
 491
 492                                 /* The sdev is not guaranteed to be valid post
 493                                  * scsi_done upcall.
 494                                  */
 495                                 if (cmd)
 496                                         scsi_done(cmd);
 497
 498                                 /*
 499                                  * We expect there is an abort thread waiting
 500                                  * for command completion wake up the thread.
 501                                  */
 502                                 spin_lock_irqsave(&psb->buf_lock, iflag);
 503                                 psb->cur_iocbq.cmd_flag &=
 504                                         ~LPFC_DRIVER_ABORTED;
 505                                 if (psb->waitq)
 506                                         wake_up(psb->waitq);
 507                                 spin_unlock_irqrestore(&psb->buf_lock, iflag);
 508                         }
 509
 510                         lpfc_release_scsi_buf_s4(phba, psb);
 511                         if (rrq_empty)
 512                                 lpfc_worker_wake_up(phba);
 513                         if (!offline)
 514                                 return;
 515                         spin_lock_irqsave(&phba->hbalock, iflag);
 516                         spin_lock(&qp->abts_io_buf_list_lock);
 517                         continue;
 518                 }
 519         }
 520         spin_unlock(&qp->abts_io_buf_list_lock);
 521         if (!offline) {
 522                 for (i = 1; i <= phba->sli.last_iotag; i++) {
 523                         iocbq = phba->sli.iocbq_lookup[i];
 524
 525                         if (!(iocbq->cmd_flag & LPFC_IO_FCP) ||
 526                             (iocbq->cmd_flag & LPFC_IO_LIBDFC))
 527                                 continue;
 528                         if (iocbq->sli4_xritag != xri)
 529                                 continue;
 530                         psb = container_of(iocbq, struct lpfc_io_buf, cur_iocbq);
 531                         psb->flags &= ~LPFC_SBUF_XBUSY;
 532                         spin_unlock_irqrestore(&phba->hbalock, iflag);
 533                         if (!list_empty(&pring->txq))
 534                                 lpfc_worker_wake_up(phba);
 535                         return;
 536                 }
 537         }
 538         spin_unlock_irqrestore(&phba->hbalock, iflag);
 539 }
 540
 541 /**
 542  * lpfc_get_scsi_buf_s3 - Get a scsi buffer from lpfc_scsi_buf_list of the HBA
 543  * @phba: The HBA for which this call is being executed.
 544  * @ndlp: pointer to a node-list data structure.
 545  * @cmnd: Pointer to scsi_cmnd data structure.
 546  *
 547  * This routine removes a scsi buffer from head of @phba lpfc_scsi_buf_list list
 548  * and returns to caller.
 549  *
 550  * Return codes:
 551  *   NULL - Error
 552  *   Pointer to lpfc_scsi_buf - Success
 553  **/
 554 static struct lpfc_io_buf *
 555 lpfc_get_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
 556                      struct scsi_cmnd *cmnd)
 557 {
 558         struct lpfc_io_buf *lpfc_cmd = NULL;
 559         struct list_head *scsi_buf_list_get = &phba->lpfc_scsi_buf_list_get;
 560         unsigned long iflag = 0;
 561
 562         spin_lock_irqsave(&phba->scsi_buf_list_get_lock, iflag);
 563         list_remove_head(scsi_buf_list_get, lpfc_cmd, struct lpfc_io_buf,
 564                          list);
 565         if (!lpfc_cmd) {
 566                 spin_lock(&phba->scsi_buf_list_put_lock);
 567                 list_splice(&phba->lpfc_scsi_buf_list_put,
 568                             &phba->lpfc_scsi_buf_list_get);
 569                 INIT_LIST_HEAD(&phba->lpfc_scsi_buf_list_put);
 570                 list_remove_head(scsi_buf_list_get, lpfc_cmd,
 571                                  struct lpfc_io_buf, list);
 572                 spin_unlock(&phba->scsi_buf_list_put_lock);
 573         }
 574         spin_unlock_irqrestore(&phba->scsi_buf_list_get_lock, iflag);
 575
 576         if (lpfc_ndlp_check_qdepth(phba, ndlp) && lpfc_cmd) {
 577                 atomic_inc(&ndlp->cmd_pending);
 578                 lpfc_cmd->flags |= LPFC_SBUF_BUMP_QDEPTH;
 579         }
 580         return  lpfc_cmd;
 581 }
 582 /**
 583  * lpfc_get_scsi_buf_s4 - Get a scsi buffer from io_buf_list of the HBA
 584  * @phba: The HBA for which this call is being executed.
 585  * @ndlp: pointer to a node-list data structure.
 586  * @cmnd: Pointer to scsi_cmnd data structure.
 587  *
 588  * This routine removes a scsi buffer from head of @hdwq io_buf_list
 589  * and returns to caller.
 590  *
 591  * Return codes:
 592  *   NULL - Error
 593  *   Pointer to lpfc_scsi_buf - Success
 594  **/
 595 static struct lpfc_io_buf *
 596 lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
 597                      struct scsi_cmnd *cmnd)
 598 {
 599         struct lpfc_io_buf *lpfc_cmd;
 600         struct lpfc_sli4_hdw_queue *qp;
 601         struct sli4_sge *sgl;
 602         dma_addr_t pdma_phys_fcp_rsp;
 603         dma_addr_t pdma_phys_fcp_cmd;
 604         uint32_t cpu, idx;
 605         int tag;
 606         struct fcp_cmd_rsp_buf *tmp = NULL;
 607
 608         cpu = raw_smp_processor_id();
 609         if (cmnd && phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) {
 610                 tag = blk_mq_unique_tag(scsi_cmd_to_rq(cmnd));
 611                 idx = blk_mq_unique_tag_to_hwq(tag);
 612         } else {
 613                 idx = phba->sli4_hba.cpu_map[cpu].hdwq;
 614         }
 615
 616         lpfc_cmd = lpfc_get_io_buf(phba, ndlp, idx,
 617                                    !phba->cfg_xri_rebalancing);
 618         if (!lpfc_cmd) {
 619                 qp = &phba->sli4_hba.hdwq[idx];
 620                 qp->empty_io_bufs++;
 621                 return NULL;
 622         }
 623
 624         /* Setup key fields in buffer that may have been changed
 625          * if other protocols used this buffer.
 626          */
 627         lpfc_cmd->cur_iocbq.cmd_flag = LPFC_IO_FCP;
 628         lpfc_cmd->prot_seg_cnt = 0;
 629         lpfc_cmd->seg_cnt = 0;
 630         lpfc_cmd->timeout = 0;
 631         lpfc_cmd->flags = 0;
 632         lpfc_cmd->start_time = jiffies;
 633         lpfc_cmd->waitq = NULL;
 634         lpfc_cmd->cpu = cpu;
 635 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 636         lpfc_cmd->prot_data_type = 0;
 637 #endif
 638         tmp = lpfc_get_cmd_rsp_buf_per_hdwq(phba, lpfc_cmd);
 639         if (!tmp) {
 640                 lpfc_release_io_buf(phba, lpfc_cmd, lpfc_cmd->hdwq);
 641                 return NULL;
 642         }
 643
 644         lpfc_cmd->fcp_cmnd = tmp->fcp_cmnd;
 645         lpfc_cmd->fcp_rsp = tmp->fcp_rsp;
 646
 647         /*
 648          * The first two SGEs are the FCP_CMD and FCP_RSP.
 649          * The balance are sg list bdes. Initialize the
 650          * first two and leave the rest for queuecommand.
 651          */
 652         sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl;
 653         pdma_phys_fcp_cmd = tmp->fcp_cmd_rsp_dma_handle;
 654         sgl->addr_hi = cpu_to_le32(putPaddrHigh(pdma_phys_fcp_cmd));
 655         sgl->addr_lo = cpu_to_le32(putPaddrLow(pdma_phys_fcp_cmd));
 656         sgl->word2 = le32_to_cpu(sgl->word2);
 657         bf_set(lpfc_sli4_sge_last, sgl, 0);
 658         sgl->word2 = cpu_to_le32(sgl->word2);
 659         sgl->sge_len = cpu_to_le32(sizeof(struct fcp_cmnd));
 660         sgl++;
 661
 662         /* Setup the physical region for the FCP RSP */
 663         pdma_phys_fcp_rsp = pdma_phys_fcp_cmd + sizeof(struct fcp_cmnd);
 664         sgl->addr_hi = cpu_to_le32(putPaddrHigh(pdma_phys_fcp_rsp));
 665         sgl->addr_lo = cpu_to_le32(putPaddrLow(pdma_phys_fcp_rsp));
 666         sgl->word2 = le32_to_cpu(sgl->word2);
 667         bf_set(lpfc_sli4_sge_last, sgl, 1);
 668         sgl->word2 = cpu_to_le32(sgl->word2);
 669         sgl->sge_len = cpu_to_le32(sizeof(struct fcp_rsp));
 670
 671         if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
 672                 atomic_inc(&ndlp->cmd_pending);
 673                 lpfc_cmd->flags |= LPFC_SBUF_BUMP_QDEPTH;
 674         }
 675         return  lpfc_cmd;
 676 }
 677 /**
 678  * lpfc_get_scsi_buf - Get a scsi buffer from lpfc_scsi_buf_list of the HBA
 679  * @phba: The HBA for which this call is being executed.
 680  * @ndlp: pointer to a node-list data structure.
 681  * @cmnd: Pointer to scsi_cmnd data structure.
 682  *
 683  * This routine removes a scsi buffer from head of @phba lpfc_scsi_buf_list list
 684  * and returns to caller.
 685  *
 686  * Return codes:
 687  *   NULL - Error
 688  *   Pointer to lpfc_scsi_buf - Success
 689  **/
 690 static struct lpfc_io_buf*
 691 lpfc_get_scsi_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
 692                   struct scsi_cmnd *cmnd)
 693 {
 694         return  phba->lpfc_get_scsi_buf(phba, ndlp, cmnd);
 695 }
 696
 697 /**
 698  * lpfc_release_scsi_buf_s3 - Return a scsi buffer back to hba scsi buf list
 699  * @phba: The Hba for which this call is being executed.
 700  * @psb: The scsi buffer which is being released.
 701  *
 702  * This routine releases @psb scsi buffer by adding it to tail of @phba
 703  * lpfc_scsi_buf_list list.
 704  **/
 705 static void
 706 lpfc_release_scsi_buf_s3(struct lpfc_hba *phba, struct lpfc_io_buf *psb)
 707 {
 708         unsigned long iflag = 0;
 709
 710         psb->seg_cnt = 0;
 711         psb->prot_seg_cnt = 0;
 712
 713         spin_lock_irqsave(&phba->scsi_buf_list_put_lock, iflag);
 714         psb->pCmd = NULL;
 715         psb->cur_iocbq.cmd_flag = LPFC_IO_FCP;
 716         list_add_tail(&psb->list, &phba->lpfc_scsi_buf_list_put);
 717         spin_unlock_irqrestore(&phba->scsi_buf_list_put_lock, iflag);
 718 }
 719
 720 /**
 721  * lpfc_release_scsi_buf_s4: Return a scsi buffer back to hba scsi buf list.
 722  * @phba: The Hba for which this call is being executed.
 723  * @psb: The scsi buffer which is being released.
 724  *
 725  * This routine releases @psb scsi buffer by adding it to tail of @hdwq
 726  * io_buf_list list. For SLI4 XRI's are tied to the scsi buffer
 727  * and cannot be reused for at least RA_TOV amount of time if it was
 728  * aborted.
 729  **/
 730 static void
 731 lpfc_release_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_io_buf *psb)
 732 {
 733         struct lpfc_sli4_hdw_queue *qp;
 734         unsigned long iflag = 0;
 735
 736         psb->seg_cnt = 0;
 737         psb->prot_seg_cnt = 0;
 738
 739         qp = psb->hdwq;
 740         if (psb->flags & LPFC_SBUF_XBUSY) {
 741                 spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag);
 742                 if (!phba->cfg_fcp_wait_abts_rsp)
 743                         psb->pCmd = NULL;
 744                 list_add_tail(&psb->list, &qp->lpfc_abts_io_buf_list);
 745                 qp->abts_scsi_io_bufs++;
 746                 spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag);
 747         } else {
 748                 lpfc_release_io_buf(phba, (struct lpfc_io_buf *)psb, qp);
 749         }
 750 }
 751
 752 /**
 753  * lpfc_release_scsi_buf: Return a scsi buffer back to hba scsi buf list.
 754  * @phba: The Hba for which this call is being executed.
 755  * @psb: The scsi buffer which is being released.
 756  *
 757  * This routine releases @psb scsi buffer by adding it to tail of @phba
 758  * lpfc_scsi_buf_list list.
 759  **/
 760 static void
 761 lpfc_release_scsi_buf(struct lpfc_hba *phba, struct lpfc_io_buf *psb)
 762 {
 763         if ((psb->flags & LPFC_SBUF_BUMP_QDEPTH) && psb->ndlp)
 764                 atomic_dec(&psb->ndlp->cmd_pending);
 765
 766         psb->flags &= ~LPFC_SBUF_BUMP_QDEPTH;
 767         phba->lpfc_release_scsi_buf(phba, psb);
 768 }
 769
 770 /**
 771  * lpfc_fcpcmd_to_iocb - copy the fcp_cmd data into the IOCB
 772  * @data: A pointer to the immediate command data portion of the IOCB.
 773  * @fcp_cmnd: The FCP Command that is provided by the SCSI layer.
 774  *
 775  * The routine copies the entire FCP command from @fcp_cmnd to @data while
 776  * byte swapping the data to big endian format for transmission on the wire.
 777  **/
 778 static void
 779 lpfc_fcpcmd_to_iocb(u8 *data, struct fcp_cmnd *fcp_cmnd)
 780 {
 781         int i, j;
 782
 783         for (i = 0, j = 0; i < sizeof(struct fcp_cmnd);
 784              i += sizeof(uint32_t), j++) {
 785                 ((uint32_t *)data)[j] = cpu_to_be32(((uint32_t *)fcp_cmnd)[j]);
 786         }
 787 }
 788
 789 /**
 790  * lpfc_scsi_prep_dma_buf_s3 - DMA mapping for scsi buffer to SLI3 IF spec
 791  * @phba: The Hba for which this call is being executed.
 792  * @lpfc_cmd: The scsi buffer which is going to be mapped.
 793  *
 794  * This routine does the pci dma mapping for scatter-gather list of scsi cmnd
 795  * field of @lpfc_cmd for device with SLI-3 interface spec. This routine scans
 796  * through sg elements and format the bde. This routine also initializes all
 797  * IOCB fields which are dependent on scsi command request buffer.
 798  *
 799  * Return codes:
 800  *   1 - Error
 801  *   0 - Success
 802  **/
 803 static int
 804 lpfc_scsi_prep_dma_buf_s3(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd)
 805 {
 806         struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd;
 807         struct scatterlist *sgel = NULL;
 808         struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd;
 809         struct ulp_bde64 *bpl = (struct ulp_bde64 *)lpfc_cmd->dma_sgl;
 810         struct lpfc_iocbq *iocbq = &lpfc_cmd->cur_iocbq;
 811         IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb;
 812         struct ulp_bde64 *data_bde = iocb_cmd->unsli3.fcp_ext.dbde;
 813         dma_addr_t physaddr;
 814         uint32_t num_bde = 0;
 815         int nseg, datadir = scsi_cmnd->sc_data_direction;
 816
 817         /*
 818          * There are three possibilities here - use scatter-gather segment, use
 819          * the single mapping, or neither.  Start the lpfc command prep by
 820          * bumping the bpl beyond the fcp_cmnd and fcp_rsp regions to the first
 821          * data bde entry.
 822          */
 823         bpl += 2;
 824         if (scsi_sg_count(scsi_cmnd)) {
 825                 /*
 826                  * The driver stores the segment count returned from dma_map_sg
 827                  * because this a count of dma-mappings used to map the use_sg
 828                  * pages.  They are not guaranteed to be the same for those
 829                  * architectures that implement an IOMMU.
 830                  */
 831
 832                 nseg = dma_map_sg(&phba->pcidev->dev, scsi_sglist(scsi_cmnd),
 833                                   scsi_sg_count(scsi_cmnd), datadir);
 834                 if (unlikely(!nseg))
 835                         return 1;
 836
 837                 lpfc_cmd->seg_cnt = nseg;
 838                 if (lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt) {
 839                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
 840                                         "9064 BLKGRD: %s: Too many sg segments"
 841                                         " from dma_map_sg.  Config %d, seg_cnt"
 842                                         " %d\n", __func__, phba->cfg_sg_seg_cnt,
 843                                         lpfc_cmd->seg_cnt);
 844                         WARN_ON_ONCE(lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt);
 845                         lpfc_cmd->seg_cnt = 0;
 846                         scsi_dma_unmap(scsi_cmnd);
 847                         return 2;
 848                 }
 849
 850                 /*
 851                  * The driver established a maximum scatter-gather segment count
 852                  * during probe that limits the number of sg elements in any
 853                  * single scsi command.  Just run through the seg_cnt and format
 854                  * the bde's.
 855                  * When using SLI-3 the driver will try to fit all the BDEs into
 856                  * the IOCB. If it can't then the BDEs get added to a BPL as it
 857                  * does for SLI-2 mode.
 858                  */
 859                 scsi_for_each_sg(scsi_cmnd, sgel, nseg, num_bde) {
 860                         physaddr = sg_dma_address(sgel);
 861                         if (phba->sli_rev == 3 &&
 862                             !(phba->sli3_options & LPFC_SLI3_BG_ENABLED) &&
 863                             !(iocbq->cmd_flag & DSS_SECURITY_OP) &&
 864                             nseg <= LPFC_EXT_DATA_BDE_COUNT) {
 865                                 data_bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
 866                                 data_bde->tus.f.bdeSize = sg_dma_len(sgel);
 867                                 data_bde->addrLow = putPaddrLow(physaddr);
 868                                 data_bde->addrHigh = putPaddrHigh(physaddr);
 869                                 data_bde++;
 870                         } else {
 871                                 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
 872                                 bpl->tus.f.bdeSize = sg_dma_len(sgel);
 873                                 bpl->tus.w = le32_to_cpu(bpl->tus.w);
 874                                 bpl->addrLow =
 875                                         le32_to_cpu(putPaddrLow(physaddr));
 876                                 bpl->addrHigh =
 877                                         le32_to_cpu(putPaddrHigh(physaddr));
 878                                 bpl++;
 879                         }
 880                 }
 881         }
 882
 883         /*
 884          * Finish initializing those IOCB fields that are dependent on the
 885          * scsi_cmnd request_buffer.  Note that for SLI-2 the bdeSize is
 886          * explicitly reinitialized and for SLI-3 the extended bde count is
 887          * explicitly reinitialized since all iocb memory resources are reused.
 888          */
 889         if (phba->sli_rev == 3 &&
 890             !(phba->sli3_options & LPFC_SLI3_BG_ENABLED) &&
 891             !(iocbq->cmd_flag & DSS_SECURITY_OP)) {
 892                 if (num_bde > LPFC_EXT_DATA_BDE_COUNT) {
 893                         /*
 894                          * The extended IOCB format can only fit 3 BDE or a BPL.
 895                          * This I/O has more than 3 BDE so the 1st data bde will
 896                          * be a BPL that is filled in here.
 897                          */
 898                         physaddr = lpfc_cmd->dma_handle;
 899                         data_bde->tus.f.bdeFlags = BUFF_TYPE_BLP_64;
 900                         data_bde->tus.f.bdeSize = (num_bde *
 901                                                    sizeof(struct ulp_bde64));
 902                         physaddr += (sizeof(struct fcp_cmnd) +
 903                                      sizeof(struct fcp_rsp) +
 904                                      (2 * sizeof(struct ulp_bde64)));
 905                         data_bde->addrHigh = putPaddrHigh(physaddr);
 906                         data_bde->addrLow = putPaddrLow(physaddr);
 907                         /* ebde count includes the response bde and data bpl */
 908                         iocb_cmd->unsli3.fcp_ext.ebde_count = 2;
 909                 } else {
 910                         /* ebde count includes the response bde and data bdes */
 911                         iocb_cmd->unsli3.fcp_ext.ebde_count = (num_bde + 1);
 912                 }
 913         } else {
 914                 iocb_cmd->un.fcpi64.bdl.bdeSize =
 915                         ((num_bde + 2) * sizeof(struct ulp_bde64));
 916                 iocb_cmd->unsli3.fcp_ext.ebde_count = (num_bde + 1);
 917         }
 918         fcp_cmnd->fcpDl = cpu_to_be32(scsi_bufflen(scsi_cmnd));
 919
 920         /*
 921          * Due to difference in data length between DIF/non-DIF paths,
 922          * we need to set word 4 of IOCB here
 923          */
 924         iocb_cmd->un.fcpi.fcpi_parm = scsi_bufflen(scsi_cmnd);
 925         lpfc_fcpcmd_to_iocb(iocb_cmd->unsli3.fcp_ext.icd, fcp_cmnd);
 926         return 0;
 927 }
 928
 929 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 930
 931 /* Return BG_ERR_INIT if error injection is detected by Initiator */
 932 #define BG_ERR_INIT     0x1
 933 /* Return BG_ERR_TGT if error injection is detected by Target */
 934 #define BG_ERR_TGT      0x2
 935 /* Return BG_ERR_SWAP if swapping CSUM<-->CRC is required for error injection */
 936 #define BG_ERR_SWAP     0x10
 937 /*
 938  * Return BG_ERR_CHECK if disabling Guard/Ref/App checking is required for
 939  * error injection
 940  */
 941 #define BG_ERR_CHECK    0x20
 942
 943 /**
 944  * lpfc_bg_err_inject - Determine if we should inject an error
 945  * @phba: The Hba for which this call is being executed.
 946  * @sc: The SCSI command to examine
 947  * @reftag: (out) BlockGuard reference tag for transmitted data
 948  * @apptag: (out) BlockGuard application tag for transmitted data
 949  * @new_guard: (in) Value to replace CRC with if needed
 950  *
 951  * Returns BG_ERR_* bit mask or 0 if request ignored
 952  **/
 953 static int
 954 lpfc_bg_err_inject(struct lpfc_hba *phba, struct scsi_cmnd *sc,
 955                 uint32_t *reftag, uint16_t *apptag, uint32_t new_guard)
 956 {
 957         struct scatterlist *sgpe; /* s/g prot entry */
 958         struct lpfc_io_buf *lpfc_cmd = NULL;
 959         struct scsi_dif_tuple *src = NULL;
 960         struct lpfc_nodelist *ndlp;
 961         struct lpfc_rport_data *rdata;
 962         uint32_t op = scsi_get_prot_op(sc);
 963         uint32_t blksize;
 964         uint32_t numblks;
 965         u32 lba;
 966         int rc = 0;
 967         int blockoff = 0;
 968
 969         if (op == SCSI_PROT_NORMAL)
 970                 return 0;
 971
 972         sgpe = scsi_prot_sglist(sc);
 973         lba = scsi_prot_ref_tag(sc);
 974
 975         /* First check if we need to match the LBA */
 976         if (phba->lpfc_injerr_lba != LPFC_INJERR_LBA_OFF) {
 977                 blksize = scsi_prot_interval(sc);
 978                 numblks = (scsi_bufflen(sc) + blksize - 1) / blksize;
 979
 980                 /* Make sure we have the right LBA if one is specified */
 981                 if (phba->lpfc_injerr_lba < (u64)lba ||
 982                     (phba->lpfc_injerr_lba >= (u64)(lba + numblks)))
 983                         return 0;
 984                 if (sgpe) {
 985                         blockoff = phba->lpfc_injerr_lba - (u64)lba;
 986                         numblks = sg_dma_len(sgpe) /
 987                                 sizeof(struct scsi_dif_tuple);
 988                         if (numblks < blockoff)
 989                                 blockoff = numblks;
 990                 }
 991         }
 992
 993         /* Next check if we need to match the remote NPortID or WWPN */
 994         rdata = lpfc_rport_data_from_scsi_device(sc->device);
 995         if (rdata && rdata->pnode) {
 996                 ndlp = rdata->pnode;
 997
 998                 /* Make sure we have the right NPortID if one is specified */
 999                 if (phba->lpfc_injerr_nportid  &&
1000                         (phba->lpfc_injerr_nportid != ndlp->nlp_DID))
1001                         return 0;
1002
1003                 /*
1004                  * Make sure we have the right WWPN if one is specified.
1005                  * wwn[0] should be a non-zero NAA in a good WWPN.
1006                  */
1007                 if (phba->lpfc_injerr_wwpn.u.wwn[0]  &&
1008                         (memcmp(&ndlp->nlp_portname, &phba->lpfc_injerr_wwpn,
1009                                 sizeof(struct lpfc_name)) != 0))
1010                         return 0;
1011         }
1012
1013         /* Setup a ptr to the protection data if the SCSI host provides it */
1014         if (sgpe) {
1015                 src = (struct scsi_dif_tuple *)sg_virt(sgpe);
1016                 src += blockoff;
1017                 lpfc_cmd = (struct lpfc_io_buf *)sc->host_scribble;
1018         }
1019
1020         /* Should we change the Reference Tag */
1021         if (reftag) {
1022                 if (phba->lpfc_injerr_wref_cnt) {
1023                         switch (op) {
1024                         case SCSI_PROT_WRITE_PASS:
1025                                 if (src) {
1026                                         /*
1027                                          * For WRITE_PASS, force the error
1028                                          * to be sent on the wire. It should
1029                                          * be detected by the Target.
1030                                          * If blockoff != 0 error will be
1031                                          * inserted in middle of the IO.
1032                                          */
1033
1034                                         lpfc_printf_log(phba, KERN_ERR,
1035                                                         LOG_TRACE_EVENT,
1036                                         "9076 BLKGRD: Injecting reftag error: "
1037                                         "write lba x%lx + x%x oldrefTag x%x\n",
1038                                         (unsigned long)lba, blockoff,
1039                                         be32_to_cpu(src->ref_tag));
1040
1041                                         /*
1042                                          * Save the old ref_tag so we can
1043                                          * restore it on completion.
1044                                          */
1045                                         if (lpfc_cmd) {
1046                                                 lpfc_cmd->prot_data_type =
1047                                                         LPFC_INJERR_REFTAG;
1048                                                 lpfc_cmd->prot_data_segment =
1049                                                         src;
1050                                                 lpfc_cmd->prot_data =
1051                                                         src->ref_tag;
1052                                         }
1053                                         src->ref_tag = cpu_to_be32(0xDEADBEEF);
1054                                         phba->lpfc_injerr_wref_cnt--;
1055                                         if (phba->lpfc_injerr_wref_cnt == 0) {
1056                                                 phba->lpfc_injerr_nportid = 0;
1057                                                 phba->lpfc_injerr_lba =
1058                                                         LPFC_INJERR_LBA_OFF;
1059                                                 memset(&phba->lpfc_injerr_wwpn,
1060                                                   0, sizeof(struct lpfc_name));
1061                                         }
1062                                         rc = BG_ERR_TGT | BG_ERR_CHECK;
1063
1064                                         break;
1065                                 }
1066                                 fallthrough;
1067                         case SCSI_PROT_WRITE_INSERT:
1068                                 /*
1069                                  * For WRITE_INSERT, force the error
1070                                  * to be sent on the wire. It should be
1071                                  * detected by the Target.
1072                                  */
1073                                 /* DEADBEEF will be the reftag on the wire */
1074                                 *reftag = 0xDEADBEEF;
1075                                 phba->lpfc_injerr_wref_cnt--;
1076                                 if (phba->lpfc_injerr_wref_cnt == 0) {
1077                                         phba->lpfc_injerr_nportid = 0;
1078                                         phba->lpfc_injerr_lba =
1079                                         LPFC_INJERR_LBA_OFF;
1080                                         memset(&phba->lpfc_injerr_wwpn,
1081                                                 0, sizeof(struct lpfc_name));
1082                                 }
1083                                 rc = BG_ERR_TGT | BG_ERR_CHECK;
1084
1085                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1086                                         "9078 BLKGRD: Injecting reftag error: "
1087                                         "write lba x%lx\n", (unsigned long)lba);
1088                                 break;
1089                         case SCSI_PROT_WRITE_STRIP:
1090                                 /*
1091                                  * For WRITE_STRIP and WRITE_PASS,
1092                                  * force the error on data
1093                                  * being copied from SLI-Host to SLI-Port.
1094                                  */
1095                                 *reftag = 0xDEADBEEF;
1096                                 phba->lpfc_injerr_wref_cnt--;
1097                                 if (phba->lpfc_injerr_wref_cnt == 0) {
1098                                         phba->lpfc_injerr_nportid = 0;
1099                                         phba->lpfc_injerr_lba =
1100                                                 LPFC_INJERR_LBA_OFF;
1101                                         memset(&phba->lpfc_injerr_wwpn,
1102                                                 0, sizeof(struct lpfc_name));
1103                                 }
1104                                 rc = BG_ERR_INIT;
1105
1106                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1107                                         "9077 BLKGRD: Injecting reftag error: "
1108                                         "write lba x%lx\n", (unsigned long)lba);
1109                                 break;
1110                         }
1111                 }
1112                 if (phba->lpfc_injerr_rref_cnt) {
1113                         switch (op) {
1114                         case SCSI_PROT_READ_INSERT:
1115                         case SCSI_PROT_READ_STRIP:
1116                         case SCSI_PROT_READ_PASS:
1117                                 /*
1118                                  * For READ_STRIP and READ_PASS, force the
1119                                  * error on data being read off the wire. It
1120                                  * should force an IO error to the driver.
1121                                  */
1122                                 *reftag = 0xDEADBEEF;
1123                                 phba->lpfc_injerr_rref_cnt--;
1124                                 if (phba->lpfc_injerr_rref_cnt == 0) {
1125                                         phba->lpfc_injerr_nportid = 0;
1126                                         phba->lpfc_injerr_lba =
1127                                                 LPFC_INJERR_LBA_OFF;
1128                                         memset(&phba->lpfc_injerr_wwpn,
1129                                                 0, sizeof(struct lpfc_name));
1130                                 }
1131                                 rc = BG_ERR_INIT;
1132
1133                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1134                                         "9079 BLKGRD: Injecting reftag error: "
1135                                         "read lba x%lx\n", (unsigned long)lba);
1136                                 break;
1137                         }
1138                 }
1139         }
1140
1141         /* Should we change the Application Tag */
1142         if (apptag) {
1143                 if (phba->lpfc_injerr_wapp_cnt) {
1144                         switch (op) {
1145                         case SCSI_PROT_WRITE_PASS:
1146                                 if (src) {
1147                                         /*
1148                                          * For WRITE_PASS, force the error
1149                                          * to be sent on the wire. It should
1150                                          * be detected by the Target.
1151                                          * If blockoff != 0 error will be
1152                                          * inserted in middle of the IO.
1153                                          */
1154
1155                                         lpfc_printf_log(phba, KERN_ERR,
1156                                                         LOG_TRACE_EVENT,
1157                                         "9080 BLKGRD: Injecting apptag error: "
1158                                         "write lba x%lx + x%x oldappTag x%x\n",
1159                                         (unsigned long)lba, blockoff,
1160                                         be16_to_cpu(src->app_tag));
1161
1162                                         /*
1163                                          * Save the old app_tag so we can
1164                                          * restore it on completion.
1165                                          */
1166                                         if (lpfc_cmd) {
1167                                                 lpfc_cmd->prot_data_type =
1168                                                         LPFC_INJERR_APPTAG;
1169                                                 lpfc_cmd->prot_data_segment =
1170                                                         src;
1171                                                 lpfc_cmd->prot_data =
1172                                                         src->app_tag;
1173                                         }
1174                                         src->app_tag = cpu_to_be16(0xDEAD);
1175                                         phba->lpfc_injerr_wapp_cnt--;
1176                                         if (phba->lpfc_injerr_wapp_cnt == 0) {
1177                                                 phba->lpfc_injerr_nportid = 0;
1178                                                 phba->lpfc_injerr_lba =
1179                                                         LPFC_INJERR_LBA_OFF;
1180                                                 memset(&phba->lpfc_injerr_wwpn,
1181                                                   0, sizeof(struct lpfc_name));
1182                                         }
1183                                         rc = BG_ERR_TGT | BG_ERR_CHECK;
1184                                         break;
1185                                 }
1186                                 fallthrough;
1187                         case SCSI_PROT_WRITE_INSERT:
1188                                 /*
1189                                  * For WRITE_INSERT, force the
1190                                  * error to be sent on the wire. It should be
1191                                  * detected by the Target.
1192                                  */
1193                                 /* DEAD will be the apptag on the wire */
1194                                 *apptag = 0xDEAD;
1195                                 phba->lpfc_injerr_wapp_cnt--;
1196                                 if (phba->lpfc_injerr_wapp_cnt == 0) {
1197                                         phba->lpfc_injerr_nportid = 0;
1198                                         phba->lpfc_injerr_lba =
1199                                                 LPFC_INJERR_LBA_OFF;
1200                                         memset(&phba->lpfc_injerr_wwpn,
1201                                                 0, sizeof(struct lpfc_name));
1202                                 }
1203                                 rc = BG_ERR_TGT | BG_ERR_CHECK;
1204
1205                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1206                                         "0813 BLKGRD: Injecting apptag error: "
1207                                         "write lba x%lx\n", (unsigned long)lba);
1208                                 break;
1209                         case SCSI_PROT_WRITE_STRIP:
1210                                 /*
1211                                  * For WRITE_STRIP and WRITE_PASS,
1212                                  * force the error on data
1213                                  * being copied from SLI-Host to SLI-Port.
1214                                  */
1215                                 *apptag = 0xDEAD;
1216                                 phba->lpfc_injerr_wapp_cnt--;
1217                                 if (phba->lpfc_injerr_wapp_cnt == 0) {
1218                                         phba->lpfc_injerr_nportid = 0;
1219                                         phba->lpfc_injerr_lba =
1220                                                 LPFC_INJERR_LBA_OFF;
1221                                         memset(&phba->lpfc_injerr_wwpn,
1222                                                 0, sizeof(struct lpfc_name));
1223                                 }
1224                                 rc = BG_ERR_INIT;
1225
1226                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1227                                         "0812 BLKGRD: Injecting apptag error: "
1228                                         "write lba x%lx\n", (unsigned long)lba);
1229                                 break;
1230                         }
1231                 }
1232                 if (phba->lpfc_injerr_rapp_cnt) {
1233                         switch (op) {
1234                         case SCSI_PROT_READ_INSERT:
1235                         case SCSI_PROT_READ_STRIP:
1236                         case SCSI_PROT_READ_PASS:
1237                                 /*
1238                                  * For READ_STRIP and READ_PASS, force the
1239                                  * error on data being read off the wire. It
1240                                  * should force an IO error to the driver.
1241                                  */
1242                                 *apptag = 0xDEAD;
1243                                 phba->lpfc_injerr_rapp_cnt--;
1244                                 if (phba->lpfc_injerr_rapp_cnt == 0) {
1245                                         phba->lpfc_injerr_nportid = 0;
1246                                         phba->lpfc_injerr_lba =
1247                                                 LPFC_INJERR_LBA_OFF;
1248                                         memset(&phba->lpfc_injerr_wwpn,
1249                                                 0, sizeof(struct lpfc_name));
1250                                 }
1251                                 rc = BG_ERR_INIT;
1252
1253                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1254                                         "0814 BLKGRD: Injecting apptag error: "
1255                                         "read lba x%lx\n", (unsigned long)lba);
1256                                 break;
1257                         }
1258                 }
1259         }
1260
1261
1262         /* Should we change the Guard Tag */
1263         if (new_guard) {
1264                 if (phba->lpfc_injerr_wgrd_cnt) {
1265                         switch (op) {
1266                         case SCSI_PROT_WRITE_PASS:
1267                                 rc = BG_ERR_CHECK;
1268                                 fallthrough;
1269
1270                         case SCSI_PROT_WRITE_INSERT:
1271                                 /*
1272                                  * For WRITE_INSERT, force the
1273                                  * error to be sent on the wire. It should be
1274                                  * detected by the Target.
1275                                  */
1276                                 phba->lpfc_injerr_wgrd_cnt--;
1277                                 if (phba->lpfc_injerr_wgrd_cnt == 0) {
1278                                         phba->lpfc_injerr_nportid = 0;
1279                                         phba->lpfc_injerr_lba =
1280                                                 LPFC_INJERR_LBA_OFF;
1281                                         memset(&phba->lpfc_injerr_wwpn,
1282                                                 0, sizeof(struct lpfc_name));
1283                                 }
1284
1285                                 rc |= BG_ERR_TGT | BG_ERR_SWAP;
1286                                 /* Signals the caller to swap CRC->CSUM */
1287
1288                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1289                                         "0817 BLKGRD: Injecting guard error: "
1290                                         "write lba x%lx\n", (unsigned long)lba);
1291                                 break;
1292                         case SCSI_PROT_WRITE_STRIP:
1293                                 /*
1294                                  * For WRITE_STRIP and WRITE_PASS,
1295                                  * force the error on data
1296                                  * being copied from SLI-Host to SLI-Port.
1297                                  */
1298                                 phba->lpfc_injerr_wgrd_cnt--;
1299                                 if (phba->lpfc_injerr_wgrd_cnt == 0) {
1300                                         phba->lpfc_injerr_nportid = 0;
1301                                         phba->lpfc_injerr_lba =
1302                                                 LPFC_INJERR_LBA_OFF;
1303                                         memset(&phba->lpfc_injerr_wwpn,
1304                                                 0, sizeof(struct lpfc_name));
1305                                 }
1306
1307                                 rc = BG_ERR_INIT | BG_ERR_SWAP;
1308                                 /* Signals the caller to swap CRC->CSUM */
1309
1310                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1311                                         "0816 BLKGRD: Injecting guard error: "
1312                                         "write lba x%lx\n", (unsigned long)lba);
1313                                 break;
1314                         }
1315                 }
1316                 if (phba->lpfc_injerr_rgrd_cnt) {
1317                         switch (op) {
1318                         case SCSI_PROT_READ_INSERT:
1319                         case SCSI_PROT_READ_STRIP:
1320                         case SCSI_PROT_READ_PASS:
1321                                 /*
1322                                  * For READ_STRIP and READ_PASS, force the
1323                                  * error on data being read off the wire. It
1324                                  * should force an IO error to the driver.
1325                                  */
1326                                 phba->lpfc_injerr_rgrd_cnt--;
1327                                 if (phba->lpfc_injerr_rgrd_cnt == 0) {
1328                                         phba->lpfc_injerr_nportid = 0;
1329                                         phba->lpfc_injerr_lba =
1330                                                 LPFC_INJERR_LBA_OFF;
1331                                         memset(&phba->lpfc_injerr_wwpn,
1332                                                 0, sizeof(struct lpfc_name));
1333                                 }
1334
1335                                 rc = BG_ERR_INIT | BG_ERR_SWAP;
1336                                 /* Signals the caller to swap CRC->CSUM */
1337
1338                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1339                                         "0818 BLKGRD: Injecting guard error: "
1340                                         "read lba x%lx\n", (unsigned long)lba);
1341                         }
1342                 }
1343         }
1344
1345         return rc;
1346 }
1347 #endif
1348
1349 /**
1350  * lpfc_sc_to_bg_opcodes - Determine the BlockGuard opcodes to be used with
1351  * the specified SCSI command.
1352  * @phba: The Hba for which this call is being executed.
1353  * @sc: The SCSI command to examine
1354  * @txop: (out) BlockGuard operation for transmitted data
1355  * @rxop: (out) BlockGuard operation for received data
1356  *
1357  * Returns: zero on success; non-zero if tx and/or rx op cannot be determined
1358  *
1359  **/
1360 static int
1361 lpfc_sc_to_bg_opcodes(struct lpfc_hba *phba, struct scsi_cmnd *sc,
1362                 uint8_t *txop, uint8_t *rxop)
1363 {
1364         uint8_t ret = 0;
1365
1366         if (sc->prot_flags & SCSI_PROT_IP_CHECKSUM) {
1367                 switch (scsi_get_prot_op(sc)) {
1368                 case SCSI_PROT_READ_INSERT:
1369                 case SCSI_PROT_WRITE_STRIP:
1370                         *rxop = BG_OP_IN_NODIF_OUT_CSUM;
1371                         *txop = BG_OP_IN_CSUM_OUT_NODIF;
1372                         break;
1373
1374                 case SCSI_PROT_READ_STRIP:
1375                 case SCSI_PROT_WRITE_INSERT:
1376                         *rxop = BG_OP_IN_CRC_OUT_NODIF;
1377                         *txop = BG_OP_IN_NODIF_OUT_CRC;
1378                         break;
1379
1380                 case SCSI_PROT_READ_PASS:
1381                 case SCSI_PROT_WRITE_PASS:
1382                         *rxop = BG_OP_IN_CRC_OUT_CSUM;
1383                         *txop = BG_OP_IN_CSUM_OUT_CRC;
1384                         break;
1385
1386                 case SCSI_PROT_NORMAL:
1387                 default:
1388                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1389                                 "9063 BLKGRD: Bad op/guard:%d/IP combination\n",
1390                                         scsi_get_prot_op(sc));
1391                         ret = 1;
1392                         break;
1393
1394                 }
1395         } else {
1396                 switch (scsi_get_prot_op(sc)) {
1397                 case SCSI_PROT_READ_STRIP:
1398                 case SCSI_PROT_WRITE_INSERT:
1399                         *rxop = BG_OP_IN_CRC_OUT_NODIF;
1400                         *txop = BG_OP_IN_NODIF_OUT_CRC;
1401                         break;
1402
1403                 case SCSI_PROT_READ_PASS:
1404                 case SCSI_PROT_WRITE_PASS:
1405                         *rxop = BG_OP_IN_CRC_OUT_CRC;
1406                         *txop = BG_OP_IN_CRC_OUT_CRC;
1407                         break;
1408
1409                 case SCSI_PROT_READ_INSERT:
1410                 case SCSI_PROT_WRITE_STRIP:
1411                         *rxop = BG_OP_IN_NODIF_OUT_CRC;
1412                         *txop = BG_OP_IN_CRC_OUT_NODIF;
1413                         break;
1414
1415                 case SCSI_PROT_NORMAL:
1416                 default:
1417                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1418                                 "9075 BLKGRD: Bad op/guard:%d/CRC combination\n",
1419                                         scsi_get_prot_op(sc));
1420                         ret = 1;
1421                         break;
1422                 }
1423         }
1424
1425         return ret;
1426 }
1427
1428 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1429 /**
1430  * lpfc_bg_err_opcodes - reDetermine the BlockGuard opcodes to be used with
1431  * the specified SCSI command in order to force a guard tag error.
1432  * @phba: The Hba for which this call is being executed.
1433  * @sc: The SCSI command to examine
1434  * @txop: (out) BlockGuard operation for transmitted data
1435  * @rxop: (out) BlockGuard operation for received data
1436  *
1437  * Returns: zero on success; non-zero if tx and/or rx op cannot be determined
1438  *
1439  **/
1440 static int
1441 lpfc_bg_err_opcodes(struct lpfc_hba *phba, struct scsi_cmnd *sc,
1442                 uint8_t *txop, uint8_t *rxop)
1443 {
1444
1445         if (sc->prot_flags & SCSI_PROT_IP_CHECKSUM) {
1446                 switch (scsi_get_prot_op(sc)) {
1447                 case SCSI_PROT_READ_INSERT:
1448                 case SCSI_PROT_WRITE_STRIP:
1449                         *rxop = BG_OP_IN_NODIF_OUT_CRC;
1450                         *txop = BG_OP_IN_CRC_OUT_NODIF;
1451                         break;
1452
1453                 case SCSI_PROT_READ_STRIP:
1454                 case SCSI_PROT_WRITE_INSERT:
1455                         *rxop = BG_OP_IN_CSUM_OUT_NODIF;
1456                         *txop = BG_OP_IN_NODIF_OUT_CSUM;
1457                         break;
1458
1459                 case SCSI_PROT_READ_PASS:
1460                 case SCSI_PROT_WRITE_PASS:
1461                         *rxop = BG_OP_IN_CSUM_OUT_CRC;
1462                         *txop = BG_OP_IN_CRC_OUT_CSUM;
1463                         break;
1464
1465                 case SCSI_PROT_NORMAL:
1466                 default:
1467                         break;
1468
1469                 }
1470         } else {
1471                 switch (scsi_get_prot_op(sc)) {
1472                 case SCSI_PROT_READ_STRIP:
1473                 case SCSI_PROT_WRITE_INSERT:
1474                         *rxop = BG_OP_IN_CSUM_OUT_NODIF;
1475                         *txop = BG_OP_IN_NODIF_OUT_CSUM;
1476                         break;
1477
1478                 case SCSI_PROT_READ_PASS:
1479                 case SCSI_PROT_WRITE_PASS:
1480                         *rxop = BG_OP_IN_CSUM_OUT_CSUM;
1481                         *txop = BG_OP_IN_CSUM_OUT_CSUM;
1482                         break;
1483
1484                 case SCSI_PROT_READ_INSERT:
1485                 case SCSI_PROT_WRITE_STRIP:
1486                         *rxop = BG_OP_IN_NODIF_OUT_CSUM;
1487                         *txop = BG_OP_IN_CSUM_OUT_NODIF;
1488                         break;
1489
1490                 case SCSI_PROT_NORMAL:
1491                 default:
1492                         break;
1493                 }
1494         }
1495
1496         return 0;
1497 }
1498 #endif
1499
1500 /**
1501  * lpfc_bg_setup_bpl - Setup BlockGuard BPL with no protection data
1502  * @phba: The Hba for which this call is being executed.
1503  * @sc: pointer to scsi command we're working on
1504  * @bpl: pointer to buffer list for protection groups
1505  * @datasegcnt: number of segments of data that have been dma mapped
1506  *
1507  * This function sets up BPL buffer list for protection groups of
1508  * type LPFC_PG_TYPE_NO_DIF
1509  *
1510  * This is usually used when the HBA is instructed to generate
1511  * DIFs and insert them into data stream (or strip DIF from
1512  * incoming data stream)
1513  *
1514  * The buffer list consists of just one protection group described
1515  * below:
1516  *                                +-------------------------+
1517  *   start of prot group  -->     |          PDE_5          |
1518  *                                +-------------------------+
1519  *                                |          PDE_6          |
1520  *                                +-------------------------+
1521  *                                |         Data BDE        |
1522  *                                +-------------------------+
1523  *                                |more Data BDE's ... (opt)|
1524  *                                +-------------------------+
1525  *
1526  *
1527  * Note: Data s/g buffers have been dma mapped
1528  *
1529  * Returns the number of BDEs added to the BPL.
1530  **/
1531 static int
1532 lpfc_bg_setup_bpl(struct lpfc_hba *phba, struct scsi_cmnd *sc,
1533                 struct ulp_bde64 *bpl, int datasegcnt)
1534 {
1535         struct scatterlist *sgde = NULL; /* s/g data entry */
1536         struct lpfc_pde5 *pde5 = NULL;
1537         struct lpfc_pde6 *pde6 = NULL;
1538         dma_addr_t physaddr;
1539         int i = 0, num_bde = 0, status;
1540         int datadir = sc->sc_data_direction;
1541 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1542         uint32_t rc;
1543 #endif
1544         uint32_t checking = 1;
1545         uint32_t reftag;
1546         uint8_t txop, rxop;
1547
1548         status  = lpfc_sc_to_bg_opcodes(phba, sc, &txop, &rxop);
1549         if (status)
1550                 goto out;
1551
1552         /* extract some info from the scsi command for pde*/
1553         reftag = scsi_prot_ref_tag(sc);
1554
1555 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1556         rc = lpfc_bg_err_inject(phba, sc, &reftag, NULL, 1);
1557         if (rc) {
1558                 if (rc & BG_ERR_SWAP)
1559                         lpfc_bg_err_opcodes(phba, sc, &txop, &rxop);
1560                 if (rc & BG_ERR_CHECK)
1561                         checking = 0;
1562         }
1563 #endif
1564
1565         /* setup PDE5 with what we have */
1566         pde5 = (struct lpfc_pde5 *) bpl;
1567         memset(pde5, 0, sizeof(struct lpfc_pde5));
1568         bf_set(pde5_type, pde5, LPFC_PDE5_DESCRIPTOR);
1569
1570         /* Endianness conversion if necessary for PDE5 */
1571         pde5->word0 = cpu_to_le32(pde5->word0);
1572         pde5->reftag = cpu_to_le32(reftag);
1573
1574         /* advance bpl and increment bde count */
1575         num_bde++;
1576         bpl++;
1577         pde6 = (struct lpfc_pde6 *) bpl;
1578
1579         /* setup PDE6 with the rest of the info */
1580         memset(pde6, 0, sizeof(struct lpfc_pde6));
1581         bf_set(pde6_type, pde6, LPFC_PDE6_DESCRIPTOR);
1582         bf_set(pde6_optx, pde6, txop);
1583         bf_set(pde6_oprx, pde6, rxop);
1584
1585         /*
1586          * We only need to check the data on READs, for WRITEs
1587          * protection data is automatically generated, not checked.
1588          */
1589         if (datadir == DMA_FROM_DEVICE) {
1590                 if (sc->prot_flags & SCSI_PROT_GUARD_CHECK)
1591                         bf_set(pde6_ce, pde6, checking);
1592                 else
1593                         bf_set(pde6_ce, pde6, 0);
1594
1595                 if (sc->prot_flags & SCSI_PROT_REF_CHECK)
1596                         bf_set(pde6_re, pde6, checking);
1597                 else
1598                         bf_set(pde6_re, pde6, 0);
1599         }
1600         bf_set(pde6_ai, pde6, 1);
1601         bf_set(pde6_ae, pde6, 0);
1602         bf_set(pde6_apptagval, pde6, 0);
1603
1604         /* Endianness conversion if necessary for PDE6 */
1605         pde6->word0 = cpu_to_le32(pde6->word0);
1606         pde6->word1 = cpu_to_le32(pde6->word1);
1607         pde6->word2 = cpu_to_le32(pde6->word2);
1608
1609         /* advance bpl and increment bde count */
1610         num_bde++;
1611         bpl++;
1612
1613         /* assumption: caller has already run dma_map_sg on command data */
1614         scsi_for_each_sg(sc, sgde, datasegcnt, i) {
1615                 physaddr = sg_dma_address(sgde);
1616                 bpl->addrLow = le32_to_cpu(putPaddrLow(physaddr));
1617                 bpl->addrHigh = le32_to_cpu(putPaddrHigh(physaddr));
1618                 bpl->tus.f.bdeSize = sg_dma_len(sgde);
1619                 if (datadir == DMA_TO_DEVICE)
1620                         bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1621                 else
1622                         bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I;
1623                 bpl->tus.w = le32_to_cpu(bpl->tus.w);
1624                 bpl++;
1625                 num_bde++;
1626         }
1627
1628 out:
1629         return num_bde;
1630 }
1631
1632 /**
1633  * lpfc_bg_setup_bpl_prot - Setup BlockGuard BPL with protection data
1634  * @phba: The Hba for which this call is being executed.
1635  * @sc: pointer to scsi command we're working on
1636  * @bpl: pointer to buffer list for protection groups
1637  * @datacnt: number of segments of data that have been dma mapped
1638  * @protcnt: number of segment of protection data that have been dma mapped
1639  *
1640  * This function sets up BPL buffer list for protection groups of
1641  * type LPFC_PG_TYPE_DIF
1642  *
1643  * This is usually used when DIFs are in their own buffers,
1644  * separate from the data. The HBA can then by instructed
1645  * to place the DIFs in the outgoing stream.  For read operations,
1646  * The HBA could extract the DIFs and place it in DIF buffers.
1647  *
1648  * The buffer list for this type consists of one or more of the
1649  * protection groups described below:
1650  *                                    +-------------------------+
1651  *   start of first prot group  -->   |          PDE_5          |
1652  *                                    +-------------------------+
1653  *                                    |          PDE_6          |
1654  *                                    +-------------------------+
1655  *                                    |      PDE_7 (Prot BDE)   |
1656  *                                    +-------------------------+
1657  *                                    |        Data BDE         |
1658  *                                    +-------------------------+
1659  *                                    |more Data BDE's ... (opt)|
1660  *                                    +-------------------------+
1661  *   start of new  prot group  -->    |          PDE_5          |
1662  *                                    +-------------------------+
1663  *                                    |          ...            |
1664  *                                    +-------------------------+
1665  *
1666  * Note: It is assumed that both data and protection s/g buffers have been
1667  *       mapped for DMA
1668  *
1669  * Returns the number of BDEs added to the BPL.
1670  **/
1671 static int
1672 lpfc_bg_setup_bpl_prot(struct lpfc_hba *phba, struct scsi_cmnd *sc,
1673                 struct ulp_bde64 *bpl, int datacnt, int protcnt)
1674 {
1675         struct scatterlist *sgde = NULL; /* s/g data entry */
1676         struct scatterlist *sgpe = NULL; /* s/g prot entry */
1677         struct lpfc_pde5 *pde5 = NULL;
1678         struct lpfc_pde6 *pde6 = NULL;
1679         struct lpfc_pde7 *pde7 = NULL;
1680         dma_addr_t dataphysaddr, protphysaddr;
1681         unsigned short curr_prot = 0;
1682         unsigned int split_offset;
1683         unsigned int protgroup_len, protgroup_offset = 0, protgroup_remainder;
1684         unsigned int protgrp_blks, protgrp_bytes;
1685         unsigned int remainder, subtotal;
1686         int status;
1687         int datadir = sc->sc_data_direction;
1688         unsigned char pgdone = 0, alldone = 0;
1689         unsigned blksize;
1690 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1691         uint32_t rc;
1692 #endif
1693         uint32_t checking = 1;
1694         uint32_t reftag;
1695         uint8_t txop, rxop;
1696         int num_bde = 0;
1697
1698         sgpe = scsi_prot_sglist(sc);
1699         sgde = scsi_sglist(sc);
1700
1701         if (!sgpe || !sgde) {
1702                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1703                                 "9020 Invalid s/g entry: data=x%px prot=x%px\n",
1704                                 sgpe, sgde);
1705                 return 0;
1706         }
1707
1708         status = lpfc_sc_to_bg_opcodes(phba, sc, &txop, &rxop);
1709         if (status)
1710                 goto out;
1711
1712         /* extract some info from the scsi command */
1713         blksize = scsi_prot_interval(sc);
1714         reftag = scsi_prot_ref_tag(sc);
1715
1716 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1717         rc = lpfc_bg_err_inject(phba, sc, &reftag, NULL, 1);
1718         if (rc) {
1719                 if (rc & BG_ERR_SWAP)
1720                         lpfc_bg_err_opcodes(phba, sc, &txop, &rxop);
1721                 if (rc & BG_ERR_CHECK)
1722                         checking = 0;
1723         }
1724 #endif
1725
1726         split_offset = 0;
1727         do {
1728                 /* Check to see if we ran out of space */
1729                 if (num_bde >= (phba->cfg_total_seg_cnt - 2))
1730                         return num_bde + 3;
1731
1732                 /* setup PDE5 with what we have */
1733                 pde5 = (struct lpfc_pde5 *) bpl;
1734                 memset(pde5, 0, sizeof(struct lpfc_pde5));
1735                 bf_set(pde5_type, pde5, LPFC_PDE5_DESCRIPTOR);
1736
1737                 /* Endianness conversion if necessary for PDE5 */
1738                 pde5->word0 = cpu_to_le32(pde5->word0);
1739                 pde5->reftag = cpu_to_le32(reftag);
1740
1741                 /* advance bpl and increment bde count */
1742                 num_bde++;
1743                 bpl++;
1744                 pde6 = (struct lpfc_pde6 *) bpl;
1745
1746                 /* setup PDE6 with the rest of the info */
1747                 memset(pde6, 0, sizeof(struct lpfc_pde6));
1748                 bf_set(pde6_type, pde6, LPFC_PDE6_DESCRIPTOR);
1749                 bf_set(pde6_optx, pde6, txop);
1750                 bf_set(pde6_oprx, pde6, rxop);
1751
1752                 if (sc->prot_flags & SCSI_PROT_GUARD_CHECK)
1753                         bf_set(pde6_ce, pde6, checking);
1754                 else
1755                         bf_set(pde6_ce, pde6, 0);
1756
1757                 if (sc->prot_flags & SCSI_PROT_REF_CHECK)
1758                         bf_set(pde6_re, pde6, checking);
1759                 else
1760                         bf_set(pde6_re, pde6, 0);
1761
1762                 bf_set(pde6_ai, pde6, 1);
1763                 bf_set(pde6_ae, pde6, 0);
1764                 bf_set(pde6_apptagval, pde6, 0);
1765
1766                 /* Endianness conversion if necessary for PDE6 */
1767                 pde6->word0 = cpu_to_le32(pde6->word0);
1768                 pde6->word1 = cpu_to_le32(pde6->word1);
1769                 pde6->word2 = cpu_to_le32(pde6->word2);
1770
1771                 /* advance bpl and increment bde count */
1772                 num_bde++;
1773                 bpl++;
1774
1775                 /* setup the first BDE that points to protection buffer */
1776                 protphysaddr = sg_dma_address(sgpe) + protgroup_offset;
1777                 protgroup_len = sg_dma_len(sgpe) - protgroup_offset;
1778
1779                 /* must be integer multiple of the DIF block length */
1780                 BUG_ON(protgroup_len % 8);
1781
1782                 pde7 = (struct lpfc_pde7 *) bpl;
1783                 memset(pde7, 0, sizeof(struct lpfc_pde7));
1784                 bf_set(pde7_type, pde7, LPFC_PDE7_DESCRIPTOR);
1785
1786                 pde7->addrHigh = le32_to_cpu(putPaddrHigh(protphysaddr));
1787                 pde7->addrLow = le32_to_cpu(putPaddrLow(protphysaddr));
1788
1789                 protgrp_blks = protgroup_len / 8;
1790                 protgrp_bytes = protgrp_blks * blksize;
1791
1792                 /* check if this pde is crossing the 4K boundary; if so split */
1793                 if ((pde7->addrLow & 0xfff) + protgroup_len > 0x1000) {
1794                         protgroup_remainder = 0x1000 - (pde7->addrLow & 0xfff);
1795                         protgroup_offset += protgroup_remainder;
1796                         protgrp_blks = protgroup_remainder / 8;
1797                         protgrp_bytes = protgrp_blks * blksize;
1798                 } else {
1799                         protgroup_offset = 0;
1800                         curr_prot++;
1801                 }
1802
1803                 num_bde++;
1804
1805                 /* setup BDE's for data blocks associated with DIF data */
1806                 pgdone = 0;
1807                 subtotal = 0; /* total bytes processed for current prot grp */
1808                 while (!pgdone) {
1809                         /* Check to see if we ran out of space */
1810                         if (num_bde >= phba->cfg_total_seg_cnt)
1811                                 return num_bde + 1;
1812
1813                         if (!sgde) {
1814                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1815                                         "9065 BLKGRD:%s Invalid data segment\n",
1816                                                 __func__);
1817                                 return 0;
1818                         }
1819                         bpl++;
1820                         dataphysaddr = sg_dma_address(sgde) + split_offset;
1821                         bpl->addrLow = le32_to_cpu(putPaddrLow(dataphysaddr));
1822                         bpl->addrHigh = le32_to_cpu(putPaddrHigh(dataphysaddr));
1823
1824                         remainder = sg_dma_len(sgde) - split_offset;
1825
1826                         if ((subtotal + remainder) <= protgrp_bytes) {
1827                                 /* we can use this whole buffer */
1828                                 bpl->tus.f.bdeSize = remainder;
1829                                 split_offset = 0;
1830
1831                                 if ((subtotal + remainder) == protgrp_bytes)
1832                                         pgdone = 1;
1833                         } else {
1834                                 /* must split this buffer with next prot grp */
1835                                 bpl->tus.f.bdeSize = protgrp_bytes - subtotal;
1836                                 split_offset += bpl->tus.f.bdeSize;
1837                         }
1838
1839                         subtotal += bpl->tus.f.bdeSize;
1840
1841                         if (datadir == DMA_TO_DEVICE)
1842                                 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1843                         else
1844                                 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I;
1845                         bpl->tus.w = le32_to_cpu(bpl->tus.w);
1846
1847                         num_bde++;
1848
1849                         if (split_offset)
1850                                 break;
1851
1852                         /* Move to the next s/g segment if possible */
1853                         sgde = sg_next(sgde);
1854
1855                 }
1856
1857                 if (protgroup_offset) {
1858                         /* update the reference tag */
1859                         reftag += protgrp_blks;
1860                         bpl++;
1861                         continue;
1862                 }
1863
1864                 /* are we done ? */
1865                 if (curr_prot == protcnt) {
1866                         alldone = 1;
1867                 } else if (curr_prot < protcnt) {
1868                         /* advance to next prot buffer */
1869                         sgpe = sg_next(sgpe);
1870                         bpl++;
1871
1872                         /* update the reference tag */
1873                         reftag += protgrp_blks;
1874                 } else {
1875                         /* if we're here, we have a bug */
1876                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1877                                         "9054 BLKGRD: bug in %s\n", __func__);
1878                 }
1879
1880         } while (!alldone);
1881 out:
1882
1883         return num_bde;
1884 }
1885
1886 /**
1887  * lpfc_bg_setup_sgl - Setup BlockGuard SGL with no protection data
1888  * @phba: The Hba for which this call is being executed.
1889  * @sc: pointer to scsi command we're working on
1890  * @sgl: pointer to buffer list for protection groups
1891  * @datasegcnt: number of segments of data that have been dma mapped
1892  * @lpfc_cmd: lpfc scsi command object pointer.
1893  *
1894  * This function sets up SGL buffer list for protection groups of
1895  * type LPFC_PG_TYPE_NO_DIF
1896  *
1897  * This is usually used when the HBA is instructed to generate
1898  * DIFs and insert them into data stream (or strip DIF from
1899  * incoming data stream)
1900  *
1901  * The buffer list consists of just one protection group described
1902  * below:
1903  *                                +-------------------------+
1904  *   start of prot group  -->     |         DI_SEED         |
1905  *                                +-------------------------+
1906  *                                |         Data SGE        |
1907  *                                +-------------------------+
1908  *                                |more Data SGE's ... (opt)|
1909  *                                +-------------------------+
1910  *
1911  *
1912  * Note: Data s/g buffers have been dma mapped
1913  *
1914  * Returns the number of SGEs added to the SGL.
1915  **/
1916 static uint32_t
1917 lpfc_bg_setup_sgl(struct lpfc_hba *phba, struct scsi_cmnd *sc,
1918                 struct sli4_sge *sgl, int datasegcnt,
1919                 struct lpfc_io_buf *lpfc_cmd)
1920 {
1921         struct scatterlist *sgde = NULL; /* s/g data entry */
1922         struct sli4_sge_diseed *diseed = NULL;
1923         dma_addr_t physaddr;
1924         int i = 0, status;
1925         uint32_t reftag, num_sge = 0;
1926         uint8_t txop, rxop;
1927 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1928         uint32_t rc;
1929 #endif
1930         uint32_t checking = 1;
1931         uint32_t dma_len;
1932         uint32_t dma_offset = 0;
1933         struct sli4_hybrid_sgl *sgl_xtra = NULL;
1934         int j;
1935         bool lsp_just_set = false;
1936
1937         status  = lpfc_sc_to_bg_opcodes(phba, sc, &txop, &rxop);
1938         if (status)
1939                 goto out;
1940
1941         /* extract some info from the scsi command for pde*/
1942         reftag = scsi_prot_ref_tag(sc);
1943
1944 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1945         rc = lpfc_bg_err_inject(phba, sc, &reftag, NULL, 1);
1946         if (rc) {
1947                 if (rc & BG_ERR_SWAP)
1948                         lpfc_bg_err_opcodes(phba, sc, &txop, &rxop);
1949                 if (rc & BG_ERR_CHECK)
1950                         checking = 0;
1951         }
1952 #endif
1953
1954         /* setup DISEED with what we have */
1955         diseed = (struct sli4_sge_diseed *) sgl;
1956         memset(diseed, 0, sizeof(struct sli4_sge_diseed));
1957         bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DISEED);
1958
1959         /* Endianness conversion if necessary */
1960         diseed->ref_tag = cpu_to_le32(reftag);
1961         diseed->ref_tag_tran = diseed->ref_tag;
1962
1963         /*
1964          * We only need to check the data on READs, for WRITEs
1965          * protection data is automatically generated, not checked.
1966          */
1967         if (sc->sc_data_direction == DMA_FROM_DEVICE) {
1968                 if (sc->prot_flags & SCSI_PROT_GUARD_CHECK)
1969                         bf_set(lpfc_sli4_sge_dif_ce, diseed, checking);
1970                 else
1971                         bf_set(lpfc_sli4_sge_dif_ce, diseed, 0);
1972
1973                 if (sc->prot_flags & SCSI_PROT_REF_CHECK)
1974                         bf_set(lpfc_sli4_sge_dif_re, diseed, checking);
1975                 else
1976                         bf_set(lpfc_sli4_sge_dif_re, diseed, 0);
1977         }
1978
1979         /* setup DISEED with the rest of the info */
1980         bf_set(lpfc_sli4_sge_dif_optx, diseed, txop);
1981         bf_set(lpfc_sli4_sge_dif_oprx, diseed, rxop);
1982
1983         bf_set(lpfc_sli4_sge_dif_ai, diseed, 1);
1984         bf_set(lpfc_sli4_sge_dif_me, diseed, 0);
1985
1986         /* Endianness conversion if necessary for DISEED */
1987         diseed->word2 = cpu_to_le32(diseed->word2);
1988         diseed->word3 = cpu_to_le32(diseed->word3);
1989
1990         /* advance bpl and increment sge count */
1991         num_sge++;
1992         sgl++;
1993
1994         /* assumption: caller has already run dma_map_sg on command data */
1995         sgde = scsi_sglist(sc);
1996         j = 3;
1997         for (i = 0; i < datasegcnt; i++) {
1998                 /* clear it */
1999                 sgl->word2 = 0;
2000
2001                 /* do we need to expand the segment */
2002                 if (!lsp_just_set && !((j + 1) % phba->border_sge_num) &&
2003                     ((datasegcnt - 1) != i)) {
2004                         /* set LSP type */
2005                         bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_LSP);
2006
2007                         sgl_xtra = lpfc_get_sgl_per_hdwq(phba, lpfc_cmd);
2008
2009                         if (unlikely(!sgl_xtra)) {
2010                                 lpfc_cmd->seg_cnt = 0;
2011                                 return 0;
2012                         }
2013                         sgl->addr_lo = cpu_to_le32(putPaddrLow(
2014                                                 sgl_xtra->dma_phys_sgl));
2015                         sgl->addr_hi = cpu_to_le32(putPaddrHigh(
2016                                                 sgl_xtra->dma_phys_sgl));
2017
2018                 } else {
2019                         bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA);
2020                 }
2021
2022                 if (!(bf_get(lpfc_sli4_sge_type, sgl) & LPFC_SGE_TYPE_LSP)) {
2023                         if ((datasegcnt - 1) == i)
2024                                 bf_set(lpfc_sli4_sge_last, sgl, 1);
2025                         physaddr = sg_dma_address(sgde);
2026                         dma_len = sg_dma_len(sgde);
2027                         sgl->addr_lo = cpu_to_le32(putPaddrLow(physaddr));
2028                         sgl->addr_hi = cpu_to_le32(putPaddrHigh(physaddr));
2029
2030                         bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
2031                         sgl->word2 = cpu_to_le32(sgl->word2);
2032                         sgl->sge_len = cpu_to_le32(dma_len);
2033
2034                         dma_offset += dma_len;
2035                         sgde = sg_next(sgde);
2036
2037                         sgl++;
2038                         num_sge++;
2039                         lsp_just_set = false;
2040
2041                 } else {
2042                         sgl->word2 = cpu_to_le32(sgl->word2);
2043                         sgl->sge_len = cpu_to_le32(phba->cfg_sg_dma_buf_size);
2044
2045                         sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
2046                         i = i - 1;
2047
2048                         lsp_just_set = true;
2049                 }
2050
2051                 j++;
2052
2053         }
2054
2055 out:
2056         return num_sge;
2057 }
2058
2059 /**
2060  * lpfc_bg_setup_sgl_prot - Setup BlockGuard SGL with protection data
2061  * @phba: The Hba for which this call is being executed.
2062  * @sc: pointer to scsi command we're working on
2063  * @sgl: pointer to buffer list for protection groups
2064  * @datacnt: number of segments of data that have been dma mapped
2065  * @protcnt: number of segment of protection data that have been dma mapped
2066  * @lpfc_cmd: lpfc scsi command object pointer.
2067  *
2068  * This function sets up SGL buffer list for protection groups of
2069  * type LPFC_PG_TYPE_DIF
2070  *
2071  * This is usually used when DIFs are in their own buffers,
2072  * separate from the data. The HBA can then by instructed
2073  * to place the DIFs in the outgoing stream.  For read operations,
2074  * The HBA could extract the DIFs and place it in DIF buffers.
2075  *
2076  * The buffer list for this type consists of one or more of the
2077  * protection groups described below:
2078  *                                    +-------------------------+
2079  *   start of first prot group  -->   |         DISEED          |
2080  *                                    +-------------------------+
2081  *                                    |      DIF (Prot SGE)     |
2082  *                                    +-------------------------+
2083  *                                    |        Data SGE         |
2084  *                                    +-------------------------+
2085  *                                    |more Data SGE's ... (opt)|
2086  *                                    +-------------------------+
2087  *   start of new  prot group  -->    |         DISEED          |
2088  *                                    +-------------------------+
2089  *                                    |          ...            |
2090  *                                    +-------------------------+
2091  *
2092  * Note: It is assumed that both data and protection s/g buffers have been
2093  *       mapped for DMA
2094  *
2095  * Returns the number of SGEs added to the SGL.
2096  **/
2097 static uint32_t
2098 lpfc_bg_setup_sgl_prot(struct lpfc_hba *phba, struct scsi_cmnd *sc,
2099                 struct sli4_sge *sgl, int datacnt, int protcnt,
2100                 struct lpfc_io_buf *lpfc_cmd)
2101 {
2102         struct scatterlist *sgde = NULL; /* s/g data entry */
2103         struct scatterlist *sgpe = NULL; /* s/g prot entry */
2104         struct sli4_sge_diseed *diseed = NULL;
2105         dma_addr_t dataphysaddr, protphysaddr;
2106         unsigned short curr_prot = 0;
2107         unsigned int split_offset;
2108         unsigned int protgroup_len, protgroup_offset = 0, protgroup_remainder;
2109         unsigned int protgrp_blks, protgrp_bytes;
2110         unsigned int remainder, subtotal;
2111         int status;
2112         unsigned char pgdone = 0, alldone = 0;
2113         unsigned blksize;
2114         uint32_t reftag;
2115         uint8_t txop, rxop;
2116         uint32_t dma_len;
2117 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
2118         uint32_t rc;
2119 #endif
2120         uint32_t checking = 1;
2121         uint32_t dma_offset = 0, num_sge = 0;
2122         int j = 2;
2123         struct sli4_hybrid_sgl *sgl_xtra = NULL;
2124
2125         sgpe = scsi_prot_sglist(sc);
2126         sgde = scsi_sglist(sc);
2127
2128         if (!sgpe || !sgde) {
2129                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2130                                 "9082 Invalid s/g entry: data=x%px prot=x%px\n",
2131                                 sgpe, sgde);
2132                 return 0;
2133         }
2134
2135         status = lpfc_sc_to_bg_opcodes(phba, sc, &txop, &rxop);
2136         if (status)
2137                 goto out;
2138
2139         /* extract some info from the scsi command */
2140         blksize = scsi_prot_interval(sc);
2141         reftag = scsi_prot_ref_tag(sc);
2142
2143 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
2144         rc = lpfc_bg_err_inject(phba, sc, &reftag, NULL, 1);
2145         if (rc) {
2146                 if (rc & BG_ERR_SWAP)
2147                         lpfc_bg_err_opcodes(phba, sc, &txop, &rxop);
2148                 if (rc & BG_ERR_CHECK)
2149                         checking = 0;
2150         }
2151 #endif
2152
2153         split_offset = 0;
2154         do {
2155                 /* Check to see if we ran out of space */
2156                 if ((num_sge >= (phba->cfg_total_seg_cnt - 2)) &&
2157                     !(phba->cfg_xpsgl))
2158                         return num_sge + 3;
2159
2160                 /* DISEED and DIF have to be together */
2161                 if (!((j + 1) % phba->border_sge_num) ||
2162                     !((j + 2) % phba->border_sge_num) ||
2163                     !((j + 3) % phba->border_sge_num)) {
2164                         sgl->word2 = 0;
2165
2166                         /* set LSP type */
2167                         bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_LSP);
2168
2169                         sgl_xtra = lpfc_get_sgl_per_hdwq(phba, lpfc_cmd);
2170
2171                         if (unlikely(!sgl_xtra)) {
2172                                 goto out;
2173                         } else {
2174                                 sgl->addr_lo = cpu_to_le32(putPaddrLow(
2175                                                 sgl_xtra->dma_phys_sgl));
2176                                 sgl->addr_hi = cpu_to_le32(putPaddrHigh(
2177                                                        sgl_xtra->dma_phys_sgl));
2178                         }
2179
2180                         sgl->word2 = cpu_to_le32(sgl->word2);
2181                         sgl->sge_len = cpu_to_le32(phba->cfg_sg_dma_buf_size);
2182
2183                         sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
2184                         j = 0;
2185                 }
2186
2187                 /* setup DISEED with what we have */
2188                 diseed = (struct sli4_sge_diseed *) sgl;
2189                 memset(diseed, 0, sizeof(struct sli4_sge_diseed));
2190                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DISEED);
2191
2192                 /* Endianness conversion if necessary */
2193                 diseed->ref_tag = cpu_to_le32(reftag);
2194                 diseed->ref_tag_tran = diseed->ref_tag;
2195
2196                 if (sc->prot_flags & SCSI_PROT_GUARD_CHECK) {
2197                         bf_set(lpfc_sli4_sge_dif_ce, diseed, checking);
2198                 } else {
2199                         bf_set(lpfc_sli4_sge_dif_ce, diseed, 0);
2200                         /*
2201                          * When in this mode, the hardware will replace
2202                          * the guard tag from the host with a
2203                          * newly generated good CRC for the wire.
2204                          * Switch to raw mode here to avoid this
2205                          * behavior. What the host sends gets put on the wire.
2206                          */
2207                         if (txop == BG_OP_IN_CRC_OUT_CRC) {
2208                                 txop = BG_OP_RAW_MODE;
2209                                 rxop = BG_OP_RAW_MODE;
2210                         }
2211                 }
2212
2213
2214                 if (sc->prot_flags & SCSI_PROT_REF_CHECK)
2215                         bf_set(lpfc_sli4_sge_dif_re, diseed, checking);
2216                 else
2217                         bf_set(lpfc_sli4_sge_dif_re, diseed, 0);
2218
2219                 /* setup DISEED with the rest of the info */
2220                 bf_set(lpfc_sli4_sge_dif_optx, diseed, txop);
2221                 bf_set(lpfc_sli4_sge_dif_oprx, diseed, rxop);
2222
2223                 bf_set(lpfc_sli4_sge_dif_ai, diseed, 1);
2224                 bf_set(lpfc_sli4_sge_dif_me, diseed, 0);
2225
2226                 /* Endianness conversion if necessary for DISEED */
2227                 diseed->word2 = cpu_to_le32(diseed->word2);
2228                 diseed->word3 = cpu_to_le32(diseed->word3);
2229
2230                 /* advance sgl and increment bde count */
2231                 num_sge++;
2232
2233                 sgl++;
2234                 j++;
2235
2236                 /* setup the first BDE that points to protection buffer */
2237                 protphysaddr = sg_dma_address(sgpe) + protgroup_offset;
2238                 protgroup_len = sg_dma_len(sgpe) - protgroup_offset;
2239
2240                 /* must be integer multiple of the DIF block length */
2241                 BUG_ON(protgroup_len % 8);
2242
2243                 /* Now setup DIF SGE */
2244                 sgl->word2 = 0;
2245                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DIF);
2246                 sgl->addr_hi = le32_to_cpu(putPaddrHigh(protphysaddr));
2247                 sgl->addr_lo = le32_to_cpu(putPaddrLow(protphysaddr));
2248                 sgl->word2 = cpu_to_le32(sgl->word2);
2249                 sgl->sge_len = 0;
2250
2251                 protgrp_blks = protgroup_len / 8;
2252                 protgrp_bytes = protgrp_blks * blksize;
2253
2254                 /* check if DIF SGE is crossing the 4K boundary; if so split */
2255                 if ((sgl->addr_lo & 0xfff) + protgroup_len > 0x1000) {
2256                         protgroup_remainder = 0x1000 - (sgl->addr_lo & 0xfff);
2257                         protgroup_offset += protgroup_remainder;
2258                         protgrp_blks = protgroup_remainder / 8;
2259                         protgrp_bytes = protgrp_blks * blksize;
2260                 } else {
2261                         protgroup_offset = 0;
2262                         curr_prot++;
2263                 }
2264
2265                 num_sge++;
2266
2267                 /* setup SGE's for data blocks associated with DIF data */
2268                 pgdone = 0;
2269                 subtotal = 0; /* total bytes processed for current prot grp */
2270
2271                 sgl++;
2272                 j++;
2273
2274                 while (!pgdone) {
2275                         /* Check to see if we ran out of space */
2276                         if ((num_sge >= phba->cfg_total_seg_cnt) &&
2277                             !phba->cfg_xpsgl)
2278                                 return num_sge + 1;
2279
2280                         if (!sgde) {
2281                                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2282                                         "9086 BLKGRD:%s Invalid data segment\n",
2283                                                 __func__);
2284                                 return 0;
2285                         }
2286
2287                         if (!((j + 1) % phba->border_sge_num)) {
2288                                 sgl->word2 = 0;
2289
2290                                 /* set LSP type */
2291                                 bf_set(lpfc_sli4_sge_type, sgl,
2292                                        LPFC_SGE_TYPE_LSP);
2293
2294                                 sgl_xtra = lpfc_get_sgl_per_hdwq(phba,
2295                                                                  lpfc_cmd);
2296
2297                                 if (unlikely(!sgl_xtra)) {
2298                                         goto out;
2299                                 } else {
2300                                         sgl->addr_lo = cpu_to_le32(
2301                                           putPaddrLow(sgl_xtra->dma_phys_sgl));
2302                                         sgl->addr_hi = cpu_to_le32(
2303                                           putPaddrHigh(sgl_xtra->dma_phys_sgl));
2304                                 }
2305
2306                                 sgl->word2 = cpu_to_le32(sgl->word2);
2307                                 sgl->sge_len = cpu_to_le32(
2308                                                      phba->cfg_sg_dma_buf_size);
2309
2310                                 sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
2311                         } else {
2312                                 dataphysaddr = sg_dma_address(sgde) +
2313                                                                    split_offset;
2314
2315                                 remainder = sg_dma_len(sgde) - split_offset;
2316
2317                                 if ((subtotal + remainder) <= protgrp_bytes) {
2318                                         /* we can use this whole buffer */
2319                                         dma_len = remainder;
2320                                         split_offset = 0;
2321
2322                                         if ((subtotal + remainder) ==
2323                                                                   protgrp_bytes)
2324                                                 pgdone = 1;
2325                                 } else {
2326                                         /* must split this buffer with next
2327                                          * prot grp
2328                                          */
2329                                         dma_len = protgrp_bytes - subtotal;
2330                                         split_offset += dma_len;
2331                                 }
2332
2333                                 subtotal += dma_len;
2334
2335                                 sgl->word2 = 0;
2336                                 sgl->addr_lo = cpu_to_le32(putPaddrLow(
2337                                                                  dataphysaddr));
2338                                 sgl->addr_hi = cpu_to_le32(putPaddrHigh(
2339                                                                  dataphysaddr));
2340                                 bf_set(lpfc_sli4_sge_last, sgl, 0);
2341                                 bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
2342                                 bf_set(lpfc_sli4_sge_type, sgl,
2343                                        LPFC_SGE_TYPE_DATA);
2344
2345                                 sgl->sge_len = cpu_to_le32(dma_len);
2346                                 dma_offset += dma_len;
2347
2348                                 num_sge++;
2349
2350                                 if (split_offset) {
2351                                         sgl++;
2352                                         j++;
2353                                         break;
2354                                 }
2355
2356                                 /* Move to the next s/g segment if possible */
2357                                 sgde = sg_next(sgde);
2358
2359                                 sgl++;
2360                         }
2361
2362                         j++;
2363                 }
2364
2365                 if (protgroup_offset) {
2366                         /* update the reference tag */
2367                         reftag += protgrp_blks;
2368                         continue;
2369                 }
2370
2371                 /* are we done ? */
2372                 if (curr_prot == protcnt) {
2373                         /* mark the last SGL */
2374                         sgl--;
2375                         bf_set(lpfc_sli4_sge_last, sgl, 1);
2376                         alldone = 1;
2377                 } else if (curr_prot < protcnt) {
2378                         /* advance to next prot buffer */
2379                         sgpe = sg_next(sgpe);
2380
2381                         /* update the reference tag */
2382                         reftag += protgrp_blks;
2383                 } else {
2384                         /* if we're here, we have a bug */
2385                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2386                                         "9085 BLKGRD: bug in %s\n", __func__);
2387                 }
2388
2389         } while (!alldone);
2390
2391 out:
2392
2393         return num_sge;
2394 }
2395
2396 /**
2397  * lpfc_prot_group_type - Get prtotection group type of SCSI command
2398  * @phba: The Hba for which this call is being executed.
2399  * @sc: pointer to scsi command we're working on
2400  *
2401  * Given a SCSI command that supports DIF, determine composition of protection
2402  * groups involved in setting up buffer lists
2403  *
2404  * Returns: Protection group type (with or without DIF)
2405  *
2406  **/
2407 static int
2408 lpfc_prot_group_type(struct lpfc_hba *phba, struct scsi_cmnd *sc)
2409 {
2410         int ret = LPFC_PG_TYPE_INVALID;
2411         unsigned char op = scsi_get_prot_op(sc);
2412
2413         switch (op) {
2414         case SCSI_PROT_READ_STRIP:
2415         case SCSI_PROT_WRITE_INSERT:
2416                 ret = LPFC_PG_TYPE_NO_DIF;
2417                 break;
2418         case SCSI_PROT_READ_INSERT:
2419         case SCSI_PROT_WRITE_STRIP:
2420         case SCSI_PROT_READ_PASS:
2421         case SCSI_PROT_WRITE_PASS:
2422                 ret = LPFC_PG_TYPE_DIF_BUF;
2423                 break;
2424         default:
2425                 if (phba)
2426                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2427                                         "9021 Unsupported protection op:%d\n",
2428                                         op);
2429                 break;
2430         }
2431         return ret;
2432 }
2433
2434 /**
2435  * lpfc_bg_scsi_adjust_dl - Adjust SCSI data length for BlockGuard
2436  * @phba: The Hba for which this call is being executed.
2437  * @lpfc_cmd: The scsi buffer which is going to be adjusted.
2438  *
2439  * Adjust the data length to account for how much data
2440  * is actually on the wire.
2441  *
2442  * returns the adjusted data length
2443  **/
2444 static int
2445 lpfc_bg_scsi_adjust_dl(struct lpfc_hba *phba,
2446                        struct lpfc_io_buf *lpfc_cmd)
2447 {
2448         struct scsi_cmnd *sc = lpfc_cmd->pCmd;
2449         int fcpdl;
2450
2451         fcpdl = scsi_bufflen(sc);
2452
2453         /* Check if there is protection data on the wire */
2454         if (sc->sc_data_direction == DMA_FROM_DEVICE) {
2455                 /* Read check for protection data */
2456                 if (scsi_get_prot_op(sc) ==  SCSI_PROT_READ_INSERT)
2457                         return fcpdl;
2458
2459         } else {
2460                 /* Write check for protection data */
2461                 if (scsi_get_prot_op(sc) ==  SCSI_PROT_WRITE_STRIP)
2462                         return fcpdl;
2463         }
2464
2465         /*
2466          * If we are in DIF Type 1 mode every data block has a 8 byte
2467          * DIF (trailer) attached to it. Must ajust FCP data length
2468          * to account for the protection data.
2469          */
2470         fcpdl += (fcpdl / scsi_prot_interval(sc)) * 8;
2471
2472         return fcpdl;
2473 }
2474
2475 /**
2476  * lpfc_bg_scsi_prep_dma_buf_s3 - DMA mapping for scsi buffer to SLI3 IF spec
2477  * @phba: The Hba for which this call is being executed.
2478  * @lpfc_cmd: The scsi buffer which is going to be prep'ed.
2479  *
2480  * This is the protection/DIF aware version of
2481  * lpfc_scsi_prep_dma_buf(). It may be a good idea to combine the
2482  * two functions eventually, but for now, it's here.
2483  * RETURNS 0 - SUCCESS,
2484  *         1 - Failed DMA map, retry.
2485  *         2 - Invalid scsi cmd or prot-type. Do not rety.
2486  **/
2487 static int
2488 lpfc_bg_scsi_prep_dma_buf_s3(struct lpfc_hba *phba,
2489                 struct lpfc_io_buf *lpfc_cmd)
2490 {
2491         struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd;
2492         struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd;
2493         struct ulp_bde64 *bpl = (struct ulp_bde64 *)lpfc_cmd->dma_sgl;
2494         IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb;
2495         uint32_t num_bde = 0;
2496         int datasegcnt, protsegcnt, datadir = scsi_cmnd->sc_data_direction;
2497         int prot_group_type = 0;
2498         int fcpdl;
2499         int ret = 1;
2500         struct lpfc_vport *vport = phba->pport;
2501
2502         /*
2503          * Start the lpfc command prep by bumping the bpl beyond fcp_cmnd
2504          *  fcp_rsp regions to the first data bde entry
2505          */
2506         bpl += 2;
2507         if (scsi_sg_count(scsi_cmnd)) {
2508                 /*
2509                  * The driver stores the segment count returned from dma_map_sg
2510                  * because this a count of dma-mappings used to map the use_sg
2511                  * pages.  They are not guaranteed to be the same for those
2512                  * architectures that implement an IOMMU.
2513                  */
2514                 datasegcnt = dma_map_sg(&phba->pcidev->dev,
2515                                         scsi_sglist(scsi_cmnd),
2516                                         scsi_sg_count(scsi_cmnd), datadir);
2517                 if (unlikely(!datasegcnt))
2518                         return 1;
2519
2520                 lpfc_cmd->seg_cnt = datasegcnt;
2521
2522                 /* First check if data segment count from SCSI Layer is good */
2523                 if (lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt) {
2524                         WARN_ON_ONCE(lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt);
2525                         ret = 2;
2526                         goto err;
2527                 }
2528
2529                 prot_group_type = lpfc_prot_group_type(phba, scsi_cmnd);
2530
2531                 switch (prot_group_type) {
2532                 case LPFC_PG_TYPE_NO_DIF:
2533
2534                         /* Here we need to add a PDE5 and PDE6 to the count */
2535                         if ((lpfc_cmd->seg_cnt + 2) > phba->cfg_total_seg_cnt) {
2536                                 ret = 2;
2537                                 goto err;
2538                         }
2539
2540                         num_bde = lpfc_bg_setup_bpl(phba, scsi_cmnd, bpl,
2541                                         datasegcnt);
2542                         /* we should have 2 or more entries in buffer list */
2543                         if (num_bde < 2) {
2544                                 ret = 2;
2545                                 goto err;
2546                         }
2547                         break;
2548
2549                 case LPFC_PG_TYPE_DIF_BUF:
2550                         /*
2551                          * This type indicates that protection buffers are
2552                          * passed to the driver, so that needs to be prepared
2553                          * for DMA
2554                          */
2555                         protsegcnt = dma_map_sg(&phba->pcidev->dev,
2556                                         scsi_prot_sglist(scsi_cmnd),
2557                                         scsi_prot_sg_count(scsi_cmnd), datadir);
2558                         if (unlikely(!protsegcnt)) {
2559                                 scsi_dma_unmap(scsi_cmnd);
2560                                 return 1;
2561                         }
2562
2563                         lpfc_cmd->prot_seg_cnt = protsegcnt;
2564
2565                         /*
2566                          * There is a minimun of 4 BPLs used for every
2567                          * protection data segment.
2568                          */
2569                         if ((lpfc_cmd->prot_seg_cnt * 4) >
2570                             (phba->cfg_total_seg_cnt - 2)) {
2571                                 ret = 2;
2572                                 goto err;
2573                         }
2574
2575                         num_bde = lpfc_bg_setup_bpl_prot(phba, scsi_cmnd, bpl,
2576                                         datasegcnt, protsegcnt);
2577                         /* we should have 3 or more entries in buffer list */
2578                         if ((num_bde < 3) ||
2579                             (num_bde > phba->cfg_total_seg_cnt)) {
2580                                 ret = 2;
2581                                 goto err;
2582                         }
2583                         break;
2584
2585                 case LPFC_PG_TYPE_INVALID:
2586                 default:
2587                         scsi_dma_unmap(scsi_cmnd);
2588                         lpfc_cmd->seg_cnt = 0;
2589
2590                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2591                                         "9022 Unexpected protection group %i\n",
2592                                         prot_group_type);
2593                         return 2;
2594                 }
2595         }
2596
2597         /*
2598          * Finish initializing those IOCB fields that are dependent on the
2599          * scsi_cmnd request_buffer.  Note that the bdeSize is explicitly
2600          * reinitialized since all iocb memory resources are used many times
2601          * for transmit, receive, and continuation bpl's.
2602          */
2603         iocb_cmd->un.fcpi64.bdl.bdeSize = (2 * sizeof(struct ulp_bde64));
2604         iocb_cmd->un.fcpi64.bdl.bdeSize += (num_bde * sizeof(struct ulp_bde64));
2605         iocb_cmd->ulpBdeCount = 1;
2606         iocb_cmd->ulpLe = 1;
2607
2608         fcpdl = lpfc_bg_scsi_adjust_dl(phba, lpfc_cmd);
2609         fcp_cmnd->fcpDl = be32_to_cpu(fcpdl);
2610
2611         /*
2612          * Due to difference in data length between DIF/non-DIF paths,
2613          * we need to set word 4 of IOCB here
2614          */
2615         iocb_cmd->un.fcpi.fcpi_parm = fcpdl;
2616
2617         /*
2618          * For First burst, we may need to adjust the initial transfer
2619          * length for DIF
2620          */
2621         if (iocb_cmd->un.fcpi.fcpi_XRdy &&
2622             (fcpdl < vport->cfg_first_burst_size))
2623                 iocb_cmd->un.fcpi.fcpi_XRdy = fcpdl;
2624
2625         return 0;
2626 err:
2627         if (lpfc_cmd->seg_cnt)
2628                 scsi_dma_unmap(scsi_cmnd);
2629         if (lpfc_cmd->prot_seg_cnt)
2630                 dma_unmap_sg(&phba->pcidev->dev, scsi_prot_sglist(scsi_cmnd),
2631                              scsi_prot_sg_count(scsi_cmnd),
2632                              scsi_cmnd->sc_data_direction);
2633
2634         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2635                         "9023 Cannot setup S/G List for HBA"
2636                         "IO segs %d/%d BPL %d SCSI %d: %d %d\n",
2637                         lpfc_cmd->seg_cnt, lpfc_cmd->prot_seg_cnt,
2638                         phba->cfg_total_seg_cnt, phba->cfg_sg_seg_cnt,
2639                         prot_group_type, num_bde);
2640
2641         lpfc_cmd->seg_cnt = 0;
2642         lpfc_cmd->prot_seg_cnt = 0;
2643         return ret;
2644 }
2645
2646 /*
2647  * This function calcuates the T10 DIF guard tag
2648  * on the specified data using a CRC algorithmn
2649  * using crc_t10dif.
2650  */
2651 static uint16_t
2652 lpfc_bg_crc(uint8_t *data, int count)
2653 {
2654         uint16_t crc = 0;
2655         uint16_t x;
2656
2657         crc = crc_t10dif(data, count);
2658         x = cpu_to_be16(crc);
2659         return x;
2660 }
2661
2662 /*
2663  * This function calcuates the T10 DIF guard tag
2664  * on the specified data using a CSUM algorithmn
2665  * using ip_compute_csum.
2666  */
2667 static uint16_t
2668 lpfc_bg_csum(uint8_t *data, int count)
2669 {
2670         uint16_t ret;
2671
2672         ret = ip_compute_csum(data, count);
2673         return ret;
2674 }
2675
2676 /*
2677  * This function examines the protection data to try to determine
2678  * what type of T10-DIF error occurred.
2679  */
2680 static void
2681 lpfc_calc_bg_err(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd)
2682 {
2683         struct scatterlist *sgpe; /* s/g prot entry */
2684         struct scatterlist *sgde; /* s/g data entry */
2685         struct scsi_cmnd *cmd = lpfc_cmd->pCmd;
2686         struct scsi_dif_tuple *src = NULL;
2687         uint8_t *data_src = NULL;
2688         uint16_t guard_tag;
2689         uint16_t start_app_tag, app_tag;
2690         uint32_t start_ref_tag, ref_tag;
2691         int prot, protsegcnt;
2692         int err_type, len, data_len;
2693         int chk_ref, chk_app, chk_guard;
2694         uint16_t sum;
2695         unsigned blksize;
2696
2697         err_type = BGS_GUARD_ERR_MASK;
2698         sum = 0;
2699         guard_tag = 0;
2700
2701         /* First check to see if there is protection data to examine */
2702         prot = scsi_get_prot_op(cmd);
2703         if ((prot == SCSI_PROT_READ_STRIP) ||
2704             (prot == SCSI_PROT_WRITE_INSERT) ||
2705             (prot == SCSI_PROT_NORMAL))
2706                 goto out;
2707
2708         /* Currently the driver just supports ref_tag and guard_tag checking */
2709         chk_ref = 1;
2710         chk_app = 0;
2711         chk_guard = 0;
2712
2713         /* Setup a ptr to the protection data provided by the SCSI host */
2714         sgpe = scsi_prot_sglist(cmd);
2715         protsegcnt = lpfc_cmd->prot_seg_cnt;
2716
2717         if (sgpe && protsegcnt) {
2718
2719                 /*
2720                  * We will only try to verify guard tag if the segment
2721                  * data length is a multiple of the blksize.
2722                  */
2723                 sgde = scsi_sglist(cmd);
2724                 blksize = scsi_prot_interval(cmd);
2725                 data_src = (uint8_t *)sg_virt(sgde);
2726                 data_len = sg_dma_len(sgde);
2727                 if ((data_len & (blksize - 1)) == 0)
2728                         chk_guard = 1;
2729
2730                 src = (struct scsi_dif_tuple *)sg_virt(sgpe);
2731                 start_ref_tag = scsi_prot_ref_tag(cmd);
2732                 start_app_tag = src->app_tag;
2733                 len = sg_dma_len(sgpe);
2734                 while (src && protsegcnt) {
2735                         while (len) {
2736
2737                                 /*
2738                                  * First check to see if a protection data
2739                                  * check is valid
2740                                  */
2741                                 if ((src->ref_tag == T10_PI_REF_ESCAPE) ||
2742                                     (src->app_tag == T10_PI_APP_ESCAPE)) {
2743                                         start_ref_tag++;
2744                                         goto skipit;
2745                                 }
2746
2747                                 /* First Guard Tag checking */
2748                                 if (chk_guard) {
2749                                         guard_tag = src->guard_tag;
2750                                         if (cmd->prot_flags
2751                                             & SCSI_PROT_IP_CHECKSUM)
2752                                                 sum = lpfc_bg_csum(data_src,
2753                                                                    blksize);
2754                                         else
2755                                                 sum = lpfc_bg_crc(data_src,
2756                                                                   blksize);
2757                                         if ((guard_tag != sum)) {
2758                                                 err_type = BGS_GUARD_ERR_MASK;
2759                                                 goto out;
2760                                         }
2761                                 }
2762
2763                                 /* Reference Tag checking */
2764                                 ref_tag = be32_to_cpu(src->ref_tag);
2765                                 if (chk_ref && (ref_tag != start_ref_tag)) {
2766                                         err_type = BGS_REFTAG_ERR_MASK;
2767                                         goto out;
2768                                 }
2769                                 start_ref_tag++;
2770
2771                                 /* App Tag checking */
2772                                 app_tag = src->app_tag;
2773                                 if (chk_app && (app_tag != start_app_tag)) {
2774                                         err_type = BGS_APPTAG_ERR_MASK;
2775                                         goto out;
2776                                 }
2777 skipit:
2778                                 len -= sizeof(struct scsi_dif_tuple);
2779                                 if (len < 0)
2780                                         len = 0;
2781                                 src++;
2782
2783                                 data_src += blksize;
2784                                 data_len -= blksize;
2785
2786                                 /*
2787                                  * Are we at the end of the Data segment?
2788                                  * The data segment is only used for Guard
2789                                  * tag checking.
2790                                  */
2791                                 if (chk_guard && (data_len == 0)) {
2792                                         chk_guard = 0;
2793                                         sgde = sg_next(sgde);
2794                                         if (!sgde)
2795                                                 goto out;
2796
2797                                         data_src = (uint8_t *)sg_virt(sgde);
2798                                         data_len = sg_dma_len(sgde);
2799                                         if ((data_len & (blksize - 1)) == 0)
2800                                                 chk_guard = 1;
2801                                 }
2802                         }
2803
2804                         /* Goto the next Protection data segment */
2805                         sgpe = sg_next(sgpe);
2806                         if (sgpe) {
2807                                 src = (struct scsi_dif_tuple *)sg_virt(sgpe);
2808                                 len = sg_dma_len(sgpe);
2809                         } else {
2810                                 src = NULL;
2811                         }
2812                         protsegcnt--;
2813                 }
2814         }
2815 out:
2816         if (err_type == BGS_GUARD_ERR_MASK) {
2817                 scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x1);
2818                 set_host_byte(cmd, DID_ABORT);
2819                 phba->bg_guard_err_cnt++;
2820                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
2821                                 "9069 BLKGRD: reftag %x grd_tag err %x != %x\n",
2822                                 scsi_prot_ref_tag(cmd),
2823                                 sum, guard_tag);
2824
2825         } else if (err_type == BGS_REFTAG_ERR_MASK) {
2826                 scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x3);
2827                 set_host_byte(cmd, DID_ABORT);
2828
2829                 phba->bg_reftag_err_cnt++;
2830                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
2831                                 "9066 BLKGRD: reftag %x ref_tag err %x != %x\n",
2832                                 scsi_prot_ref_tag(cmd),
2833                                 ref_tag, start_ref_tag);
2834
2835         } else if (err_type == BGS_APPTAG_ERR_MASK) {
2836                 scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x2);
2837                 set_host_byte(cmd, DID_ABORT);
2838
2839                 phba->bg_apptag_err_cnt++;
2840                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
2841                                 "9041 BLKGRD: reftag %x app_tag err %x != %x\n",
2842                                 scsi_prot_ref_tag(cmd),
2843                                 app_tag, start_app_tag);
2844         }
2845 }
2846
2847 /*
2848  * This function checks for BlockGuard errors detected by
2849  * the HBA.  In case of errors, the ASC/ASCQ fields in the
2850  * sense buffer will be set accordingly, paired with
2851  * ILLEGAL_REQUEST to signal to the kernel that the HBA
2852  * detected corruption.
2853  *
2854  * Returns:
2855  *  0 - No error found
2856  *  1 - BlockGuard error found
2857  * -1 - Internal error (bad profile, ...etc)
2858  */
2859 static int
2860 lpfc_parse_bg_err(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd,
2861                   struct lpfc_iocbq *pIocbOut)
2862 {
2863         struct scsi_cmnd *cmd = lpfc_cmd->pCmd;
2864         struct sli3_bg_fields *bgf;
2865         int ret = 0;
2866         struct lpfc_wcqe_complete *wcqe;
2867         u32 status;
2868         u32 bghm = 0;
2869         u32 bgstat = 0;
2870         u64 failing_sector = 0;
2871
2872         if (phba->sli_rev == LPFC_SLI_REV4) {
2873                 wcqe = &pIocbOut->wcqe_cmpl;
2874                 status = bf_get(lpfc_wcqe_c_status, wcqe);
2875
2876                 if (status == CQE_STATUS_DI_ERROR) {
2877                         /* Guard Check failed */
2878                         if (bf_get(lpfc_wcqe_c_bg_ge, wcqe))
2879                                 bgstat |= BGS_GUARD_ERR_MASK;
2880
2881                         /* AppTag Check failed */
2882                         if (bf_get(lpfc_wcqe_c_bg_ae, wcqe))
2883                                 bgstat |= BGS_APPTAG_ERR_MASK;
2884
2885                         /* RefTag Check failed */
2886                         if (bf_get(lpfc_wcqe_c_bg_re, wcqe))
2887                                 bgstat |= BGS_REFTAG_ERR_MASK;
2888
2889                         /* Check to see if there was any good data before the
2890                          * error
2891                          */
2892                         if (bf_get(lpfc_wcqe_c_bg_tdpv, wcqe)) {
2893                                 bgstat |= BGS_HI_WATER_MARK_PRESENT_MASK;
2894                                 bghm = wcqe->total_data_placed;
2895                         }
2896
2897                         /*
2898                          * Set ALL the error bits to indicate we don't know what
2899                          * type of error it is.
2900                          */
2901                         if (!bgstat)
2902                                 bgstat |= (BGS_REFTAG_ERR_MASK |
2903                                            BGS_APPTAG_ERR_MASK |
2904                                            BGS_GUARD_ERR_MASK);
2905                 }
2906
2907         } else {
2908                 bgf = &pIocbOut->iocb.unsli3.sli3_bg;
2909                 bghm = bgf->bghm;
2910                 bgstat = bgf->bgstat;
2911         }
2912
2913         if (lpfc_bgs_get_invalid_prof(bgstat)) {
2914                 cmd->result = DID_ERROR << 16;
2915                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
2916                                 "9072 BLKGRD: Invalid BG Profile in cmd "
2917                                 "0x%x reftag 0x%x blk cnt 0x%x "
2918                                 "bgstat=x%x bghm=x%x\n", cmd->cmnd[0],
2919                                 scsi_prot_ref_tag(cmd),
2920                                 scsi_logical_block_count(cmd), bgstat, bghm);
2921                 ret = (-1);
2922                 goto out;
2923         }
2924
2925         if (lpfc_bgs_get_uninit_dif_block(bgstat)) {
2926                 cmd->result = DID_ERROR << 16;
2927                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
2928                                 "9073 BLKGRD: Invalid BG PDIF Block in cmd "
2929                                 "0x%x reftag 0x%x blk cnt 0x%x "
2930                                 "bgstat=x%x bghm=x%x\n", cmd->cmnd[0],
2931                                 scsi_prot_ref_tag(cmd),
2932                                 scsi_logical_block_count(cmd), bgstat, bghm);
2933                 ret = (-1);
2934                 goto out;
2935         }
2936
2937         if (lpfc_bgs_get_guard_err(bgstat)) {
2938                 ret = 1;
2939                 scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x1);
2940                 set_host_byte(cmd, DID_ABORT);
2941                 phba->bg_guard_err_cnt++;
2942                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
2943                                 "9055 BLKGRD: Guard Tag error in cmd "
2944                                 "0x%x reftag 0x%x blk cnt 0x%x "
2945                                 "bgstat=x%x bghm=x%x\n", cmd->cmnd[0],
2946                                 scsi_prot_ref_tag(cmd),
2947                                 scsi_logical_block_count(cmd), bgstat, bghm);
2948         }
2949
2950         if (lpfc_bgs_get_reftag_err(bgstat)) {
2951                 ret = 1;
2952                 scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x3);
2953                 set_host_byte(cmd, DID_ABORT);
2954                 phba->bg_reftag_err_cnt++;
2955                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
2956                                 "9056 BLKGRD: Ref Tag error in cmd "
2957                                 "0x%x reftag 0x%x blk cnt 0x%x "
2958                                 "bgstat=x%x bghm=x%x\n", cmd->cmnd[0],
2959                                 scsi_prot_ref_tag(cmd),
2960                                 scsi_logical_block_count(cmd), bgstat, bghm);
2961         }
2962
2963         if (lpfc_bgs_get_apptag_err(bgstat)) {
2964                 ret = 1;
2965                 scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x2);
2966                 set_host_byte(cmd, DID_ABORT);
2967                 phba->bg_apptag_err_cnt++;
2968                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
2969                                 "9061 BLKGRD: App Tag error in cmd "
2970                                 "0x%x reftag 0x%x blk cnt 0x%x "
2971                                 "bgstat=x%x bghm=x%x\n", cmd->cmnd[0],
2972                                 scsi_prot_ref_tag(cmd),
2973                                 scsi_logical_block_count(cmd), bgstat, bghm);
2974         }
2975
2976         if (lpfc_bgs_get_hi_water_mark_present(bgstat)) {
2977                 /*
2978                  * setup sense data descriptor 0 per SPC-4 as an information
2979                  * field, and put the failing LBA in it.
2980                  * This code assumes there was also a guard/app/ref tag error
2981                  * indication.
2982                  */
2983                 cmd->sense_buffer[7] = 0xc;   /* Additional sense length */
2984                 cmd->sense_buffer[8] = 0;     /* Information descriptor type */
2985                 cmd->sense_buffer[9] = 0xa;   /* Additional descriptor length */
2986                 cmd->sense_buffer[10] = 0x80; /* Validity bit */
2987
2988                 /* bghm is a "on the wire" FC frame based count */
2989                 switch (scsi_get_prot_op(cmd)) {
2990                 case SCSI_PROT_READ_INSERT:
2991                 case SCSI_PROT_WRITE_STRIP:
2992                         bghm /= cmd->device->sector_size;
2993                         break;
2994                 case SCSI_PROT_READ_STRIP:
2995                 case SCSI_PROT_WRITE_INSERT:
2996                 case SCSI_PROT_READ_PASS:
2997                 case SCSI_PROT_WRITE_PASS:
2998                         bghm /= (cmd->device->sector_size +
2999                                 sizeof(struct scsi_dif_tuple));
3000                         break;
3001                 }
3002
3003                 failing_sector = scsi_get_lba(cmd);
3004                 failing_sector += bghm;
3005
3006                 /* Descriptor Information */
3007                 put_unaligned_be64(failing_sector, &cmd->sense_buffer[12]);
3008         }
3009
3010         if (!ret) {
3011                 /* No error was reported - problem in FW? */
3012                 lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
3013                                 "9057 BLKGRD: Unknown error in cmd "
3014                                 "0x%x reftag 0x%x blk cnt 0x%x "
3015                                 "bgstat=x%x bghm=x%x\n", cmd->cmnd[0],
3016                                 scsi_prot_ref_tag(cmd),
3017                                 scsi_logical_block_count(cmd), bgstat, bghm);
3018
3019                 /* Calculate what type of error it was */
3020                 lpfc_calc_bg_err(phba, lpfc_cmd);
3021         }
3022 out:
3023         return ret;
3024 }
3025
3026 /**
3027  * lpfc_scsi_prep_dma_buf_s4 - DMA mapping for scsi buffer to SLI4 IF spec
3028  * @phba: The Hba for which this call is being executed.
3029  * @lpfc_cmd: The scsi buffer which is going to be mapped.
3030  *
3031  * This routine does the pci dma mapping for scatter-gather list of scsi cmnd
3032  * field of @lpfc_cmd for device with SLI-4 interface spec.
3033  *
3034  * Return codes:
3035  *      2 - Error - Do not retry
3036  *      1 - Error - Retry
3037  *      0 - Success
3038  **/
3039 static int
3040 lpfc_scsi_prep_dma_buf_s4(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd)
3041 {
3042         struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd;
3043         struct scatterlist *sgel = NULL;
3044         struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd;
3045         struct sli4_sge *sgl = (struct sli4_sge *)lpfc_cmd->dma_sgl;
3046         struct sli4_sge *first_data_sgl;
3047         struct lpfc_iocbq *pwqeq = &lpfc_cmd->cur_iocbq;
3048         struct lpfc_vport *vport = phba->pport;
3049         union lpfc_wqe128 *wqe = &pwqeq->wqe;
3050         dma_addr_t physaddr;
3051         uint32_t dma_len;
3052         uint32_t dma_offset = 0;
3053         int nseg, i, j;
3054         struct ulp_bde64 *bde;
3055         bool lsp_just_set = false;
3056         struct sli4_hybrid_sgl *sgl_xtra = NULL;
3057
3058         /*
3059          * There are three possibilities here - use scatter-gather segment, use
3060          * the single mapping, or neither.  Start the lpfc command prep by
3061          * bumping the bpl beyond the fcp_cmnd and fcp_rsp regions to the first
3062          * data bde entry.
3063          */
3064         if (scsi_sg_count(scsi_cmnd)) {
3065                 /*
3066                  * The driver stores the segment count returned from dma_map_sg
3067                  * because this a count of dma-mappings used to map the use_sg
3068                  * pages.  They are not guaranteed to be the same for those
3069                  * architectures that implement an IOMMU.
3070                  */
3071
3072                 nseg = scsi_dma_map(scsi_cmnd);
3073                 if (unlikely(nseg <= 0))
3074                         return 1;
3075                 sgl += 1;
3076                 /* clear the last flag in the fcp_rsp map entry */
3077                 sgl->word2 = le32_to_cpu(sgl->word2);
3078                 bf_set(lpfc_sli4_sge_last, sgl, 0);
3079                 sgl->word2 = cpu_to_le32(sgl->word2);
3080                 sgl += 1;
3081                 first_data_sgl = sgl;
3082                 lpfc_cmd->seg_cnt = nseg;
3083                 if (!phba->cfg_xpsgl &&
3084                     lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt) {
3085                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3086                                         "9074 BLKGRD:"
3087                                         " %s: Too many sg segments from "
3088                                         "dma_map_sg.  Config %d, seg_cnt %d\n",
3089                                         __func__, phba->cfg_sg_seg_cnt,
3090                                         lpfc_cmd->seg_cnt);
3091                         WARN_ON_ONCE(lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt);
3092                         lpfc_cmd->seg_cnt = 0;
3093                         scsi_dma_unmap(scsi_cmnd);
3094                         return 2;
3095                 }
3096
3097                 /*
3098                  * The driver established a maximum scatter-gather segment count
3099                  * during probe that limits the number of sg elements in any
3100                  * single scsi command.  Just run through the seg_cnt and format
3101                  * the sge's.
3102                  * When using SLI-3 the driver will try to fit all the BDEs into
3103                  * the IOCB. If it can't then the BDEs get added to a BPL as it
3104                  * does for SLI-2 mode.
3105                  */
3106
3107                 /* for tracking segment boundaries */
3108                 sgel = scsi_sglist(scsi_cmnd);
3109                 j = 2;
3110                 for (i = 0; i < nseg; i++) {
3111                         sgl->word2 = 0;
3112                         if (nseg == 1) {
3113                                 bf_set(lpfc_sli4_sge_last, sgl, 1);
3114                                 bf_set(lpfc_sli4_sge_type, sgl,
3115                                        LPFC_SGE_TYPE_DATA);
3116                         } else {
3117                                 bf_set(lpfc_sli4_sge_last, sgl, 0);
3118
3119                                 /* do we need to expand the segment */
3120                                 if (!lsp_just_set &&
3121                                     !((j + 1) % phba->border_sge_num) &&
3122                                     ((nseg - 1) != i)) {
3123                                         /* set LSP type */
3124                                         bf_set(lpfc_sli4_sge_type, sgl,
3125                                                LPFC_SGE_TYPE_LSP);
3126
3127                                         sgl_xtra = lpfc_get_sgl_per_hdwq(
3128                                                         phba, lpfc_cmd);
3129
3130                                         if (unlikely(!sgl_xtra)) {
3131                                                 lpfc_cmd->seg_cnt = 0;
3132                                                 scsi_dma_unmap(scsi_cmnd);
3133                                                 return 1;
3134                                         }
3135                                         sgl->addr_lo = cpu_to_le32(putPaddrLow(
3136                                                        sgl_xtra->dma_phys_sgl));
3137                                         sgl->addr_hi = cpu_to_le32(putPaddrHigh(
3138                                                        sgl_xtra->dma_phys_sgl));
3139
3140                                 } else {
3141                                         bf_set(lpfc_sli4_sge_type, sgl,
3142                                                LPFC_SGE_TYPE_DATA);
3143                                 }
3144                         }
3145
3146                         if (!(bf_get(lpfc_sli4_sge_type, sgl) &
3147                                      LPFC_SGE_TYPE_LSP)) {
3148                                 if ((nseg - 1) == i)
3149                                         bf_set(lpfc_sli4_sge_last, sgl, 1);
3150
3151                                 physaddr = sg_dma_address(sgel);
3152                                 dma_len = sg_dma_len(sgel);
3153                                 sgl->addr_lo = cpu_to_le32(putPaddrLow(
3154                                                            physaddr));
3155                                 sgl->addr_hi = cpu_to_le32(putPaddrHigh(
3156                                                            physaddr));
3157
3158                                 bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
3159                                 sgl->word2 = cpu_to_le32(sgl->word2);
3160                                 sgl->sge_len = cpu_to_le32(dma_len);
3161
3162                                 dma_offset += dma_len;
3163                                 sgel = sg_next(sgel);
3164
3165                                 sgl++;
3166                                 lsp_just_set = false;
3167
3168                         } else {
3169                                 sgl->word2 = cpu_to_le32(sgl->word2);
3170                                 sgl->sge_len = cpu_to_le32(
3171                                                      phba->cfg_sg_dma_buf_size);
3172
3173                                 sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
3174                                 i = i - 1;
3175
3176                                 lsp_just_set = true;
3177                         }
3178
3179                         j++;
3180                 }
3181
3182                 /* PBDE support for first data SGE only.
3183                  * For FCoE, we key off Performance Hints.
3184                  * For FC, we key off lpfc_enable_pbde.
3185                  */
3186                 if (nseg == 1 &&
3187                     ((phba->sli3_options & LPFC_SLI4_PERFH_ENABLED) ||
3188                      phba->cfg_enable_pbde)) {
3189                         /* Words 13-15 */
3190                         bde = (struct ulp_bde64 *)
3191                                 &wqe->words[13];
3192                         bde->addrLow = first_data_sgl->addr_lo;
3193                         bde->addrHigh = first_data_sgl->addr_hi;
3194                         bde->tus.f.bdeSize =
3195                                         le32_to_cpu(first_data_sgl->sge_len);
3196                         bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
3197                         bde->tus.w = cpu_to_le32(bde->tus.w);
3198
3199                         /* Word 11 - set PBDE bit */
3200                         bf_set(wqe_pbde, &wqe->generic.wqe_com, 1);
3201                 } else {
3202                         memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3));
3203                         /* Word 11 - PBDE bit disabled by default template */
3204                 }
3205         } else {
3206                 sgl += 1;
3207                 /* set the last flag in the fcp_rsp map entry */
3208                 sgl->word2 = le32_to_cpu(sgl->word2);
3209                 bf_set(lpfc_sli4_sge_last, sgl, 1);
3210                 sgl->word2 = cpu_to_le32(sgl->word2);
3211
3212                 if ((phba->sli3_options & LPFC_SLI4_PERFH_ENABLED) ||
3213                     phba->cfg_enable_pbde) {
3214                         bde = (struct ulp_bde64 *)
3215                                 &wqe->words[13];
3216                         memset(bde, 0, (sizeof(uint32_t) * 3));
3217                 }
3218         }
3219
3220         /*
3221          * Finish initializing those IOCB fields that are dependent on the
3222          * scsi_cmnd request_buffer.  Note that for SLI-2 the bdeSize is
3223          * explicitly reinitialized.
3224          * all iocb memory resources are reused.
3225          */
3226         fcp_cmnd->fcpDl = cpu_to_be32(scsi_bufflen(scsi_cmnd));
3227         /* Set first-burst provided it was successfully negotiated */
3228         if (!(phba->hba_flag & HBA_FCOE_MODE) &&
3229             vport->cfg_first_burst_size &&
3230             scsi_cmnd->sc_data_direction == DMA_TO_DEVICE) {
3231                 u32 init_len, total_len;
3232
3233                 total_len = be32_to_cpu(fcp_cmnd->fcpDl);
3234                 init_len = min(total_len, vport->cfg_first_burst_size);
3235
3236                 /* Word 4 & 5 */
3237                 wqe->fcp_iwrite.initial_xfer_len = init_len;
3238                 wqe->fcp_iwrite.total_xfer_len = total_len;
3239         } else {
3240                 /* Word 4 */
3241                 wqe->fcp_iwrite.total_xfer_len =
3242                         be32_to_cpu(fcp_cmnd->fcpDl);
3243         }
3244
3245         /*
3246          * If the OAS driver feature is enabled and the lun is enabled for
3247          * OAS, set the oas iocb related flags.
3248          */
3249         if ((phba->cfg_fof) && ((struct lpfc_device_data *)
3250                 scsi_cmnd->device->hostdata)->oas_enabled) {
3251                 lpfc_cmd->cur_iocbq.cmd_flag |= (LPFC_IO_OAS | LPFC_IO_FOF);
3252                 lpfc_cmd->cur_iocbq.priority = ((struct lpfc_device_data *)
3253                         scsi_cmnd->device->hostdata)->priority;
3254
3255                 /* Word 10 */
3256                 bf_set(wqe_oas, &wqe->generic.wqe_com, 1);
3257                 bf_set(wqe_ccpe, &wqe->generic.wqe_com, 1);
3258
3259                 if (lpfc_cmd->cur_iocbq.priority)
3260                         bf_set(wqe_ccp, &wqe->generic.wqe_com,
3261                                (lpfc_cmd->cur_iocbq.priority << 1));
3262                 else
3263                         bf_set(wqe_ccp, &wqe->generic.wqe_com,
3264                                (phba->cfg_XLanePriority << 1));
3265         }
3266
3267         return 0;
3268 }
3269
3270 /**
3271  * lpfc_bg_scsi_prep_dma_buf_s4 - DMA mapping for scsi buffer to SLI4 IF spec
3272  * @phba: The Hba for which this call is being executed.
3273  * @lpfc_cmd: The scsi buffer which is going to be mapped.
3274  *
3275  * This is the protection/DIF aware version of
3276  * lpfc_scsi_prep_dma_buf(). It may be a good idea to combine the
3277  * two functions eventually, but for now, it's here
3278  * Return codes:
3279  *      2 - Error - Do not retry
3280  *      1 - Error - Retry
3281  *      0 - Success
3282  **/
3283 static int
3284 lpfc_bg_scsi_prep_dma_buf_s4(struct lpfc_hba *phba,
3285                 struct lpfc_io_buf *lpfc_cmd)
3286 {
3287         struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd;
3288         struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd;
3289         struct sli4_sge *sgl = (struct sli4_sge *)(lpfc_cmd->dma_sgl);
3290         struct lpfc_iocbq *pwqeq = &lpfc_cmd->cur_iocbq;
3291         union lpfc_wqe128 *wqe = &pwqeq->wqe;
3292         uint32_t num_sge = 0;
3293         int datasegcnt, protsegcnt, datadir = scsi_cmnd->sc_data_direction;
3294         int prot_group_type = 0;
3295         int fcpdl;
3296         int ret = 1;
3297         struct lpfc_vport *vport = phba->pport;
3298
3299         /*
3300          * Start the lpfc command prep by bumping the sgl beyond fcp_cmnd
3301          *  fcp_rsp regions to the first data sge entry
3302          */
3303         if (scsi_sg_count(scsi_cmnd)) {
3304                 /*
3305                  * The driver stores the segment count returned from dma_map_sg
3306                  * because this a count of dma-mappings used to map the use_sg
3307                  * pages.  They are not guaranteed to be the same for those
3308                  * architectures that implement an IOMMU.
3309                  */
3310                 datasegcnt = dma_map_sg(&phba->pcidev->dev,
3311                                         scsi_sglist(scsi_cmnd),
3312                                         scsi_sg_count(scsi_cmnd), datadir);
3313                 if (unlikely(!datasegcnt))
3314                         return 1;
3315
3316                 sgl += 1;
3317                 /* clear the last flag in the fcp_rsp map entry */
3318                 sgl->word2 = le32_to_cpu(sgl->word2);
3319                 bf_set(lpfc_sli4_sge_last, sgl, 0);
3320                 sgl->word2 = cpu_to_le32(sgl->word2);
3321
3322                 sgl += 1;
3323                 lpfc_cmd->seg_cnt = datasegcnt;
3324
3325                 /* First check if data segment count from SCSI Layer is good */
3326                 if (lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt &&
3327                     !phba->cfg_xpsgl) {
3328                         WARN_ON_ONCE(lpfc_cmd->seg_cnt > phba->cfg_sg_seg_cnt);
3329                         ret = 2;
3330                         goto err;
3331                 }
3332
3333                 prot_group_type = lpfc_prot_group_type(phba, scsi_cmnd);
3334
3335                 switch (prot_group_type) {
3336                 case LPFC_PG_TYPE_NO_DIF:
3337                         /* Here we need to add a DISEED to the count */
3338                         if (((lpfc_cmd->seg_cnt + 1) >
3339                                         phba->cfg_total_seg_cnt) &&
3340                             !phba->cfg_xpsgl) {
3341                                 ret = 2;
3342                                 goto err;
3343                         }
3344
3345                         num_sge = lpfc_bg_setup_sgl(phba, scsi_cmnd, sgl,
3346                                         datasegcnt, lpfc_cmd);
3347
3348                         /* we should have 2 or more entries in buffer list */
3349                         if (num_sge < 2) {
3350                                 ret = 2;
3351                                 goto err;
3352                         }
3353                         break;
3354
3355                 case LPFC_PG_TYPE_DIF_BUF:
3356                         /*
3357                          * This type indicates that protection buffers are
3358                          * passed to the driver, so that needs to be prepared
3359                          * for DMA
3360                          */
3361                         protsegcnt = dma_map_sg(&phba->pcidev->dev,
3362                                         scsi_prot_sglist(scsi_cmnd),
3363                                         scsi_prot_sg_count(scsi_cmnd), datadir);
3364                         if (unlikely(!protsegcnt)) {
3365                                 scsi_dma_unmap(scsi_cmnd);
3366                                 return 1;
3367                         }
3368
3369                         lpfc_cmd->prot_seg_cnt = protsegcnt;
3370                         /*
3371                          * There is a minimun of 3 SGEs used for every
3372                          * protection data segment.
3373                          */
3374                         if (((lpfc_cmd->prot_seg_cnt * 3) >
3375                                         (phba->cfg_total_seg_cnt - 2)) &&
3376                             !phba->cfg_xpsgl) {
3377                                 ret = 2;
3378                                 goto err;
3379                         }
3380
3381                         num_sge = lpfc_bg_setup_sgl_prot(phba, scsi_cmnd, sgl,
3382                                         datasegcnt, protsegcnt, lpfc_cmd);
3383
3384                         /* we should have 3 or more entries in buffer list */
3385                         if (num_sge < 3 ||
3386                             (num_sge > phba->cfg_total_seg_cnt &&
3387                              !phba->cfg_xpsgl)) {
3388                                 ret = 2;
3389                                 goto err;
3390                         }
3391                         break;
3392
3393                 case LPFC_PG_TYPE_INVALID:
3394                 default:
3395                         scsi_dma_unmap(scsi_cmnd);
3396                         lpfc_cmd->seg_cnt = 0;
3397
3398                         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3399                                         "9083 Unexpected protection group %i\n",
3400                                         prot_group_type);
3401                         return 2;
3402                 }
3403         }
3404
3405         switch (scsi_get_prot_op(scsi_cmnd)) {
3406         case SCSI_PROT_WRITE_STRIP:
3407         case SCSI_PROT_READ_STRIP:
3408                 lpfc_cmd->cur_iocbq.cmd_flag |= LPFC_IO_DIF_STRIP;
3409                 break;
3410         case SCSI_PROT_WRITE_INSERT:
3411         case SCSI_PROT_READ_INSERT:
3412                 lpfc_cmd->cur_iocbq.cmd_flag |= LPFC_IO_DIF_INSERT;
3413                 break;
3414         case SCSI_PROT_WRITE_PASS:
3415         case SCSI_PROT_READ_PASS:
3416                 lpfc_cmd->cur_iocbq.cmd_flag |= LPFC_IO_DIF_PASS;
3417                 break;
3418         }
3419
3420         fcpdl = lpfc_bg_scsi_adjust_dl(phba, lpfc_cmd);
3421         fcp_cmnd->fcpDl = be32_to_cpu(fcpdl);
3422
3423         /* Set first-burst provided it was successfully negotiated */
3424         if (!(phba->hba_flag & HBA_FCOE_MODE) &&
3425             vport->cfg_first_burst_size &&
3426             scsi_cmnd->sc_data_direction == DMA_TO_DEVICE) {
3427                 u32 init_len, total_len;
3428
3429                 total_len = be32_to_cpu(fcp_cmnd->fcpDl);
3430                 init_len = min(total_len, vport->cfg_first_burst_size);
3431
3432                 /* Word 4 & 5 */
3433                 wqe->fcp_iwrite.initial_xfer_len = init_len;
3434                 wqe->fcp_iwrite.total_xfer_len = total_len;
3435         } else {
3436                 /* Word 4 */
3437                 wqe->fcp_iwrite.total_xfer_len =
3438                         be32_to_cpu(fcp_cmnd->fcpDl);
3439         }
3440
3441         /*
3442          * If the OAS driver feature is enabled and the lun is enabled for
3443          * OAS, set the oas iocb related flags.
3444          */
3445         if ((phba->cfg_fof) && ((struct lpfc_device_data *)
3446                 scsi_cmnd->device->hostdata)->oas_enabled) {
3447                 lpfc_cmd->cur_iocbq.cmd_flag |= (LPFC_IO_OAS | LPFC_IO_FOF);
3448
3449                 /* Word 10 */
3450                 bf_set(wqe_oas, &wqe->generic.wqe_com, 1);
3451                 bf_set(wqe_ccpe, &wqe->generic.wqe_com, 1);
3452                 bf_set(wqe_ccp, &wqe->generic.wqe_com,
3453                        (phba->cfg_XLanePriority << 1));
3454         }
3455
3456         /* Word 7. DIF Flags */
3457         if (lpfc_cmd->cur_iocbq.cmd_flag & LPFC_IO_DIF_PASS)
3458                 bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_PASSTHRU);
3459         else if (lpfc_cmd->cur_iocbq.cmd_flag & LPFC_IO_DIF_STRIP)
3460                 bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_STRIP);
3461         else if (lpfc_cmd->cur_iocbq.cmd_flag & LPFC_IO_DIF_INSERT)
3462                 bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_INSERT);
3463
3464         lpfc_cmd->cur_iocbq.cmd_flag &= ~(LPFC_IO_DIF_PASS |
3465                                  LPFC_IO_DIF_STRIP | LPFC_IO_DIF_INSERT);
3466
3467         return 0;
3468 err:
3469         if (lpfc_cmd->seg_cnt)
3470                 scsi_dma_unmap(scsi_cmnd);
3471         if (lpfc_cmd->prot_seg_cnt)
3472                 dma_unmap_sg(&phba->pcidev->dev, scsi_prot_sglist(scsi_cmnd),
3473                              scsi_prot_sg_count(scsi_cmnd),
3474                              scsi_cmnd->sc_data_direction);
3475
3476         lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
3477                         "9084 Cannot setup S/G List for HBA "
3478                         "IO segs %d/%d SGL %d SCSI %d: %d %d %d\n",
3479                         lpfc_cmd->seg_cnt, lpfc_cmd->prot_seg_cnt,
3480                         phba->cfg_total_seg_cnt, phba->cfg_sg_seg_cnt,
3481                         prot_group_type, num_sge, ret);
3482
3483         lpfc_cmd->seg_cnt = 0;
3484         lpfc_cmd->prot_seg_cnt = 0;
3485         return ret;
3486 }
3487
3488 /**
3489  * lpfc_scsi_prep_dma_buf - Wrapper function for DMA mapping of scsi buffer
3490  * @phba: The Hba for which this call is being executed.
3491  * @lpfc_cmd: The scsi buffer which is going to be mapped.
3492  *
3493  * This routine wraps the actual DMA mapping function pointer from the
3494  * lpfc_hba struct.
3495  *
3496  * Return codes:
3497  *      1 - Error
3498  *      0 - Success
3499  **/
3500 static inline int
3501 lpfc_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd)
3502 {
3503         return phba->lpfc_scsi_prep_dma_buf(phba, lpfc_cmd);
3504 }
3505
3506 /**
3507  * lpfc_bg_scsi_prep_dma_buf - Wrapper function for DMA mapping of scsi buffer
3508  * using BlockGuard.
3509  * @phba: The Hba for which this call is being executed.
3510  * @lpfc_cmd: The scsi buffer which is going to be mapped.
3511  *
3512  * This routine wraps the actual DMA mapping function pointer from the
3513  * lpfc_hba struct.
3514  *
3515  * Return codes:
3516  *      1 - Error
3517  *      0 - Success
3518  **/
3519 static inline int
3520 lpfc_bg_scsi_prep_dma_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd)
3521 {
3522         return phba->lpfc_bg_scsi_prep_dma_buf(phba, lpfc_cmd);
3523 }
3524
3525 /**
3526  * lpfc_scsi_prep_cmnd_buf - Wrapper function for IOCB/WQE mapping of scsi
3527  * buffer
3528  * @vport: Pointer to vport object.
3529  * @lpfc_cmd: The scsi buffer which is going to be mapped.
3530  * @tmo: Timeout value for IO
3531  *
3532  * This routine initializes IOCB/WQE data structure from scsi command
3533  *
3534  * Return codes:
3535  *      1 - Error
3536  *      0 - Success
3537  **/
3538 static inline int
3539 lpfc_scsi_prep_cmnd_buf(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd,
3540                         uint8_t tmo)
3541 {
3542         return vport->phba->lpfc_scsi_prep_cmnd_buf(vport, lpfc_cmd, tmo);
3543 }
3544
3545 /**
3546  * lpfc_send_scsi_error_event - Posts an event when there is SCSI error
3547  * @phba: Pointer to hba context object.
3548  * @vport: Pointer to vport object.
3549  * @lpfc_cmd: Pointer to lpfc scsi command which reported the error.
3550  * @fcpi_parm: FCP Initiator parameter.
3551  *
3552  * This function posts an event when there is a SCSI command reporting
3553  * error from the scsi device.
3554  **/
3555 static void
3556 lpfc_send_scsi_error_event(struct lpfc_hba *phba, struct lpfc_vport *vport,
3557                 struct lpfc_io_buf *lpfc_cmd, uint32_t fcpi_parm) {
3558         struct scsi_cmnd *cmnd = lpfc_cmd->pCmd;
3559         struct fcp_rsp *fcprsp = lpfc_cmd->fcp_rsp;
3560         uint32_t resp_info = fcprsp->rspStatus2;
3561         uint32_t scsi_status = fcprsp->rspStatus3;
3562         struct lpfc_fast_path_event *fast_path_evt = NULL;
3563         struct lpfc_nodelist *pnode = lpfc_cmd->rdata->pnode;
3564         unsigned long flags;
3565
3566         if (!pnode)
3567                 return;
3568
3569         /* If there is queuefull or busy condition send a scsi event */
3570         if ((cmnd->result == SAM_STAT_TASK_SET_FULL) ||
3571                 (cmnd->result == SAM_STAT_BUSY)) {
3572                 fast_path_evt = lpfc_alloc_fast_evt(phba);
3573                 if (!fast_path_evt)
3574                         return;
3575                 fast_path_evt->un.scsi_evt.event_type =
3576                         FC_REG_SCSI_EVENT;
3577                 fast_path_evt->un.scsi_evt.subcategory =
3578                 (cmnd->result == SAM_STAT_TASK_SET_FULL) ?
3579                 LPFC_EVENT_QFULL : LPFC_EVENT_DEVBSY;
3580                 fast_path_evt->un.scsi_evt.lun = cmnd->device->lun;
3581                 memcpy(&fast_path_evt->un.scsi_evt.wwpn,
3582                         &pnode->nlp_portname, sizeof(struct lpfc_name));
3583                 memcpy(&fast_path_evt->un.scsi_evt.wwnn,
3584                         &pnode->nlp_nodename, sizeof(struct lpfc_name));
3585         } else if ((resp_info & SNS_LEN_VALID) && fcprsp->rspSnsLen &&
3586                 ((cmnd->cmnd[0] == READ_10) || (cmnd->cmnd[0] == WRITE_10))) {
3587                 fast_path_evt = lpfc_alloc_fast_evt(phba);
3588                 if (!fast_path_evt)
3589                         return;
3590                 fast_path_evt->un.check_cond_evt.scsi_event.event_type =
3591                         FC_REG_SCSI_EVENT;
3592                 fast_path_evt->un.check_cond_evt.scsi_event.subcategory =
3593                         LPFC_EVENT_CHECK_COND;
3594                 fast_path_evt->un.check_cond_evt.scsi_event.lun =
3595                         cmnd->device->lun;
3596                 memcpy(&fast_path_evt->un.check_cond_evt.scsi_event.wwpn,
3597                         &pnode->nlp_portname, sizeof(struct lpfc_name));
3598                 memcpy(&fast_path_evt->un.check_cond_evt.scsi_event.wwnn,
3599                         &pnode->nlp_nodename, sizeof(struct lpfc_name));
3600                 fast_path_evt->un.check_cond_evt.sense_key =
3601                         cmnd->sense_buffer[2] & 0xf;
3602                 fast_path_evt->un.check_cond_evt.asc = cmnd->sense_buffer[12];
3603                 fast_path_evt->un.check_cond_evt.ascq = cmnd->sense_buffer[13];
3604         } else if ((cmnd->sc_data_direction == DMA_FROM_DEVICE) &&
3605                      fcpi_parm &&
3606                      ((be32_to_cpu(fcprsp->rspResId) != fcpi_parm) ||
3607                         ((scsi_status == SAM_STAT_GOOD) &&
3608                         !(resp_info & (RESID_UNDER | RESID_OVER))))) {
3609                 /*
3610                  * If status is good or resid does not match with fcp_param and
3611                  * there is valid fcpi_parm, then there is a read_check error
3612                  */
3613                 fast_path_evt = lpfc_alloc_fast_evt(phba);
3614                 if (!fast_path_evt)
3615                         return;
3616                 fast_path_evt->un.read_check_error.header.event_type =
3617                         FC_REG_FABRIC_EVENT;
3618                 fast_path_evt->un.read_check_error.header.subcategory =
3619                         LPFC_EVENT_FCPRDCHKERR;
3620                 memcpy(&fast_path_evt->un.read_check_error.header.wwpn,
3621                         &pnode->nlp_portname, sizeof(struct lpfc_name));
3622                 memcpy(&fast_path_evt->un.read_check_error.header.wwnn,
3623                         &pnode->nlp_nodename, sizeof(struct lpfc_name));
3624                 fast_path_evt->un.read_check_error.lun = cmnd->device->lun;
3625                 fast_path_evt->un.read_check_error.opcode = cmnd->cmnd[0];
3626                 fast_path_evt->un.read_check_error.fcpiparam =
3627                         fcpi_parm;
3628         } else
3629                 return;
3630
3631         fast_path_evt->vport = vport;
3632         spin_lock_irqsave(&phba->hbalock, flags);
3633         list_add_tail(&fast_path_evt->work_evt.evt_listp, &phba->work_list);
3634         spin_unlock_irqrestore(&phba->hbalock, flags);
3635         lpfc_worker_wake_up(phba);
3636         return;
3637 }
3638
3639 /**
3640  * lpfc_scsi_unprep_dma_buf - Un-map DMA mapping of SG-list for dev
3641  * @phba: The HBA for which this call is being executed.
3642  * @psb: The scsi buffer which is going to be un-mapped.
3643  *
3644  * This routine does DMA un-mapping of scatter gather list of scsi command
3645  * field of @lpfc_cmd for device with SLI-3 interface spec.
3646  **/
3647 static void
3648 lpfc_scsi_unprep_dma_buf(struct lpfc_hba *phba, struct lpfc_io_buf *psb)
3649 {
3650         /*
3651          * There are only two special cases to consider.  (1) the scsi command
3652          * requested scatter-gather usage or (2) the scsi command allocated
3653          * a request buffer, but did not request use_sg.  There is a third
3654          * case, but it does not require resource deallocation.
3655          */
3656         if (psb->seg_cnt > 0)
3657                 scsi_dma_unmap(psb->pCmd);
3658         if (psb->prot_seg_cnt > 0)
3659                 dma_unmap_sg(&phba->pcidev->dev, scsi_prot_sglist(psb->pCmd),
3660                                 scsi_prot_sg_count(psb->pCmd),
3661                                 psb->pCmd->sc_data_direction);
3662 }
3663
3664 /**
3665  * lpfc_unblock_requests - allow further commands to be queued.
3666  * @phba: pointer to phba object
3667  *
3668  * For single vport, just call scsi_unblock_requests on physical port.
3669  * For multiple vports, send scsi_unblock_requests for all the vports.
3670  */
3671 void
3672 lpfc_unblock_requests(struct lpfc_hba *phba)
3673 {
3674         struct lpfc_vport **vports;
3675         struct Scsi_Host  *shost;
3676         int i;
3677
3678         if (phba->sli_rev == LPFC_SLI_REV4 &&
3679             !phba->sli4_hba.max_cfg_param.vpi_used) {
3680                 shost = lpfc_shost_from_vport(phba->pport);
3681                 scsi_unblock_requests(shost);
3682                 return;
3683         }
3684
3685         vports = lpfc_create_vport_work_array(phba);
3686         if (vports != NULL)
3687                 for (i = 0; i <= phba->max_vports && vports[i] != NULL; i++) {
3688                         shost = lpfc_shost_from_vport(vports[i]);
3689                         scsi_unblock_requests(shost);
3690                 }
3691         lpfc_destroy_vport_work_array(phba, vports);
3692 }
3693
3694 /**
3695  * lpfc_block_requests - prevent further commands from being queued.
3696  * @phba: pointer to phba object
3697  *
3698  * For single vport, just call scsi_block_requests on physical port.
3699  * For multiple vports, send scsi_block_requests for all the vports.
3700  */
3701 void
3702 lpfc_block_requests(struct lpfc_hba *phba)
3703 {
3704         struct lpfc_vport **vports;
3705         struct Scsi_Host  *shost;
3706         int i;
3707
3708         if (atomic_read(&phba->cmf_stop_io))
3709                 return;
3710
3711         if (phba->sli_rev == LPFC_SLI_REV4 &&
3712             !phba->sli4_hba.max_cfg_param.vpi_used) {
3713                 shost = lpfc_shost_from_vport(phba->pport);
3714                 scsi_block_requests(shost);
3715                 return;
3716         }
3717
3718         vports = lpfc_create_vport_work_array(phba);
3719         if (vports != NULL)
3720                 for (i = 0; i <= phba->max_vports && vports[i] != NULL; i++) {
3721                         shost = lpfc_shost_from_vport(vports[i]);
3722                         scsi_block_requests(shost);
3723                 }
3724         lpfc_destroy_vport_work_array(phba, vports);
3725 }
3726
3727 /**
3728  * lpfc_update_cmf_cmpl - Adjust CMF counters for IO completion
3729  * @phba: The HBA for which this call is being executed.
3730  * @time: The latency of the IO that completed (in ns)
3731  * @size: The size of the IO that completed
3732  * @shost: SCSI host the IO completed on (NULL for a NVME IO)
3733  *
3734  * The routine adjusts the various Burst and Bandwidth counters used in
3735  * Congestion management and E2E. If time is set to LPFC_CGN_NOT_SENT,
3736  * that means the IO was never issued to the HBA, so this routine is
3737  * just being called to cleanup the counter from a previous
3738  * lpfc_update_cmf_cmd call.
3739  */
3740 int
3741 lpfc_update_cmf_cmpl(struct lpfc_hba *phba,
3742                      uint64_t time, uint32_t size, struct Scsi_Host *shost)
3743 {
3744         struct lpfc_cgn_stat *cgs;
3745
3746         if (time != LPFC_CGN_NOT_SENT) {
3747                 /* lat is ns coming in, save latency in us */
3748                 if (time < 1000)
3749                         time = 1;
3750                 else
3751                         time = div_u64(time + 500, 1000); /* round it */
3752
3753                 cgs = per_cpu_ptr(phba->cmf_stat, raw_smp_processor_id());
3754                 atomic64_add(size, &cgs->rcv_bytes);
3755                 atomic64_add(time, &cgs->rx_latency);
3756                 atomic_inc(&cgs->rx_io_cnt);
3757         }
3758         return 0;
3759 }
3760
3761 /**
3762  * lpfc_update_cmf_cmd - Adjust CMF counters for IO submission
3763  * @phba: The HBA for which this call is being executed.
3764  * @size: The size of the IO that will be issued
3765  *
3766  * The routine adjusts the various Burst and Bandwidth counters used in
3767  * Congestion management and E2E.
3768  */
3769 int
3770 lpfc_update_cmf_cmd(struct lpfc_hba *phba, uint32_t size)
3771 {
3772         uint64_t total;
3773         struct lpfc_cgn_stat *cgs;
3774         int cpu;
3775
3776         /* At this point we are either LPFC_CFG_MANAGED or LPFC_CFG_MONITOR */
3777         if (phba->cmf_active_mode == LPFC_CFG_MANAGED &&
3778             phba->cmf_max_bytes_per_interval) {
3779                 total = 0;
3780                 for_each_present_cpu(cpu) {
3781                         cgs = per_cpu_ptr(phba->cmf_stat, cpu);
3782                         total += atomic64_read(&cgs->total_bytes);
3783                 }
3784                 if (total >= phba->cmf_max_bytes_per_interval) {
3785                         if (!atomic_xchg(&phba->cmf_bw_wait, 1)) {
3786                                 lpfc_block_requests(phba);
3787                                 phba->cmf_last_ts =
3788                                         lpfc_calc_cmf_latency(phba);
3789                         }
3790                         atomic_inc(&phba->cmf_busy);
3791                         return -EBUSY;
3792                 }
3793                 if (size > atomic_read(&phba->rx_max_read_cnt))
3794                         atomic_set(&phba->rx_max_read_cnt, size);
3795         }
3796
3797         cgs = per_cpu_ptr(phba->cmf_stat, raw_smp_processor_id());
3798         atomic64_add(size, &cgs->total_bytes);
3799         return 0;
3800 }
3801
3802 /**
3803  * lpfc_handle_fcp_err - FCP response handler
3804  * @vport: The virtual port for which this call is being executed.
3805  * @lpfc_cmd: Pointer to lpfc_io_buf data structure.
3806  * @fcpi_parm: FCP Initiator parameter.
3807  *
3808  * This routine is called to process response IOCB with status field
3809  * IOSTAT_FCP_RSP_ERROR. This routine sets result field of scsi command
3810  * based upon SCSI and FCP error.
3811  **/
3812 static void
3813 lpfc_handle_fcp_err(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd,
3814                     uint32_t fcpi_parm)
3815 {
3816         struct scsi_cmnd *cmnd = lpfc_cmd->pCmd;
3817         struct fcp_cmnd *fcpcmd = lpfc_cmd->fcp_cmnd;
3818         struct fcp_rsp *fcprsp = lpfc_cmd->fcp_rsp;
3819         uint32_t resp_info = fcprsp->rspStatus2;
3820         uint32_t scsi_status = fcprsp->rspStatus3;
3821         uint32_t *lp;
3822         uint32_t host_status = DID_OK;
3823         uint32_t rsplen = 0;
3824         uint32_t fcpDl;
3825         uint32_t logit = LOG_FCP | LOG_FCP_ERROR;
3826
3827
3828         /*
3829          *  If this is a task management command, there is no
3830          *  scsi packet associated with this lpfc_cmd.  The driver
3831          *  consumes it.
3832          */
3833         if (fcpcmd->fcpCntl2) {
3834                 scsi_status = 0;
3835                 goto out;
3836         }
3837
3838         if (resp_info & RSP_LEN_VALID) {
3839                 rsplen = be32_to_cpu(fcprsp->rspRspLen);
3840                 if (rsplen != 0 && rsplen != 4 && rsplen != 8) {
3841                         lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
3842                                          "2719 Invalid response length: "
3843                                          "tgt x%x lun x%llx cmnd x%x rsplen "
3844                                          "x%x\n", cmnd->device->id,
3845                                          cmnd->device->lun, cmnd->cmnd[0],
3846                                          rsplen);
3847                         host_status = DID_ERROR;
3848                         goto out;
3849                 }
3850                 if (fcprsp->rspInfo3 != RSP_NO_FAILURE) {
3851                         lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
3852                                  "2757 Protocol failure detected during "
3853                                  "processing of FCP I/O op: "
3854                                  "tgt x%x lun x%llx cmnd x%x rspInfo3 x%x\n",
3855                                  cmnd->device->id,
3856                                  cmnd->device->lun, cmnd->cmnd[0],
3857                                  fcprsp->rspInfo3);
3858                         host_status = DID_ERROR;
3859                         goto out;
3860                 }
3861         }
3862
3863         if ((resp_info & SNS_LEN_VALID) && fcprsp->rspSnsLen) {
3864                 uint32_t snslen = be32_to_cpu(fcprsp->rspSnsLen);
3865                 if (snslen > SCSI_SENSE_BUFFERSIZE)
3866                         snslen = SCSI_SENSE_BUFFERSIZE;
3867
3868                 if (resp_info & RSP_LEN_VALID)
3869                   rsplen = be32_to_cpu(fcprsp->rspRspLen);
3870                 memcpy(cmnd->sense_buffer, &fcprsp->rspInfo0 + rsplen, snslen);
3871         }
3872         lp = (uint32_t *)cmnd->sense_buffer;
3873
3874         /* special handling for under run conditions */
3875         if (!scsi_status && (resp_info & RESID_UNDER)) {
3876                 /* don't log under runs if fcp set... */
3877                 if (vport->cfg_log_verbose & LOG_FCP)
3878                         logit = LOG_FCP_ERROR;
3879                 /* unless operator says so */
3880                 if (vport->cfg_log_verbose & LOG_FCP_UNDER)
3881                         logit = LOG_FCP_UNDER;
3882         }
3883
3884         lpfc_printf_vlog(vport, KERN_WARNING, logit,
3885                          "9024 FCP command x%x failed: x%x SNS x%x x%x "
3886                          "Data: x%x x%x x%x x%x x%x\n",
3887                          cmnd->cmnd[0], scsi_status,
3888                          be32_to_cpu(*lp), be32_to_cpu(*(lp + 3)), resp_info,
3889                          be32_to_cpu(fcprsp->rspResId),
3890                          be32_to_cpu(fcprsp->rspSnsLen),
3891                          be32_to_cpu(fcprsp->rspRspLen),
3892                          fcprsp->rspInfo3);
3893
3894         scsi_set_resid(cmnd, 0);
3895         fcpDl = be32_to_cpu(fcpcmd->fcpDl);
3896         if (resp_info & RESID_UNDER) {
3897                 scsi_set_resid(cmnd, be32_to_cpu(fcprsp->rspResId));
3898
3899                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP_UNDER,
3900                                  "9025 FCP Underrun, expected %d, "
3901                                  "residual %d Data: x%x x%x x%x\n",
3902                                  fcpDl,
3903                                  scsi_get_resid(cmnd), fcpi_parm, cmnd->cmnd[0],
3904                                  cmnd->underflow);
3905
3906                 /*
3907                  * If there is an under run, check if under run reported by
3908                  * storage array is same as the under run reported by HBA.
3909                  * If this is not same, there is a dropped frame.
3910                  */
3911                 if (fcpi_parm && (scsi_get_resid(cmnd) != fcpi_parm)) {
3912                         lpfc_printf_vlog(vport, KERN_WARNING,
3913                                          LOG_FCP | LOG_FCP_ERROR,
3914                                          "9026 FCP Read Check Error "
3915                                          "and Underrun Data: x%x x%x x%x x%x\n",
3916                                          fcpDl,
3917                                          scsi_get_resid(cmnd), fcpi_parm,
3918                                          cmnd->cmnd[0]);
3919                         scsi_set_resid(cmnd, scsi_bufflen(cmnd));
3920                         host_status = DID_ERROR;
3921                 }
3922                 /*
3923                  * The cmnd->underflow is the minimum number of bytes that must
3924                  * be transferred for this command.  Provided a sense condition
3925                  * is not present, make sure the actual amount transferred is at
3926                  * least the underflow value or fail.
3927                  */
3928                 if (!(resp_info & SNS_LEN_VALID) &&
3929                     (scsi_status == SAM_STAT_GOOD) &&
3930                     (scsi_bufflen(cmnd) - scsi_get_resid(cmnd)
3931                      < cmnd->underflow)) {
3932                         lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
3933                                          "9027 FCP command x%x residual "
3934                                          "underrun converted to error "
3935                                          "Data: x%x x%x x%x\n",
3936                                          cmnd->cmnd[0], scsi_bufflen(cmnd),
3937                                          scsi_get_resid(cmnd), cmnd->underflow);
3938                         host_status = DID_ERROR;
3939                 }
3940         } else if (resp_info & RESID_OVER) {
3941                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
3942                                  "9028 FCP command x%x residual overrun error. "
3943                                  "Data: x%x x%x\n", cmnd->cmnd[0],
3944                                  scsi_bufflen(cmnd), scsi_get_resid(cmnd));
3945                 host_status = DID_ERROR;
3946
3947         /*
3948          * Check SLI validation that all the transfer was actually done
3949          * (fcpi_parm should be zero). Apply check only to reads.
3950          */
3951         } else if (fcpi_parm) {
3952                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP | LOG_FCP_ERROR,
3953                                  "9029 FCP %s Check Error Data: "
3954                                  "x%x x%x x%x x%x x%x\n",
3955                                  ((cmnd->sc_data_direction == DMA_FROM_DEVICE) ?
3956                                  "Read" : "Write"),
3957                                  fcpDl, be32_to_cpu(fcprsp->rspResId),
3958                                  fcpi_parm, cmnd->cmnd[0], scsi_status);
3959
3960                 /* There is some issue with the LPe12000 that causes it
3961                  * to miscalculate the fcpi_parm and falsely trip this
3962                  * recovery logic.  Detect this case and don't error when true.
3963                  */
3964                 if (fcpi_parm > fcpDl)
3965                         goto out;
3966
3967                 switch (scsi_status) {
3968                 case SAM_STAT_GOOD:
3969                 case SAM_STAT_CHECK_CONDITION:
3970                         /* Fabric dropped a data frame. Fail any successful
3971                          * command in which we detected dropped frames.
3972                          * A status of good or some check conditions could
3973                          * be considered a successful command.
3974                          */
3975                         host_status = DID_ERROR;
3976                         break;
3977                 }
3978                 scsi_set_resid(cmnd, scsi_bufflen(cmnd));
3979         }
3980
3981  out:
3982         cmnd->result = host_status << 16 | scsi_status;
3983         lpfc_send_scsi_error_event(vport->phba, vport, lpfc_cmd, fcpi_parm);
3984 }
3985
3986 /**
3987  * lpfc_fcp_io_cmd_wqe_cmpl - Complete a FCP IO
3988  * @phba: The hba for which this call is being executed.
3989  * @pwqeIn: The command WQE for the scsi cmnd.
3990  * @pwqeOut: Pointer to driver response WQE object.
3991  *
3992  * This routine assigns scsi command result by looking into response WQE
3993  * status field appropriately. This routine handles QUEUE FULL condition as
3994  * well by ramping down device queue depth.
3995  **/
3996 static void
3997 lpfc_fcp_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
3998                          struct lpfc_iocbq *pwqeOut)
3999 {
4000         struct lpfc_io_buf *lpfc_cmd = pwqeIn->io_buf;
4001         struct lpfc_wcqe_complete *wcqe = &pwqeOut->wcqe_cmpl;
4002         struct lpfc_vport *vport = pwqeIn->vport;
4003         struct lpfc_rport_data *rdata;
4004         struct lpfc_nodelist *ndlp;
4005         struct scsi_cmnd *cmd;
4006         unsigned long flags;
4007         struct lpfc_fast_path_event *fast_path_evt;
4008         struct Scsi_Host *shost;
4009         u32 logit = LOG_FCP;
4010         u32 idx;
4011         u32 lat;
4012         u8 wait_xb_clr = 0;
4013
4014         /* Sanity check on return of outstanding command */
4015         if (!lpfc_cmd) {
4016                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
4017                                  "9032 Null lpfc_cmd pointer. No "
4018                                  "release, skip completion\n");
4019                 return;
4020         }
4021
4022         rdata = lpfc_cmd->rdata;
4023         ndlp = rdata->pnode;
4024
4025         /* Sanity check on return of outstanding command */
4026         cmd = lpfc_cmd->pCmd;
4027         if (!cmd) {
4028                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
4029                                  "9042 I/O completion: Not an active IO\n");
4030                 lpfc_release_scsi_buf(phba, lpfc_cmd);
4031                 return;
4032         }
4033         /* Guard against abort handler being called at same time */
4034         spin_lock(&lpfc_cmd->buf_lock);
4035         idx = lpfc_cmd->cur_iocbq.hba_wqidx;
4036         if (phba->sli4_hba.hdwq)
4037                 phba->sli4_hba.hdwq[idx].scsi_cstat.io_cmpls++;
4038
4039 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
4040         if (unlikely(phba->hdwqstat_on & LPFC_CHECK_SCSI_IO))
4041                 this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);
4042 #endif
4043         shost = cmd->device->host;
4044
4045         lpfc_cmd->status = bf_get(lpfc_wcqe_c_status, wcqe);
4046         lpfc_cmd->result = (wcqe->parameter & IOERR_PARAM_MASK);
4047
4048         lpfc_cmd->flags &= ~LPFC_SBUF_XBUSY;
4049         if (bf_get(lpfc_wcqe_c_xb, wcqe)) {
4050                 lpfc_cmd->flags |= LPFC_SBUF_XBUSY;
4051                 if (phba->cfg_fcp_wait_abts_rsp)
4052                         wait_xb_clr = 1;
4053         }
4054
4055 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
4056         if (lpfc_cmd->prot_data_type) {
4057                 struct scsi_dif_tuple *src = NULL;
4058
4059                 src =  (struct scsi_dif_tuple *)lpfc_cmd->prot_data_segment;
4060                 /*
4061                  * Used to restore any changes to protection
4062                  * data for error injection.
4063                  */
4064                 switch (lpfc_cmd->prot_data_type) {
4065                 case LPFC_INJERR_REFTAG:
4066                         src->ref_tag =
4067                                 lpfc_cmd->prot_data;
4068                         break;
4069                 case LPFC_INJERR_APPTAG:
4070                         src->app_tag =
4071                                 (uint16_t)lpfc_cmd->prot_data;
4072                         break;
4073                 case LPFC_INJERR_GUARD:
4074                         src->guard_tag =
4075                                 (uint16_t)lpfc_cmd->prot_data;
4076                         break;
4077                 default:
4078                         break;
4079                 }
4080
4081                 lpfc_cmd->prot_data = 0;
4082                 lpfc_cmd->prot_data_type = 0;
4083                 lpfc_cmd->prot_data_segment = NULL;
4084         }
4085 #endif
4086         if (unlikely(lpfc_cmd->status)) {
4087                 if (lpfc_cmd->status == IOSTAT_FCP_RSP_ERROR &&
4088                     !lpfc_cmd->fcp_rsp->rspStatus3 &&
4089                     (lpfc_cmd->fcp_rsp->rspStatus2 & RESID_UNDER) &&
4090                     !(vport->cfg_log_verbose & LOG_FCP_UNDER))
4091                         logit = 0;
4092                 else
4093                         logit = LOG_FCP | LOG_FCP_UNDER;
4094                 lpfc_printf_vlog(vport, KERN_WARNING, logit,
4095                                  "9034 FCP cmd x%x failed <%d/%lld> "
4096                                  "status: x%x result: x%x "
4097                                  "sid: x%x did: x%x oxid: x%x "
4098                                  "Data: x%x x%x x%x\n",
4099                                  cmd->cmnd[0],
4100                                  cmd->device ? cmd->device->id : 0xffff,
4101                                  cmd->device ? cmd->device->lun : 0xffff,
4102                                  lpfc_cmd->status, lpfc_cmd->result,
4103                                  vport->fc_myDID,
4104                                  (ndlp) ? ndlp->nlp_DID : 0,
4105                                  lpfc_cmd->cur_iocbq.sli4_xritag,
4106                                  wcqe->parameter, wcqe->total_data_placed,
4107                                  lpfc_cmd->cur_iocbq.iotag);
4108         }
4109
4110         switch (lpfc_cmd->status) {
4111         case CQE_STATUS_SUCCESS:
4112                 cmd->result = DID_OK << 16;
4113                 break;
4114         case CQE_STATUS_FCP_RSP_FAILURE:
4115                 lpfc_handle_fcp_err(vport, lpfc_cmd,
4116                                     pwqeIn->wqe.fcp_iread.total_xfer_len -
4117                                     wcqe->total_data_placed);
4118                 break;
4119         case CQE_STATUS_NPORT_BSY:
4120         case CQE_STATUS_FABRIC_BSY:
4121                 cmd->result = DID_TRANSPORT_DISRUPTED << 16;
4122                 fast_path_evt = lpfc_alloc_fast_evt(phba);
4123                 if (!fast_path_evt)
4124                         break;
4125                 fast_path_evt->un.fabric_evt.event_type =
4126                         FC_REG_FABRIC_EVENT;
4127                 fast_path_evt->un.fabric_evt.subcategory =
4128                         (lpfc_cmd->status == IOSTAT_NPORT_BSY) ?
4129                         LPFC_EVENT_PORT_BUSY : LPFC_EVENT_FABRIC_BUSY;
4130                 if (ndlp) {
4131                         memcpy(&fast_path_evt->un.fabric_evt.wwpn,
4132                                &ndlp->nlp_portname,
4133                                 sizeof(struct lpfc_name));
4134                         memcpy(&fast_path_evt->un.fabric_evt.wwnn,
4135                                &ndlp->nlp_nodename,
4136                                 sizeof(struct lpfc_name));
4137                 }
4138                 fast_path_evt->vport = vport;
4139                 fast_path_evt->work_evt.evt =
4140                         LPFC_EVT_FASTPATH_MGMT_EVT;
4141                 spin_lock_irqsave(&phba->hbalock, flags);
4142                 list_add_tail(&fast_path_evt->work_evt.evt_listp,
4143                               &phba->work_list);
4144                 spin_unlock_irqrestore(&phba->hbalock, flags);
4145                 lpfc_worker_wake_up(phba);
4146                 lpfc_printf_vlog(vport, KERN_WARNING, logit,
4147                                  "9035 Fabric/Node busy FCP cmd x%x failed"
4148                                  " <%d/%lld> "
4149                                  "status: x%x result: x%x "
4150                                  "sid: x%x did: x%x oxid: x%x "
4151                                  "Data: x%x x%x x%x\n",
4152                                  cmd->cmnd[0],
4153                                  cmd->device ? cmd->device->id : 0xffff,
4154                                  cmd->device ? cmd->device->lun : 0xffff,
4155                                  lpfc_cmd->status, lpfc_cmd->result,
4156                                  vport->fc_myDID,
4157                                  (ndlp) ? ndlp->nlp_DID : 0,
4158                                  lpfc_cmd->cur_iocbq.sli4_xritag,
4159                                  wcqe->parameter,
4160                                  wcqe->total_data_placed,
4161                                  lpfc_cmd->cur_iocbq.iocb.ulpIoTag);
4162                 break;
4163         case CQE_STATUS_DI_ERROR:
4164                 if (bf_get(lpfc_wcqe_c_bg_edir, wcqe))
4165                         lpfc_cmd->result = IOERR_RX_DMA_FAILED;
4166                 else
4167                         lpfc_cmd->result = IOERR_TX_DMA_FAILED;
4168                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP | LOG_BG,
4169                                  "9048 DI Error xri x%x status x%x DI ext "
4170                                  "status x%x data placed x%x\n",
4171                                  lpfc_cmd->cur_iocbq.sli4_xritag,
4172                                  lpfc_cmd->status, wcqe->parameter,
4173                                  wcqe->total_data_placed);
4174                 if (scsi_get_prot_op(cmd) != SCSI_PROT_NORMAL) {
4175                         /* BG enabled cmd. Parse BG error */
4176                         lpfc_parse_bg_err(phba, lpfc_cmd, pwqeOut);
4177                         break;
4178                 }
4179                 cmd->result = DID_ERROR << 16;
4180                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_BG,
4181                                  "9040 DI Error on unprotected cmd\n");
4182                 break;
4183         case CQE_STATUS_REMOTE_STOP:
4184                 if (ndlp) {
4185                         /* This I/O was aborted by the target, we don't
4186                          * know the rxid and because we did not send the
4187                          * ABTS we cannot generate and RRQ.
4188                          */
4189                         lpfc_set_rrq_active(phba, ndlp,
4190                                             lpfc_cmd->cur_iocbq.sli4_lxritag,
4191                                             0, 0);
4192                 }
4193                 fallthrough;
4194         case CQE_STATUS_LOCAL_REJECT:
4195                 if (lpfc_cmd->result & IOERR_DRVR_MASK)
4196                         lpfc_cmd->status = IOSTAT_DRIVER_REJECT;
4197                 if (lpfc_cmd->result == IOERR_ELXSEC_KEY_UNWRAP_ERROR ||
4198                     lpfc_cmd->result ==
4199                     IOERR_ELXSEC_KEY_UNWRAP_COMPARE_ERROR ||
4200                     lpfc_cmd->result == IOERR_ELXSEC_CRYPTO_ERROR ||
4201                     lpfc_cmd->result ==
4202                     IOERR_ELXSEC_CRYPTO_COMPARE_ERROR) {
4203                         cmd->result = DID_NO_CONNECT << 16;
4204                         break;
4205                 }
4206                 if (lpfc_cmd->result == IOERR_INVALID_RPI ||
4207                     lpfc_cmd->result == IOERR_LINK_DOWN ||
4208                     lpfc_cmd->result == IOERR_NO_RESOURCES ||
4209                     lpfc_cmd->result == IOERR_ABORT_REQUESTED ||
4210                     lpfc_cmd->result == IOERR_RPI_SUSPENDED ||
4211                     lpfc_cmd->result == IOERR_SLER_CMD_RCV_FAILURE) {
4212                         cmd->result = DID_TRANSPORT_DISRUPTED << 16;
4213                         break;
4214                 }
4215                 lpfc_printf_vlog(vport, KERN_WARNING, logit,
4216                                  "9036 Local Reject FCP cmd x%x failed"
4217                                  " <%d/%lld> "
4218                                  "status: x%x result: x%x "
4219                                  "sid: x%x did: x%x oxid: x%x "
4220                                  "Data: x%x x%x x%x\n",
4221                                  cmd->cmnd[0],
4222                                  cmd->device ? cmd->device->id : 0xffff,
4223                                  cmd->device ? cmd->device->lun : 0xffff,
4224                                  lpfc_cmd->status, lpfc_cmd->result,
4225                                  vport->fc_myDID,
4226                                  (ndlp) ? ndlp->nlp_DID : 0,
4227                                  lpfc_cmd->cur_iocbq.sli4_xritag,
4228                                  wcqe->parameter,
4229                                  wcqe->total_data_placed,
4230                                  lpfc_cmd->cur_iocbq.iocb.ulpIoTag);
4231                 fallthrough;
4232         default:
4233                 cmd->result = DID_ERROR << 16;
4234                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
4235                                  "9037 FCP Completion Error: xri %x "
4236                                  "status x%x result x%x [x%x] "
4237                                  "placed x%x\n",
4238                                  lpfc_cmd->cur_iocbq.sli4_xritag,
4239                                  lpfc_cmd->status, lpfc_cmd->result,
4240                                  wcqe->parameter,
4241                                  wcqe->total_data_placed);
4242         }
4243         if (cmd->result || lpfc_cmd->fcp_rsp->rspSnsLen) {
4244                 u32 *lp = (u32 *)cmd->sense_buffer;
4245
4246                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
4247                                  "9039 Iodone <%d/%llu> cmd x%px, error "
4248                                  "x%x SNS x%x x%x LBA x%llx Data: x%x x%x\n",
4249                                  cmd->device->id, cmd->device->lun, cmd,
4250                                  cmd->result, *lp, *(lp + 3),
4251                                  (cmd->device->sector_size) ?
4252                                  (u64)scsi_get_lba(cmd) : 0,
4253                                  cmd->retries, scsi_get_resid(cmd));
4254         }
4255
4256         if (vport->cfg_max_scsicmpl_time &&
4257             time_after(jiffies, lpfc_cmd->start_time +
4258             msecs_to_jiffies(vport->cfg_max_scsicmpl_time))) {
4259                 spin_lock_irqsave(shost->host_lock, flags);
4260                 if (ndlp) {
4261                         if (ndlp->cmd_qdepth >
4262                                 atomic_read(&ndlp->cmd_pending) &&
4263                                 (atomic_read(&ndlp->cmd_pending) >
4264                                 LPFC_MIN_TGT_QDEPTH) &&
4265                                 (cmd->cmnd[0] == READ_10 ||
4266                                 cmd->cmnd[0] == WRITE_10))
4267                                 ndlp->cmd_qdepth =
4268                                         atomic_read(&ndlp->cmd_pending);
4269
4270                         ndlp->last_change_time = jiffies;
4271                 }
4272                 spin_unlock_irqrestore(shost->host_lock, flags);
4273         }
4274         lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd);
4275
4276 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
4277         if (lpfc_cmd->ts_cmd_start) {
4278                 lpfc_cmd->ts_isr_cmpl = lpfc_cmd->cur_iocbq.isr_timestamp;
4279                 lpfc_cmd->ts_data_io = ktime_get_ns();
4280                 phba->ktime_last_cmd = lpfc_cmd->ts_data_io;
4281                 lpfc_io_ktime(phba, lpfc_cmd);
4282         }
4283 #endif
4284         if (likely(!wait_xb_clr))
4285                 lpfc_cmd->pCmd = NULL;
4286         spin_unlock(&lpfc_cmd->buf_lock);
4287
4288         /* Check if IO qualified for CMF */
4289         if (phba->cmf_active_mode != LPFC_CFG_OFF &&
4290             cmd->sc_data_direction == DMA_FROM_DEVICE &&
4291             (scsi_sg_count(cmd))) {
4292                 /* Used when calculating average latency */
4293                 lat = ktime_get_ns() - lpfc_cmd->rx_cmd_start;
4294                 lpfc_update_cmf_cmpl(phba, lat, scsi_bufflen(cmd), shost);
4295         }
4296
4297         if (wait_xb_clr)
4298                 goto out;
4299
4300         /* The sdev is not guaranteed to be valid post scsi_done upcall. */
4301         scsi_done(cmd);
4302
4303         /*
4304          * If there is an abort thread waiting for command completion
4305          * wake up the thread.
4306          */
4307         spin_lock(&lpfc_cmd->buf_lock);
4308         lpfc_cmd->cur_iocbq.cmd_flag &= ~LPFC_DRIVER_ABORTED;
4309         if (lpfc_cmd->waitq)
4310                 wake_up(lpfc_cmd->waitq);
4311         spin_unlock(&lpfc_cmd->buf_lock);
4312 out:
4313         lpfc_release_scsi_buf(phba, lpfc_cmd);
4314 }
4315
4316 /**
4317  * lpfc_scsi_cmd_iocb_cmpl - Scsi cmnd IOCB completion routine
4318  * @phba: The Hba for which this call is being executed.
4319  * @pIocbIn: The command IOCBQ for the scsi cmnd.
4320  * @pIocbOut: The response IOCBQ for the scsi cmnd.
4321  *
4322  * This routine assigns scsi command result by looking into response IOCB
4323  * status field appropriately. This routine handles QUEUE FULL condition as
4324  * well by ramping down device queue depth.
4325  **/
4326 static void
4327 lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn,
4328                         struct lpfc_iocbq *pIocbOut)
4329 {
4330         struct lpfc_io_buf *lpfc_cmd =
4331                 (struct lpfc_io_buf *) pIocbIn->io_buf;
4332         struct lpfc_vport      *vport = pIocbIn->vport;
4333         struct lpfc_rport_data *rdata = lpfc_cmd->rdata;
4334         struct lpfc_nodelist *pnode = rdata->pnode;
4335         struct scsi_cmnd *cmd;
4336         unsigned long flags;
4337         struct lpfc_fast_path_event *fast_path_evt;
4338         struct Scsi_Host *shost;
4339         int idx;
4340         uint32_t logit = LOG_FCP;
4341
4342         /* Guard against abort handler being called at same time */
4343         spin_lock(&lpfc_cmd->buf_lock);
4344
4345         /* Sanity check on return of outstanding command */
4346         cmd = lpfc_cmd->pCmd;
4347         if (!cmd || !phba) {
4348                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
4349                                  "2621 IO completion: Not an active IO\n");
4350                 spin_unlock(&lpfc_cmd->buf_lock);
4351                 return;
4352         }
4353
4354         idx = lpfc_cmd->cur_iocbq.hba_wqidx;
4355         if (phba->sli4_hba.hdwq)
4356                 phba->sli4_hba.hdwq[idx].scsi_cstat.io_cmpls++;
4357
4358 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
4359         if (unlikely(phba->hdwqstat_on & LPFC_CHECK_SCSI_IO))
4360                 this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);
4361 #endif
4362         shost = cmd->device->host;
4363
4364         lpfc_cmd->result = (pIocbOut->iocb.un.ulpWord[4] & IOERR_PARAM_MASK);
4365         lpfc_cmd->status = pIocbOut->iocb.ulpStatus;
4366         /* pick up SLI4 exchange busy status from HBA */
4367         lpfc_cmd->flags &= ~LPFC_SBUF_XBUSY;
4368         if (pIocbOut->cmd_flag & LPFC_EXCHANGE_BUSY)
4369                 lpfc_cmd->flags |= LPFC_SBUF_XBUSY;
4370
4371 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
4372         if (lpfc_cmd->prot_data_type) {
4373                 struct scsi_dif_tuple *src = NULL;
4374
4375                 src =  (struct scsi_dif_tuple *)lpfc_cmd->prot_data_segment;
4376                 /*
4377                  * Used to restore any changes to protection
4378                  * data for error injection.
4379                  */
4380                 switch (lpfc_cmd->prot_data_type) {
4381                 case LPFC_INJERR_REFTAG:
4382                         src->ref_tag =
4383                                 lpfc_cmd->prot_data;
4384                         break;
4385                 case LPFC_INJERR_APPTAG:
4386                         src->app_tag =
4387                                 (uint16_t)lpfc_cmd->prot_data;
4388                         break;
4389                 case LPFC_INJERR_GUARD:
4390                         src->guard_tag =
4391                                 (uint16_t)lpfc_cmd->prot_data;
4392                         break;
4393                 default:
4394                         break;
4395                 }
4396
4397                 lpfc_cmd->prot_data = 0;
4398                 lpfc_cmd->prot_data_type = 0;
4399                 lpfc_cmd->prot_data_segment = NULL;
4400         }
4401 #endif
4402
4403         if (unlikely(lpfc_cmd->status)) {
4404                 if (lpfc_cmd->status == IOSTAT_LOCAL_REJECT &&
4405                     (lpfc_cmd->result & IOERR_DRVR_MASK))
4406                         lpfc_cmd->status = IOSTAT_DRIVER_REJECT;
4407                 else if (lpfc_cmd->status >= IOSTAT_CNT)
4408                         lpfc_cmd->status = IOSTAT_DEFAULT;
4409                 if (lpfc_cmd->status == IOSTAT_FCP_RSP_ERROR &&
4410                     !lpfc_cmd->fcp_rsp->rspStatus3 &&
4411                     (lpfc_cmd->fcp_rsp->rspStatus2 & RESID_UNDER) &&
4412                     !(vport->cfg_log_verbose & LOG_FCP_UNDER))
4413                         logit = 0;
4414                 else
4415                         logit = LOG_FCP | LOG_FCP_UNDER;
4416                 lpfc_printf_vlog(vport, KERN_WARNING, logit,
4417                          "9030 FCP cmd x%x failed <%d/%lld> "
4418                          "status: x%x result: x%x "
4419                          "sid: x%x did: x%x oxid: x%x "
4420                          "Data: x%x x%x\n",
4421                          cmd->cmnd[0],
4422                          cmd->device ? cmd->device->id : 0xffff,
4423                          cmd->device ? cmd->device->lun : 0xffff,
4424                          lpfc_cmd->status, lpfc_cmd->result,
4425                          vport->fc_myDID,
4426                          (pnode) ? pnode->nlp_DID : 0,
4427                          phba->sli_rev == LPFC_SLI_REV4 ?
4428                              lpfc_cmd->cur_iocbq.sli4_xritag : 0xffff,
4429                          pIocbOut->iocb.ulpContext,
4430                          lpfc_cmd->cur_iocbq.iocb.ulpIoTag);
4431
4432                 switch (lpfc_cmd->status) {
4433                 case IOSTAT_FCP_RSP_ERROR:
4434                         /* Call FCP RSP handler to determine result */
4435                         lpfc_handle_fcp_err(vport, lpfc_cmd,
4436                                             pIocbOut->iocb.un.fcpi.fcpi_parm);
4437                         break;
4438                 case IOSTAT_NPORT_BSY:
4439                 case IOSTAT_FABRIC_BSY:
4440                         cmd->result = DID_TRANSPORT_DISRUPTED << 16;
4441                         fast_path_evt = lpfc_alloc_fast_evt(phba);
4442                         if (!fast_path_evt)
4443                                 break;
4444                         fast_path_evt->un.fabric_evt.event_type =
4445                                 FC_REG_FABRIC_EVENT;
4446                         fast_path_evt->un.fabric_evt.subcategory =
4447                                 (lpfc_cmd->status == IOSTAT_NPORT_BSY) ?
4448                                 LPFC_EVENT_PORT_BUSY : LPFC_EVENT_FABRIC_BUSY;
4449                         if (pnode) {
4450                                 memcpy(&fast_path_evt->un.fabric_evt.wwpn,
4451                                         &pnode->nlp_portname,
4452                                         sizeof(struct lpfc_name));
4453                                 memcpy(&fast_path_evt->un.fabric_evt.wwnn,
4454                                         &pnode->nlp_nodename,
4455                                         sizeof(struct lpfc_name));
4456                         }
4457                         fast_path_evt->vport = vport;
4458                         fast_path_evt->work_evt.evt =
4459                                 LPFC_EVT_FASTPATH_MGMT_EVT;
4460                         spin_lock_irqsave(&phba->hbalock, flags);
4461                         list_add_tail(&fast_path_evt->work_evt.evt_listp,
4462                                 &phba->work_list);
4463                         spin_unlock_irqrestore(&phba->hbalock, flags);
4464                         lpfc_worker_wake_up(phba);
4465                         break;
4466                 case IOSTAT_LOCAL_REJECT:
4467                 case IOSTAT_REMOTE_STOP:
4468                         if (lpfc_cmd->result == IOERR_ELXSEC_KEY_UNWRAP_ERROR ||
4469                             lpfc_cmd->result ==
4470                                         IOERR_ELXSEC_KEY_UNWRAP_COMPARE_ERROR ||
4471                             lpfc_cmd->result == IOERR_ELXSEC_CRYPTO_ERROR ||
4472                             lpfc_cmd->result ==
4473                                         IOERR_ELXSEC_CRYPTO_COMPARE_ERROR) {
4474                                 cmd->result = DID_NO_CONNECT << 16;
4475                                 break;
4476                         }
4477                         if (lpfc_cmd->result == IOERR_INVALID_RPI ||
4478                             lpfc_cmd->result == IOERR_NO_RESOURCES ||
4479                             lpfc_cmd->result == IOERR_ABORT_REQUESTED ||
4480                             lpfc_cmd->result == IOERR_SLER_CMD_RCV_FAILURE) {
4481                                 cmd->result = DID_TRANSPORT_DISRUPTED << 16;
4482                                 break;
4483                         }
4484                         if ((lpfc_cmd->result == IOERR_RX_DMA_FAILED ||
4485                              lpfc_cmd->result == IOERR_TX_DMA_FAILED) &&
4486                              pIocbOut->iocb.unsli3.sli3_bg.bgstat) {
4487                                 if (scsi_get_prot_op(cmd) != SCSI_PROT_NORMAL) {
4488                                         /*
4489                                          * This is a response for a BG enabled
4490                                          * cmd. Parse BG error
4491                                          */
4492                                         lpfc_parse_bg_err(phba, lpfc_cmd,
4493                                                         pIocbOut);
4494                                         break;
4495                                 } else {
4496                                         lpfc_printf_vlog(vport, KERN_WARNING,
4497                                                         LOG_BG,
4498                                                         "9031 non-zero BGSTAT "
4499                                                         "on unprotected cmd\n");
4500                                 }
4501                         }
4502                         if ((lpfc_cmd->status == IOSTAT_REMOTE_STOP)
4503                                 && (phba->sli_rev == LPFC_SLI_REV4)
4504                                 && pnode) {
4505                                 /* This IO was aborted by the target, we don't
4506                                  * know the rxid and because we did not send the
4507                                  * ABTS we cannot generate and RRQ.
4508                                  */
4509                                 lpfc_set_rrq_active(phba, pnode,
4510                                         lpfc_cmd->cur_iocbq.sli4_lxritag,
4511                                         0, 0);
4512                         }
4513                         fallthrough;
4514                 default:
4515                         cmd->result = DID_ERROR << 16;
4516                         break;
4517                 }
4518
4519                 if (!pnode || (pnode->nlp_state != NLP_STE_MAPPED_NODE))
4520                         cmd->result = DID_TRANSPORT_DISRUPTED << 16 |
4521                                       SAM_STAT_BUSY;
4522         } else
4523                 cmd->result = DID_OK << 16;
4524
4525         if (cmd->result || lpfc_cmd->fcp_rsp->rspSnsLen) {
4526                 uint32_t *lp = (uint32_t *)cmd->sense_buffer;
4527
4528                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
4529                                  "0710 Iodone <%d/%llu> cmd x%px, error "
4530                                  "x%x SNS x%x x%x Data: x%x x%x\n",
4531                                  cmd->device->id, cmd->device->lun, cmd,
4532                                  cmd->result, *lp, *(lp + 3), cmd->retries,
4533                                  scsi_get_resid(cmd));
4534         }
4535
4536         if (vport->cfg_max_scsicmpl_time &&
4537            time_after(jiffies, lpfc_cmd->start_time +
4538                 msecs_to_jiffies(vport->cfg_max_scsicmpl_time))) {
4539                 spin_lock_irqsave(shost->host_lock, flags);
4540                 if (pnode) {
4541                         if (pnode->cmd_qdepth >
4542                                 atomic_read(&pnode->cmd_pending) &&
4543                                 (atomic_read(&pnode->cmd_pending) >
4544                                 LPFC_MIN_TGT_QDEPTH) &&
4545                                 ((cmd->cmnd[0] == READ_10) ||
4546                                 (cmd->cmnd[0] == WRITE_10)))
4547                                 pnode->cmd_qdepth =
4548                                         atomic_read(&pnode->cmd_pending);
4549
4550                         pnode->last_change_time = jiffies;
4551                 }
4552                 spin_unlock_irqrestore(shost->host_lock, flags);
4553         }
4554         lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd);
4555
4556         lpfc_cmd->pCmd = NULL;
4557         spin_unlock(&lpfc_cmd->buf_lock);
4558
4559 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
4560         if (lpfc_cmd->ts_cmd_start) {
4561                 lpfc_cmd->ts_isr_cmpl = pIocbIn->isr_timestamp;
4562                 lpfc_cmd->ts_data_io = ktime_get_ns();
4563                 phba->ktime_last_cmd = lpfc_cmd->ts_data_io;
4564                 lpfc_io_ktime(phba, lpfc_cmd);
4565         }
4566 #endif
4567
4568         /* The sdev is not guaranteed to be valid post scsi_done upcall. */
4569         scsi_done(cmd);
4570
4571         /*
4572          * If there is an abort thread waiting for command completion
4573          * wake up the thread.
4574          */
4575         spin_lock(&lpfc_cmd->buf_lock);
4576         lpfc_cmd->cur_iocbq.cmd_flag &= ~LPFC_DRIVER_ABORTED;
4577         if (lpfc_cmd->waitq)
4578                 wake_up(lpfc_cmd->waitq);
4579         spin_unlock(&lpfc_cmd->buf_lock);
4580
4581         lpfc_release_scsi_buf(phba, lpfc_cmd);
4582 }
4583
4584 /**
4585  * lpfc_scsi_prep_cmnd_buf_s3 - SLI-3 IOCB init for the IO
4586  * @vport: Pointer to vport object.
4587  * @lpfc_cmd: The scsi buffer which is going to be prep'ed.
4588  * @tmo: timeout value for the IO
4589  *
4590  * Based on the data-direction of the command, initialize IOCB
4591  * in the I/O buffer. Fill in the IOCB fields which are independent
4592  * of the scsi buffer
4593  *
4594  * RETURNS 0 - SUCCESS,
4595  **/
4596 static int lpfc_scsi_prep_cmnd_buf_s3(struct lpfc_vport *vport,
4597                                       struct lpfc_io_buf *lpfc_cmd,
4598                                       uint8_t tmo)
4599 {
4600         IOCB_t *iocb_cmd = &lpfc_cmd->cur_iocbq.iocb;
4601         struct lpfc_iocbq *piocbq = &lpfc_cmd->cur_iocbq;
4602         struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd;
4603         struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd;
4604         struct lpfc_nodelist *pnode = lpfc_cmd->ndlp;
4605         int datadir = scsi_cmnd->sc_data_direction;
4606         u32 fcpdl;
4607
4608         piocbq->iocb.un.fcpi.fcpi_XRdy = 0;
4609
4610         /*
4611          * There are three possibilities here - use scatter-gather segment, use
4612          * the single mapping, or neither.  Start the lpfc command prep by
4613          * bumping the bpl beyond the fcp_cmnd and fcp_rsp regions to the first
4614          * data bde entry.
4615          */
4616         if (scsi_sg_count(scsi_cmnd)) {
4617                 if (datadir == DMA_TO_DEVICE) {
4618                         iocb_cmd->ulpCommand = CMD_FCP_IWRITE64_CR;
4619                         iocb_cmd->ulpPU = PARM_READ_CHECK;
4620                         if (vport->cfg_first_burst_size &&
4621                             (pnode->nlp_flag & NLP_FIRSTBURST)) {
4622                                 u32 xrdy_len;
4623
4624                                 fcpdl = scsi_bufflen(scsi_cmnd);
4625                                 xrdy_len = min(fcpdl,
4626                                                vport->cfg_first_burst_size);
4627                                 piocbq->iocb.un.fcpi.fcpi_XRdy = xrdy_len;
4628                         }
4629                         fcp_cmnd->fcpCntl3 = WRITE_DATA;
4630                 } else {
4631                         iocb_cmd->ulpCommand = CMD_FCP_IREAD64_CR;
4632                         iocb_cmd->ulpPU = PARM_READ_CHECK;
4633                         fcp_cmnd->fcpCntl3 = READ_DATA;
4634                 }
4635         } else {
4636                 iocb_cmd->ulpCommand = CMD_FCP_ICMND64_CR;
4637                 iocb_cmd->un.fcpi.fcpi_parm = 0;
4638                 iocb_cmd->ulpPU = 0;
4639                 fcp_cmnd->fcpCntl3 = 0;
4640         }
4641
4642         /*
4643          * Finish initializing those IOCB fields that are independent
4644          * of the scsi_cmnd request_buffer
4645          */
4646         piocbq->iocb.ulpContext = pnode->nlp_rpi;
4647         if (pnode->nlp_fcp_info & NLP_FCP_2_DEVICE)
4648                 piocbq->iocb.ulpFCP2Rcvy = 1;
4649         else
4650                 piocbq->iocb.ulpFCP2Rcvy = 0;
4651
4652         piocbq->iocb.ulpClass = (pnode->nlp_fcp_info & 0x0f);
4653         piocbq->io_buf  = lpfc_cmd;
4654         if (!piocbq->cmd_cmpl)
4655                 piocbq->cmd_cmpl = lpfc_scsi_cmd_iocb_cmpl;
4656         piocbq->iocb.ulpTimeout = tmo;
4657         piocbq->vport = vport;
4658         return 0;
4659 }
4660
4661 /**
4662  * lpfc_scsi_prep_cmnd_buf_s4 - SLI-4 WQE init for the IO
4663  * @vport: Pointer to vport object.
4664  * @lpfc_cmd: The scsi buffer which is going to be prep'ed.
4665  * @tmo: timeout value for the IO
4666  *
4667  * Based on the data-direction of the command copy WQE template
4668  * to I/O buffer WQE. Fill in the WQE fields which are independent
4669  * of the scsi buffer
4670  *
4671  * RETURNS 0 - SUCCESS,
4672  **/
4673 static int lpfc_scsi_prep_cmnd_buf_s4(struct lpfc_vport *vport,
4674                                       struct lpfc_io_buf *lpfc_cmd,
4675                                       uint8_t tmo)
4676 {
4677         struct lpfc_hba *phba = vport->phba;
4678         struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd;
4679         struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd;
4680         struct lpfc_sli4_hdw_queue *hdwq = NULL;
4681         struct lpfc_iocbq *pwqeq = &lpfc_cmd->cur_iocbq;
4682         struct lpfc_nodelist *pnode = lpfc_cmd->ndlp;
4683         union lpfc_wqe128 *wqe = &pwqeq->wqe;
4684         u16 idx = lpfc_cmd->hdwq_no;
4685         int datadir = scsi_cmnd->sc_data_direction;
4686
4687         hdwq = &phba->sli4_hba.hdwq[idx];
4688
4689         /* Initialize 64 bytes only */
4690         memset(wqe, 0, sizeof(union lpfc_wqe128));
4691
4692         /*
4693          * There are three possibilities here - use scatter-gather segment, use
4694          * the single mapping, or neither.
4695          */
4696         if (scsi_sg_count(scsi_cmnd)) {
4697                 if (datadir == DMA_TO_DEVICE) {
4698                         /* From the iwrite template, initialize words 7 -  11 */
4699                         memcpy(&wqe->words[7],
4700                                &lpfc_iwrite_cmd_template.words[7],
4701                                sizeof(uint32_t) * 5);
4702
4703                         fcp_cmnd->fcpCntl3 = WRITE_DATA;
4704                         if (hdwq)
4705                                 hdwq->scsi_cstat.output_requests++;
4706                 } else {
4707                         /* From the iread template, initialize words 7 - 11 */
4708                         memcpy(&wqe->words[7],
4709                                &lpfc_iread_cmd_template.words[7],
4710                                sizeof(uint32_t) * 5);
4711
4712                         /* Word 7 */
4713                         bf_set(wqe_tmo, &wqe->fcp_iread.wqe_com, tmo);
4714
4715                         fcp_cmnd->fcpCntl3 = READ_DATA;
4716                         if (hdwq)
4717                                 hdwq->scsi_cstat.input_requests++;
4718
4719                         /* For a CMF Managed port, iod must be zero'ed */
4720                         if (phba->cmf_active_mode == LPFC_CFG_MANAGED)
4721                                 bf_set(wqe_iod, &wqe->fcp_iread.wqe_com,
4722                                        LPFC_WQE_IOD_NONE);
4723                 }
4724         } else {
4725                 /* From the icmnd template, initialize words 4 - 11 */
4726                 memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
4727                        sizeof(uint32_t) * 8);
4728
4729                 /* Word 7 */
4730                 bf_set(wqe_tmo, &wqe->fcp_icmd.wqe_com, tmo);
4731
4732                 fcp_cmnd->fcpCntl3 = 0;
4733                 if (hdwq)
4734                         hdwq->scsi_cstat.control_requests++;
4735         }
4736
4737         /*
4738          * Finish initializing those WQE fields that are independent
4739          * of the request_buffer
4740          */
4741
4742          /* Word 3 */
4743         bf_set(payload_offset_len, &wqe->fcp_icmd,
4744                sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp));
4745
4746         /* Word 6 */
4747         bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,
4748                phba->sli4_hba.rpi_ids[pnode->nlp_rpi]);
4749         bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);
4750
4751         /* Word 7*/
4752         if (pnode->nlp_fcp_info & NLP_FCP_2_DEVICE)
4753                 bf_set(wqe_erp, &wqe->generic.wqe_com, 1);
4754
4755         bf_set(wqe_class, &wqe->generic.wqe_com,
4756                (pnode->nlp_fcp_info & 0x0f));
4757
4758          /* Word 8 */
4759         wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
4760
4761         /* Word 9 */
4762         bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);
4763
4764         pwqeq->vport = vport;
4765         pwqeq->io_buf = lpfc_cmd;
4766         pwqeq->hba_wqidx = lpfc_cmd->hdwq_no;
4767         pwqeq->cmd_cmpl = lpfc_fcp_io_cmd_wqe_cmpl;
4768
4769         return 0;
4770 }
4771
4772 /**
4773  * lpfc_scsi_prep_cmnd - Wrapper func for convert scsi cmnd to FCP info unit
4774  * @vport: The virtual port for which this call is being executed.
4775  * @lpfc_cmd: The scsi command which needs to send.
4776  * @pnode: Pointer to lpfc_nodelist.
4777  *
4778  * This routine initializes fcp_cmnd and iocb data structure from scsi command
4779  * to transfer for device with SLI3 interface spec.
4780  **/
4781 static int
4782 lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd,
4783                     struct lpfc_nodelist *pnode)
4784 {
4785         struct scsi_cmnd *scsi_cmnd = lpfc_cmd->pCmd;
4786         struct fcp_cmnd *fcp_cmnd = lpfc_cmd->fcp_cmnd;
4787         u8 *ptr;
4788
4789         if (!pnode)
4790                 return 0;
4791
4792         lpfc_cmd->fcp_rsp->rspSnsLen = 0;
4793         /* clear task management bits */
4794         lpfc_cmd->fcp_cmnd->fcpCntl2 = 0;
4795
4796         int_to_scsilun(lpfc_cmd->pCmd->device->lun,
4797                        &lpfc_cmd->fcp_cmnd->fcp_lun);
4798
4799         ptr = &fcp_cmnd->fcpCdb[0];
4800         memcpy(ptr, scsi_cmnd->cmnd, scsi_cmnd->cmd_len);
4801         if (scsi_cmnd->cmd_len < LPFC_FCP_CDB_LEN) {
4802                 ptr += scsi_cmnd->cmd_len;
4803                 memset(ptr, 0, (LPFC_FCP_CDB_LEN - scsi_cmnd->cmd_len));
4804         }
4805
4806         fcp_cmnd->fcpCntl1 = SIMPLE_Q;
4807
4808         lpfc_scsi_prep_cmnd_buf(vport, lpfc_cmd, lpfc_cmd->timeout);
4809
4810         return 0;
4811 }
4812
4813 /**
4814  * lpfc_scsi_prep_task_mgmt_cmd_s3 - Convert SLI3 scsi TM cmd to FCP info unit
4815  * @vport: The virtual port for which this call is being executed.
4816  * @lpfc_cmd: Pointer to lpfc_io_buf data structure.
4817  * @lun: Logical unit number.
4818  * @task_mgmt_cmd: SCSI task management command.
4819  *
4820  * This routine creates FCP information unit corresponding to @task_mgmt_cmd
4821  * for device with SLI-3 interface spec.
4822  *
4823  * Return codes:
4824  *   0 - Error
4825  *   1 - Success
4826  **/
4827 static int
4828 lpfc_scsi_prep_task_mgmt_cmd_s3(struct lpfc_vport *vport,
4829                                 struct lpfc_io_buf *lpfc_cmd,
4830                                 u64 lun, u8 task_mgmt_cmd)
4831 {
4832         struct lpfc_iocbq *piocbq;
4833         IOCB_t *piocb;
4834         struct fcp_cmnd *fcp_cmnd;
4835         struct lpfc_rport_data *rdata = lpfc_cmd->rdata;
4836         struct lpfc_nodelist *ndlp = rdata->pnode;
4837
4838         if (!ndlp || ndlp->nlp_state != NLP_STE_MAPPED_NODE)
4839                 return 0;
4840
4841         piocbq = &(lpfc_cmd->cur_iocbq);
4842         piocbq->vport = vport;
4843
4844         piocb = &piocbq->iocb;
4845
4846         fcp_cmnd = lpfc_cmd->fcp_cmnd;
4847         /* Clear out any old data in the FCP command area */
4848         memset(fcp_cmnd, 0, sizeof(struct fcp_cmnd));
4849         int_to_scsilun(lun, &fcp_cmnd->fcp_lun);
4850         fcp_cmnd->fcpCntl2 = task_mgmt_cmd;
4851         if (!(vport->phba->sli3_options & LPFC_SLI3_BG_ENABLED))
4852                 lpfc_fcpcmd_to_iocb(piocb->unsli3.fcp_ext.icd, fcp_cmnd);
4853         piocb->ulpCommand = CMD_FCP_ICMND64_CR;
4854         piocb->ulpContext = ndlp->nlp_rpi;
4855         piocb->ulpFCP2Rcvy = (ndlp->nlp_fcp_info & NLP_FCP_2_DEVICE) ? 1 : 0;
4856         piocb->ulpClass = (ndlp->nlp_fcp_info & 0x0f);
4857         piocb->ulpPU = 0;
4858         piocb->un.fcpi.fcpi_parm = 0;
4859
4860         /* ulpTimeout is only one byte */
4861         if (lpfc_cmd->timeout > 0xff) {
4862                 /*
4863                  * Do not timeout the command at the firmware level.
4864                  * The driver will provide the timeout mechanism.
4865                  */
4866                 piocb->ulpTimeout = 0;
4867         } else
4868                 piocb->ulpTimeout = lpfc_cmd->timeout;
4869
4870         return 1;
4871 }
4872
4873 /**
4874  * lpfc_scsi_prep_task_mgmt_cmd_s4 - Convert SLI4 scsi TM cmd to FCP info unit
4875  * @vport: The virtual port for which this call is being executed.
4876  * @lpfc_cmd: Pointer to lpfc_io_buf data structure.
4877  * @lun: Logical unit number.
4878  * @task_mgmt_cmd: SCSI task management command.
4879  *
4880  * This routine creates FCP information unit corresponding to @task_mgmt_cmd
4881  * for device with SLI-4 interface spec.
4882  *
4883  * Return codes:
4884  *   0 - Error
4885  *   1 - Success
4886  **/
4887 static int
4888 lpfc_scsi_prep_task_mgmt_cmd_s4(struct lpfc_vport *vport,
4889                                 struct lpfc_io_buf *lpfc_cmd,
4890                                 u64 lun, u8 task_mgmt_cmd)
4891 {
4892         struct lpfc_iocbq *pwqeq = &lpfc_cmd->cur_iocbq;
4893         union lpfc_wqe128 *wqe = &pwqeq->wqe;
4894         struct fcp_cmnd *fcp_cmnd;
4895         struct lpfc_rport_data *rdata = lpfc_cmd->rdata;
4896         struct lpfc_nodelist *ndlp = rdata->pnode;
4897
4898         if (!ndlp || ndlp->nlp_state != NLP_STE_MAPPED_NODE)
4899                 return 0;
4900
4901         pwqeq->vport = vport;
4902         /* Initialize 64 bytes only */
4903         memset(wqe, 0, sizeof(union lpfc_wqe128));
4904
4905         /* From the icmnd template, initialize words 4 - 11 */
4906         memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
4907                sizeof(uint32_t) * 8);
4908
4909         fcp_cmnd = lpfc_cmd->fcp_cmnd;
4910         /* Clear out any old data in the FCP command area */
4911         memset(fcp_cmnd, 0, sizeof(struct fcp_cmnd));
4912         int_to_scsilun(lun, &fcp_cmnd->fcp_lun);
4913         fcp_cmnd->fcpCntl3 = 0;
4914         fcp_cmnd->fcpCntl2 = task_mgmt_cmd;
4915
4916         bf_set(payload_offset_len, &wqe->fcp_icmd,
4917                sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp));
4918         bf_set(cmd_buff_len, &wqe->fcp_icmd, 0);
4919         bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,  /* ulpContext */
4920                vport->phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
4921         bf_set(wqe_erp, &wqe->fcp_icmd.wqe_com,
4922                ((ndlp->nlp_fcp_info & NLP_FCP_2_DEVICE) ? 1 : 0));
4923         bf_set(wqe_class, &wqe->fcp_icmd.wqe_com,
4924                (ndlp->nlp_fcp_info & 0x0f));
4925
4926         /* ulpTimeout is only one byte */
4927         if (lpfc_cmd->timeout > 0xff) {
4928                 /*
4929                  * Do not timeout the command at the firmware level.
4930                  * The driver will provide the timeout mechanism.
4931                  */
4932                 bf_set(wqe_tmo, &wqe->fcp_icmd.wqe_com, 0);
4933         } else {
4934                 bf_set(wqe_tmo, &wqe->fcp_icmd.wqe_com, lpfc_cmd->timeout);
4935         }
4936
4937         lpfc_prep_embed_io(vport->phba, lpfc_cmd);
4938         bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);
4939         wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
4940         bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);
4941
4942         lpfc_sli4_set_rsp_sgl_last(vport->phba, lpfc_cmd);
4943
4944         return 1;
4945 }
4946
4947 /**
4948  * lpfc_scsi_api_table_setup - Set up scsi api function jump table
4949  * @phba: The hba struct for which this call is being executed.
4950  * @dev_grp: The HBA PCI-Device group number.
4951  *
4952  * This routine sets up the SCSI interface API function jump table in @phba
4953  * struct.
4954  * Returns: 0 - success, -ENODEV - failure.
4955  **/
4956 int
4957 lpfc_scsi_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp)
4958 {
4959
4960         phba->lpfc_scsi_unprep_dma_buf = lpfc_scsi_unprep_dma_buf;
4961
4962         switch (dev_grp) {
4963         case LPFC_PCI_DEV_LP:
4964                 phba->lpfc_scsi_prep_dma_buf = lpfc_scsi_prep_dma_buf_s3;
4965                 phba->lpfc_bg_scsi_prep_dma_buf = lpfc_bg_scsi_prep_dma_buf_s3;
4966                 phba->lpfc_release_scsi_buf = lpfc_release_scsi_buf_s3;
4967                 phba->lpfc_get_scsi_buf = lpfc_get_scsi_buf_s3;
4968                 phba->lpfc_scsi_prep_cmnd_buf = lpfc_scsi_prep_cmnd_buf_s3;
4969                 phba->lpfc_scsi_prep_task_mgmt_cmd =
4970                                         lpfc_scsi_prep_task_mgmt_cmd_s3;
4971                 break;
4972         case LPFC_PCI_DEV_OC:
4973                 phba->lpfc_scsi_prep_dma_buf = lpfc_scsi_prep_dma_buf_s4;
4974                 phba->lpfc_bg_scsi_prep_dma_buf = lpfc_bg_scsi_prep_dma_buf_s4;
4975                 phba->lpfc_release_scsi_buf = lpfc_release_scsi_buf_s4;
4976                 phba->lpfc_get_scsi_buf = lpfc_get_scsi_buf_s4;
4977                 phba->lpfc_scsi_prep_cmnd_buf = lpfc_scsi_prep_cmnd_buf_s4;
4978                 phba->lpfc_scsi_prep_task_mgmt_cmd =
4979                                         lpfc_scsi_prep_task_mgmt_cmd_s4;
4980                 break;
4981         default:
4982                 lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
4983                                 "1418 Invalid HBA PCI-device group: 0x%x\n",
4984                                 dev_grp);
4985                 return -ENODEV;
4986         }
4987         phba->lpfc_rampdown_queue_depth = lpfc_rampdown_queue_depth;
4988         return 0;
4989 }
4990
4991 /**
4992  * lpfc_tskmgmt_def_cmpl - IOCB completion routine for task management command
4993  * @phba: The Hba for which this call is being executed.
4994  * @cmdiocbq: Pointer to lpfc_iocbq data structure.
4995  * @rspiocbq: Pointer to lpfc_iocbq data structure.
4996  *
4997  * This routine is IOCB completion routine for device reset and target reset
4998  * routine. This routine release scsi buffer associated with lpfc_cmd.
4999  **/
5000 static void
5001 lpfc_tskmgmt_def_cmpl(struct lpfc_hba *phba,
5002                         struct lpfc_iocbq *cmdiocbq,
5003                         struct lpfc_iocbq *rspiocbq)
5004 {
5005         struct lpfc_io_buf *lpfc_cmd = cmdiocbq->io_buf;
5006         if (lpfc_cmd)
5007                 lpfc_release_scsi_buf(phba, lpfc_cmd);
5008         return;
5009 }
5010
5011 /**
5012  * lpfc_check_pci_resettable - Walks list of devices on pci_dev's bus to check
5013  *                             if issuing a pci_bus_reset is possibly unsafe
5014  * @phba: lpfc_hba pointer.
5015  *
5016  * Description:
5017  * Walks the bus_list to ensure only PCI devices with Emulex
5018  * vendor id, device ids that support hot reset, and only one occurrence
5019  * of function 0.
5020  *
5021  * Returns:
5022  * -EBADSLT,  detected invalid device
5023  *      0,    successful
5024  */
5025 int
5026 lpfc_check_pci_resettable(struct lpfc_hba *phba)
5027 {
5028         const struct pci_dev *pdev = phba->pcidev;
5029         struct pci_dev *ptr = NULL;
5030         u8 counter = 0;
5031
5032         /* Walk the list of devices on the pci_dev's bus */
5033         list_for_each_entry(ptr, &pdev->bus->devices, bus_list) {
5034                 /* Check for Emulex Vendor ID */
5035                 if (ptr->vendor != PCI_VENDOR_ID_EMULEX) {
5036                         lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
5037                                         "8346 Non-Emulex vendor found: "
5038                                         "0x%04x\n", ptr->vendor);
5039                         return -EBADSLT;
5040                 }
5041
5042                 /* Check for valid Emulex Device ID */
5043                 if (phba->sli_rev != LPFC_SLI_REV4 ||
5044                     phba->hba_flag & HBA_FCOE_MODE) {
5045                         lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
5046                                         "8347 Incapable PCI reset device: "
5047                                         "0x%04x\n", ptr->device);
5048                         return -EBADSLT;
5049                 }
5050
5051                 /* Check for only one function 0 ID to ensure only one HBA on
5052                  * secondary bus
5053                  */
5054                 if (ptr->devfn == 0) {
5055                         if (++counter > 1) {
5056                                 lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
5057                                                 "8348 More than one device on "
5058                                                 "secondary bus found\n");
5059                                 return -EBADSLT;
5060                         }
5061                 }
5062         }
5063
5064         return 0;
5065 }
5066
5067 /**
5068  * lpfc_info - Info entry point of scsi_host_template data structure
5069  * @host: The scsi host for which this call is being executed.
5070  *
5071  * This routine provides module information about hba.
5072  *
5073  * Reutrn code:
5074  *   Pointer to char - Success.
5075  **/
5076 const char *
5077 lpfc_info(struct Scsi_Host *host)
5078 {
5079         struct lpfc_vport *vport = (struct lpfc_vport *) host->hostdata;
5080         struct lpfc_hba   *phba = vport->phba;
5081         int link_speed = 0;
5082         static char lpfcinfobuf[384];
5083         char tmp[384] = {0};
5084
5085         memset(lpfcinfobuf, 0, sizeof(lpfcinfobuf));
5086         if (phba && phba->pcidev){
5087                 /* Model Description */
5088                 scnprintf(tmp, sizeof(tmp), phba->ModelDesc);
5089                 if (strlcat(lpfcinfobuf, tmp, sizeof(lpfcinfobuf)) >=
5090                     sizeof(lpfcinfobuf))
5091                         goto buffer_done;
5092
5093                 /* PCI Info */
5094                 scnprintf(tmp, sizeof(tmp),
5095                           " on PCI bus %02x device %02x irq %d",
5096                           phba->pcidev->bus->number, phba->pcidev->devfn,
5097                           phba->pcidev->irq);
5098                 if (strlcat(lpfcinfobuf, tmp, sizeof(lpfcinfobuf)) >=
5099                     sizeof(lpfcinfobuf))
5100                         goto buffer_done;
5101
5102                 /* Port Number */
5103                 if (phba->Port[0]) {
5104                         scnprintf(tmp, sizeof(tmp), " port %s", phba->Port);
5105                         if (strlcat(lpfcinfobuf, tmp, sizeof(lpfcinfobuf)) >=
5106                             sizeof(lpfcinfobuf))
5107                                 goto buffer_done;
5108                 }
5109
5110                 /* Link Speed */
5111                 link_speed = lpfc_sli_port_speed_get(phba);
5112                 if (link_speed != 0) {
5113                         scnprintf(tmp, sizeof(tmp),
5114                                   " Logical Link Speed: %d Mbps", link_speed);
5115                         if (strlcat(lpfcinfobuf, tmp, sizeof(lpfcinfobuf)) >=
5116                             sizeof(lpfcinfobuf))
5117                                 goto buffer_done;
5118                 }
5119
5120                 /* PCI resettable */
5121                 if (!lpfc_check_pci_resettable(phba)) {
5122                         scnprintf(tmp, sizeof(tmp), " PCI resettable");
5123                         strlcat(lpfcinfobuf, tmp, sizeof(lpfcinfobuf));
5124                 }
5125         }
5126
5127 buffer_done:
5128         return lpfcinfobuf;
5129 }
5130
5131 /**
5132  * lpfc_poll_rearm_timer - Routine to modify fcp_poll timer of hba
5133  * @phba: The Hba for which this call is being executed.
5134  *
5135  * This routine modifies fcp_poll_timer  field of @phba by cfg_poll_tmo.
5136  * The default value of cfg_poll_tmo is 10 milliseconds.
5137  **/
5138 static __inline__ void lpfc_poll_rearm_timer(struct lpfc_hba * phba)
5139 {
5140         unsigned long  poll_tmo_expires =
5141                 (jiffies + msecs_to_jiffies(phba->cfg_poll_tmo));
5142
5143         if (!list_empty(&phba->sli.sli3_ring[LPFC_FCP_RING].txcmplq))
5144                 mod_timer(&phba->fcp_poll_timer,
5145                           poll_tmo_expires);
5146 }
5147
5148 /**
5149  * lpfc_poll_start_timer - Routine to start fcp_poll_timer of HBA
5150  * @phba: The Hba for which this call is being executed.
5151  *
5152  * This routine starts the fcp_poll_timer of @phba.
5153  **/
5154 void lpfc_poll_start_timer(struct lpfc_hba * phba)
5155 {
5156         lpfc_poll_rearm_timer(phba);
5157 }
5158
5159 /**
5160  * lpfc_poll_timeout - Restart polling timer
5161  * @t: Timer construct where lpfc_hba data structure pointer is obtained.
5162  *
5163  * This routine restarts fcp_poll timer, when FCP ring  polling is enable
5164  * and FCP Ring interrupt is disable.
5165  **/
5166 void lpfc_poll_timeout(struct timer_list *t)
5167 {
5168         struct lpfc_hba *phba = from_timer(phba, t, fcp_poll_timer);
5169
5170         if (phba->cfg_poll & ENABLE_FCP_RING_POLLING) {
5171                 lpfc_sli_handle_fast_ring_event(phba,
5172                         &phba->sli.sli3_ring[LPFC_FCP_RING], HA_R0RE_REQ);
5173
5174                 if (phba->cfg_poll & DISABLE_FCP_RING_INT)
5175                         lpfc_poll_rearm_timer(phba);
5176         }
5177 }
5178
5179 /*
5180  * lpfc_is_command_vm_io - get the UUID from blk cgroup
5181  * @cmd: Pointer to scsi_cmnd data structure
5182  * Returns UUID if present, otherwise NULL
5183  */
5184 static char *lpfc_is_command_vm_io(struct scsi_cmnd *cmd)
5185 {
5186         struct bio *bio = scsi_cmd_to_rq(cmd)->bio;
5187
5188         if (!IS_ENABLED(CONFIG_BLK_CGROUP_FC_APPID) || !bio)
5189                 return NULL;
5190         return blkcg_get_fc_appid(bio);
5191 }
5192
5193 /**
5194  * lpfc_queuecommand - scsi_host_template queuecommand entry point
5195  * @shost: kernel scsi host pointer.
5196  * @cmnd: Pointer to scsi_cmnd data structure.
5197  *
5198  * Driver registers this routine to scsi midlayer to submit a @cmd to process.
5199  * This routine prepares an IOCB from scsi command and provides to firmware.
5200  * The @done callback is invoked after driver finished processing the command.
5201  *
5202  * Return value :
5203  *   0 - Success
5204  *   SCSI_MLQUEUE_HOST_BUSY - Block all devices served by this host temporarily.
5205  **/
5206 static int
5207 lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd)
5208 {
5209         struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
5210         struct lpfc_hba   *phba = vport->phba;
5211         struct lpfc_iocbq *cur_iocbq = NULL;
5212         struct lpfc_rport_data *rdata;
5213         struct lpfc_nodelist *ndlp;
5214         struct lpfc_io_buf *lpfc_cmd;
5215         struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
5216         int err, idx;
5217         u8 *uuid = NULL;
5218         uint64_t start;
5219
5220         start = ktime_get_ns();
5221         rdata = lpfc_rport_data_from_scsi_device(cmnd->device);
5222
5223         /* sanity check on references */
5224         if (unlikely(!rdata) || unlikely(!rport))
5225                 goto out_fail_command;
5226
5227         err = fc_remote_port_chkready(rport);
5228         if (err) {
5229                 cmnd->result = err;
5230                 goto out_fail_command;
5231         }
5232         ndlp = rdata->pnode;
5233
5234         if ((scsi_get_prot_op(cmnd) != SCSI_PROT_NORMAL) &&
5235                 (!(phba->sli3_options & LPFC_SLI3_BG_ENABLED))) {
5236
5237                 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
5238                                 "9058 BLKGRD: ERROR: rcvd protected cmd:%02x"
5239                                 " op:%02x str=%s without registering for"
5240                                 " BlockGuard - Rejecting command\n",
5241                                 cmnd->cmnd[0], scsi_get_prot_op(cmnd),
5242                                 dif_op_str[scsi_get_prot_op(cmnd)]);
5243                 goto out_fail_command;
5244         }
5245
5246         /*
5247          * Catch race where our node has transitioned, but the
5248          * transport is still transitioning.
5249          */
5250         if (!ndlp)
5251                 goto out_tgt_busy1;
5252
5253         /* Check if IO qualifies for CMF */
5254         if (phba->cmf_active_mode != LPFC_CFG_OFF &&
5255             cmnd->sc_data_direction == DMA_FROM_DEVICE &&
5256             (scsi_sg_count(cmnd))) {
5257                 /* Latency start time saved in rx_cmd_start later in routine */
5258                 err = lpfc_update_cmf_cmd(phba, scsi_bufflen(cmnd));
5259                 if (err)
5260                         goto out_tgt_busy1;
5261         }
5262
5263         if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
5264                 if (atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) {
5265                         lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP_ERROR,
5266                                          "3377 Target Queue Full, scsi Id:%d "
5267                                          "Qdepth:%d Pending command:%d"
5268                                          " WWNN:%02x:%02x:%02x:%02x:"
5269                                          "%02x:%02x:%02x:%02x, "
5270                                          " WWPN:%02x:%02x:%02x:%02x:"
5271                                          "%02x:%02x:%02x:%02x",
5272                                          ndlp->nlp_sid, ndlp->cmd_qdepth,
5273                                          atomic_read(&ndlp->cmd_pending),
5274                                          ndlp->nlp_nodename.u.wwn[0],
5275                                          ndlp->nlp_nodename.u.wwn[1],
5276                                          ndlp->nlp_nodename.u.wwn[2],
5277                                          ndlp->nlp_nodename.u.wwn[3],
5278                                          ndlp->nlp_nodename.u.wwn[4],
5279                                          ndlp->nlp_nodename.u.wwn[5],
5280                                          ndlp->nlp_nodename.u.wwn[6],
5281                                          ndlp->nlp_nodename.u.wwn[7],
5282                                          ndlp->nlp_portname.u.wwn[0],
5283                                          ndlp->nlp_portname.u.wwn[1],
5284                                          ndlp->nlp_portname.u.wwn[2],
5285                                          ndlp->nlp_portname.u.wwn[3],
5286                                          ndlp->nlp_portname.u.wwn[4],
5287                                          ndlp->nlp_portname.u.wwn[5],
5288                                          ndlp->nlp_portname.u.wwn[6],
5289                                          ndlp->nlp_portname.u.wwn[7]);
5290                         goto out_tgt_busy2;
5291                 }
5292         }
5293
5294         lpfc_cmd = lpfc_get_scsi_buf(phba, ndlp, cmnd);
5295         if (lpfc_cmd == NULL) {
5296                 lpfc_rampdown_queue_depth(phba);
5297
5298                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP_ERROR,
5299                                  "0707 driver's buffer pool is empty, "
5300                                  "IO busied\n");
5301                 goto out_host_busy;
5302         }
5303         lpfc_cmd->rx_cmd_start = start;
5304
5305         cur_iocbq = &lpfc_cmd->cur_iocbq;
5306         /*
5307          * Store the midlayer's command structure for the completion phase
5308          * and complete the command initialization.
5309          */
5310         lpfc_cmd->pCmd  = cmnd;
5311         lpfc_cmd->rdata = rdata;
5312         lpfc_cmd->ndlp = ndlp;
5313         cur_iocbq->cmd_cmpl = NULL;
5314         cmnd->host_scribble = (unsigned char *)lpfc_cmd;
5315
5316         err = lpfc_scsi_prep_cmnd(vport, lpfc_cmd, ndlp);
5317         if (err)
5318                 goto out_host_busy_release_buf;
5319
5320         if (scsi_get_prot_op(cmnd) != SCSI_PROT_NORMAL) {
5321                 if (vport->phba->cfg_enable_bg) {
5322                         lpfc_printf_vlog(vport,
5323                                          KERN_INFO, LOG_SCSI_CMD,
5324                                          "9033 BLKGRD: rcvd %s cmd:x%x "
5325                                          "reftag x%x cnt %u pt %x\n",
5326                                          dif_op_str[scsi_get_prot_op(cmnd)],
5327                                          cmnd->cmnd[0],
5328                                          scsi_prot_ref_tag(cmnd),
5329                                          scsi_logical_block_count(cmnd),
5330                                          (cmnd->cmnd[1]>>5));
5331                 }
5332                 err = lpfc_bg_scsi_prep_dma_buf(phba, lpfc_cmd);
5333         } else {
5334                 err = lpfc_scsi_prep_dma_buf(phba, lpfc_cmd);
5335         }
5336
5337         if (unlikely(err)) {
5338                 if (err == 2) {
5339                         cmnd->result = DID_ERROR << 16;
5340                         goto out_fail_command_release_buf;
5341                 }
5342                 goto out_host_busy_free_buf;
5343         }
5344
5345         /* check the necessary and sufficient condition to support VMID */
5346         if (lpfc_is_vmid_enabled(phba) &&
5347             (ndlp->vmid_support ||
5348              phba->pport->vmid_priority_tagging ==
5349              LPFC_VMID_PRIO_TAG_ALL_TARGETS)) {
5350                 /* is the I/O generated by a VM, get the associated virtual */
5351                 /* entity id */
5352                 uuid = lpfc_is_command_vm_io(cmnd);
5353
5354                 if (uuid) {
5355                         err = lpfc_vmid_get_appid(vport, uuid,
5356                                         cmnd->sc_data_direction,
5357                                         (union lpfc_vmid_io_tag *)
5358                                                 &cur_iocbq->vmid_tag);
5359                         if (!err)
5360                                 cur_iocbq->cmd_flag |= LPFC_IO_VMID;
5361                 }
5362         }
5363
5364 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
5365         if (unlikely(phba->hdwqstat_on & LPFC_CHECK_SCSI_IO))
5366                 this_cpu_inc(phba->sli4_hba.c_stat->xmt_io);
5367 #endif
5368         /* Issue I/O to adapter */
5369         err = lpfc_sli_issue_fcp_io(phba, LPFC_FCP_RING, cur_iocbq,
5370                                     SLI_IOCB_RET_IOCB);
5371 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
5372         if (start) {
5373                 lpfc_cmd->ts_cmd_start = start;
5374                 lpfc_cmd->ts_last_cmd = phba->ktime_last_cmd;
5375                 lpfc_cmd->ts_cmd_wqput = ktime_get_ns();
5376         } else {
5377                 lpfc_cmd->ts_cmd_start = 0;
5378         }
5379 #endif
5380         if (err) {
5381                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
5382                                  "3376 FCP could not issue iocb err %x "
5383                                  "FCP cmd x%x <%d/%llu> "
5384                                  "sid: x%x did: x%x oxid: x%x "
5385                                  "Data: x%x x%x x%x x%x\n",
5386                                  err, cmnd->cmnd[0],
5387                                  cmnd->device ? cmnd->device->id : 0xffff,
5388                                  cmnd->device ? cmnd->device->lun : (u64)-1,
5389                                  vport->fc_myDID, ndlp->nlp_DID,
5390                                  phba->sli_rev == LPFC_SLI_REV4 ?
5391                                  cur_iocbq->sli4_xritag : 0xffff,
5392                                  phba->sli_rev == LPFC_SLI_REV4 ?
5393                                  phba->sli4_hba.rpi_ids[ndlp->nlp_rpi] :
5394                                  cur_iocbq->iocb.ulpContext,
5395                                  cur_iocbq->iotag,
5396                                  phba->sli_rev == LPFC_SLI_REV4 ?
5397                                  bf_get(wqe_tmo,
5398                                         &cur_iocbq->wqe.generic.wqe_com) :
5399                                  cur_iocbq->iocb.ulpTimeout,
5400                                  (uint32_t)(scsi_cmd_to_rq(cmnd)->timeout / 1000));
5401
5402                 goto out_host_busy_free_buf;
5403         }
5404
5405         if (phba->cfg_poll & ENABLE_FCP_RING_POLLING) {
5406                 lpfc_sli_handle_fast_ring_event(phba,
5407                         &phba->sli.sli3_ring[LPFC_FCP_RING], HA_R0RE_REQ);
5408
5409                 if (phba->cfg_poll & DISABLE_FCP_RING_INT)
5410                         lpfc_poll_rearm_timer(phba);
5411         }
5412
5413         if (phba->cfg_xri_rebalancing)
5414                 lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_cmd->hdwq_no);
5415
5416         return 0;
5417
5418  out_host_busy_free_buf:
5419         idx = lpfc_cmd->hdwq_no;
5420         lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd);
5421         if (phba->sli4_hba.hdwq) {
5422                 switch (lpfc_cmd->fcp_cmnd->fcpCntl3) {
5423                 case WRITE_DATA:
5424                         phba->sli4_hba.hdwq[idx].scsi_cstat.output_requests--;
5425                         break;
5426                 case READ_DATA:
5427                         phba->sli4_hba.hdwq[idx].scsi_cstat.input_requests--;
5428                         break;
5429                 default:
5430                         phba->sli4_hba.hdwq[idx].scsi_cstat.control_requests--;
5431                 }
5432         }
5433  out_host_busy_release_buf:
5434         lpfc_release_scsi_buf(phba, lpfc_cmd);
5435  out_host_busy:
5436         lpfc_update_cmf_cmpl(phba, LPFC_CGN_NOT_SENT, scsi_bufflen(cmnd),
5437                              shost);
5438         return SCSI_MLQUEUE_HOST_BUSY;
5439
5440  out_tgt_busy2:
5441         lpfc_update_cmf_cmpl(phba, LPFC_CGN_NOT_SENT, scsi_bufflen(cmnd),
5442                              shost);
5443  out_tgt_busy1:
5444         return SCSI_MLQUEUE_TARGET_BUSY;
5445
5446  out_fail_command_release_buf:
5447         lpfc_release_scsi_buf(phba, lpfc_cmd);
5448         lpfc_update_cmf_cmpl(phba, LPFC_CGN_NOT_SENT, scsi_bufflen(cmnd),
5449                              shost);
5450
5451  out_fail_command:
5452         scsi_done(cmnd);
5453         return 0;
5454 }
5455
5456 /*
5457  * lpfc_vmid_vport_cleanup - cleans up the resources associated with a vport
5458  * @vport: The virtual port for which this call is being executed.
5459  */
5460 void lpfc_vmid_vport_cleanup(struct lpfc_vport *vport)
5461 {
5462         u32 bucket;
5463         struct lpfc_vmid *cur;
5464
5465         if (vport->port_type == LPFC_PHYSICAL_PORT)
5466                 del_timer_sync(&vport->phba->inactive_vmid_poll);
5467
5468         kfree(vport->qfpa_res);
5469         kfree(vport->vmid_priority.vmid_range);
5470         kfree(vport->vmid);
5471
5472         if (!hash_empty(vport->hash_table))
5473                 hash_for_each(vport->hash_table, bucket, cur, hnode)
5474                         hash_del(&cur->hnode);
5475
5476         vport->qfpa_res = NULL;
5477         vport->vmid_priority.vmid_range = NULL;
5478         vport->vmid = NULL;
5479         vport->cur_vmid_cnt = 0;
5480 }
5481
5482 /**
5483  * lpfc_abort_handler - scsi_host_template eh_abort_handler entry point
5484  * @cmnd: Pointer to scsi_cmnd data structure.
5485  *
5486  * This routine aborts @cmnd pending in base driver.
5487  *
5488  * Return code :
5489  *   0x2003 - Error
5490  *   0x2002 - Success
5491  **/
5492 static int
5493 lpfc_abort_handler(struct scsi_cmnd *cmnd)
5494 {
5495         struct Scsi_Host  *shost = cmnd->device->host;
5496         struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
5497         struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
5498         struct lpfc_hba   *phba = vport->phba;
5499         struct lpfc_iocbq *iocb;
5500         struct lpfc_io_buf *lpfc_cmd;
5501         int ret = SUCCESS, status = 0;
5502         struct lpfc_sli_ring *pring_s4 = NULL;
5503         struct lpfc_sli_ring *pring = NULL;
5504         int ret_val;
5505         unsigned long flags;
5506         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(waitq);
5507
5508         status = fc_block_rport(rport);
5509         if (status != 0 && status != SUCCESS)
5510                 return status;
5511
5512         lpfc_cmd = (struct lpfc_io_buf *)cmnd->host_scribble;
5513         if (!lpfc_cmd)
5514                 return ret;
5515
5516         /* Guard against IO completion being called at same time */
5517         spin_lock_irqsave(&lpfc_cmd->buf_lock, flags);
5518
5519         spin_lock(&phba->hbalock);
5520         /* driver queued commands are in process of being flushed */
5521         if (phba->hba_flag & HBA_IOQ_FLUSH) {
5522                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
5523                         "3168 SCSI Layer abort requested I/O has been "
5524                         "flushed by LLD.\n");
5525                 ret = FAILED;
5526                 goto out_unlock_hba;
5527         }
5528
5529         if (!lpfc_cmd->pCmd) {
5530                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
5531                          "2873 SCSI Layer I/O Abort Request IO CMPL Status "
5532                          "x%x ID %d LUN %llu\n",
5533                          SUCCESS, cmnd->device->id, cmnd->device->lun);
5534                 goto out_unlock_hba;
5535         }
5536
5537         iocb = &lpfc_cmd->cur_iocbq;
5538         if (phba->sli_rev == LPFC_SLI_REV4) {
5539                 pring_s4 = phba->sli4_hba.hdwq[iocb->hba_wqidx].io_wq->pring;
5540                 if (!pring_s4) {
5541                         ret = FAILED;
5542                         goto out_unlock_hba;
5543                 }
5544                 spin_lock(&pring_s4->ring_lock);
5545         }
5546         /* the command is in process of being cancelled */
5547         if (!(iocb->cmd_flag & LPFC_IO_ON_TXCMPLQ)) {
5548                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
5549                         "3169 SCSI Layer abort requested I/O has been "
5550                         "cancelled by LLD.\n");
5551                 ret = FAILED;
5552                 goto out_unlock_ring;
5553         }
5554         /*
5555          * If pCmd field of the corresponding lpfc_io_buf structure
5556          * points to a different SCSI command, then the driver has
5557          * already completed this command, but the midlayer did not
5558          * see the completion before the eh fired. Just return SUCCESS.
5559          */
5560         if (lpfc_cmd->pCmd != cmnd) {
5561                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
5562                         "3170 SCSI Layer abort requested I/O has been "
5563                         "completed by LLD.\n");
5564                 goto out_unlock_ring;
5565         }
5566
5567         WARN_ON(iocb->io_buf != lpfc_cmd);
5568
5569         /* abort issued in recovery is still in progress */
5570         if (iocb->cmd_flag & LPFC_DRIVER_ABORTED) {
5571                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
5572                          "3389 SCSI Layer I/O Abort Request is pending\n");
5573                 if (phba->sli_rev == LPFC_SLI_REV4)
5574                         spin_unlock(&pring_s4->ring_lock);
5575                 spin_unlock(&phba->hbalock);
5576                 spin_unlock_irqrestore(&lpfc_cmd->buf_lock, flags);
5577                 goto wait_for_cmpl;
5578         }
5579
5580         lpfc_cmd->waitq = &waitq;
5581         if (phba->sli_rev == LPFC_SLI_REV4) {
5582                 spin_unlock(&pring_s4->ring_lock);
5583                 ret_val = lpfc_sli4_issue_abort_iotag(phba, iocb,
5584                                                       lpfc_sli_abort_fcp_cmpl);
5585         } else {
5586                 pring = &phba->sli.sli3_ring[LPFC_FCP_RING];
5587                 ret_val = lpfc_sli_issue_abort_iotag(phba, pring, iocb,
5588                                                      lpfc_sli_abort_fcp_cmpl);
5589         }
5590
5591         /* Make sure HBA is alive */
5592         lpfc_issue_hb_tmo(phba);
5593
5594         if (ret_val != IOCB_SUCCESS) {
5595                 /* Indicate the IO is not being aborted by the driver. */
5596                 lpfc_cmd->waitq = NULL;
5597                 ret = FAILED;
5598                 goto out_unlock_hba;
5599         }
5600
5601         /* no longer need the lock after this point */
5602         spin_unlock(&phba->hbalock);
5603         spin_unlock_irqrestore(&lpfc_cmd->buf_lock, flags);
5604
5605         if (phba->cfg_poll & DISABLE_FCP_RING_INT)
5606                 lpfc_sli_handle_fast_ring_event(phba,
5607                         &phba->sli.sli3_ring[LPFC_FCP_RING], HA_R0RE_REQ);
5608
5609 wait_for_cmpl:
5610         /*
5611          * cmd_flag is set to LPFC_DRIVER_ABORTED before we wait
5612          * for abort to complete.
5613          */
5614         wait_event_timeout(waitq,
5615                           (lpfc_cmd->pCmd != cmnd),
5616                            msecs_to_jiffies(2*vport->cfg_devloss_tmo*1000));
5617
5618         spin_lock(&lpfc_cmd->buf_lock);
5619
5620         if (lpfc_cmd->pCmd == cmnd) {
5621                 ret = FAILED;
5622                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
5623                                  "0748 abort handler timed out waiting "
5624                                  "for aborting I/O (xri:x%x) to complete: "
5625                                  "ret %#x, ID %d, LUN %llu\n",
5626                                  iocb->sli4_xritag, ret,
5627                                  cmnd->device->id, cmnd->device->lun);
5628         }
5629
5630         lpfc_cmd->waitq = NULL;
5631
5632         spin_unlock(&lpfc_cmd->buf_lock);
5633         goto out;
5634
5635 out_unlock_ring:
5636         if (phba->sli_rev == LPFC_SLI_REV4)
5637                 spin_unlock(&pring_s4->ring_lock);
5638 out_unlock_hba:
5639         spin_unlock(&phba->hbalock);
5640         spin_unlock_irqrestore(&lpfc_cmd->buf_lock, flags);
5641 out:
5642         lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
5643                          "0749 SCSI Layer I/O Abort Request Status x%x ID %d "
5644                          "LUN %llu\n", ret, cmnd->device->id,
5645                          cmnd->device->lun);
5646         return ret;
5647 }
5648
5649 static char *
5650 lpfc_taskmgmt_name(uint8_t task_mgmt_cmd)
5651 {
5652         switch (task_mgmt_cmd) {
5653         case FCP_ABORT_TASK_SET:
5654                 return "ABORT_TASK_SET";
5655         case FCP_CLEAR_TASK_SET:
5656                 return "FCP_CLEAR_TASK_SET";
5657         case FCP_BUS_RESET:
5658                 return "FCP_BUS_RESET";
5659         case FCP_LUN_RESET:
5660                 return "FCP_LUN_RESET";
5661         case FCP_TARGET_RESET:
5662                 return "FCP_TARGET_RESET";
5663         case FCP_CLEAR_ACA:
5664                 return "FCP_CLEAR_ACA";
5665         case FCP_TERMINATE_TASK:
5666                 return "FCP_TERMINATE_TASK";
5667         default:
5668                 return "unknown";
5669         }
5670 }
5671
5672
5673 /**
5674  * lpfc_check_fcp_rsp - check the returned fcp_rsp to see if task failed
5675  * @vport: The virtual port for which this call is being executed.
5676  * @lpfc_cmd: Pointer to lpfc_io_buf data structure.
5677  *
5678  * This routine checks the FCP RSP INFO to see if the tsk mgmt command succeded
5679  *
5680  * Return code :
5681  *   0x2003 - Error
5682  *   0x2002 - Success
5683  **/
5684 static int
5685 lpfc_check_fcp_rsp(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd)
5686 {
5687         struct fcp_rsp *fcprsp = lpfc_cmd->fcp_rsp;
5688         uint32_t rsp_info;
5689         uint32_t rsp_len;
5690         uint8_t  rsp_info_code;
5691         int ret = FAILED;
5692
5693
5694         if (fcprsp == NULL)
5695                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
5696                                  "0703 fcp_rsp is missing\n");
5697         else {
5698                 rsp_info = fcprsp->rspStatus2;
5699                 rsp_len = be32_to_cpu(fcprsp->rspRspLen);
5700                 rsp_info_code = fcprsp->rspInfo3;
5701
5702
5703                 lpfc_printf_vlog(vport, KERN_INFO,
5704                                  LOG_FCP,
5705                                  "0706 fcp_rsp valid 0x%x,"
5706                                  " rsp len=%d code 0x%x\n",
5707                                  rsp_info,
5708                                  rsp_len, rsp_info_code);
5709
5710                 /* If FCP_RSP_LEN_VALID bit is one, then the FCP_RSP_LEN
5711                  * field specifies the number of valid bytes of FCP_RSP_INFO.
5712                  * The FCP_RSP_LEN field shall be set to 0x04 or 0x08
5713                  */
5714                 if ((fcprsp->rspStatus2 & RSP_LEN_VALID) &&
5715                     ((rsp_len == 8) || (rsp_len == 4))) {
5716                         switch (rsp_info_code) {
5717                         case RSP_NO_FAILURE:
5718                                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
5719                                                  "0715 Task Mgmt No Failure\n");
5720                                 ret = SUCCESS;
5721                                 break;
5722                         case RSP_TM_NOT_SUPPORTED: /* TM rejected */
5723                                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
5724                                                  "0716 Task Mgmt Target "
5725                                                 "reject\n");
5726                                 break;
5727                         case RSP_TM_NOT_COMPLETED: /* TM failed */
5728                                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
5729                                                  "0717 Task Mgmt Target "
5730                                                 "failed TM\n");
5731                                 break;
5732                         case RSP_TM_INVALID_LU: /* TM to invalid LU! */
5733                                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
5734                                                  "0718 Task Mgmt to invalid "
5735                                                 "LUN\n");
5736                                 break;
5737                         }
5738                 }
5739         }
5740         return ret;
5741 }
5742
5743
5744 /**
5745  * lpfc_send_taskmgmt - Generic SCSI Task Mgmt Handler
5746  * @vport: The virtual port for which this call is being executed.
5747  * @rport: Pointer to remote port
5748  * @tgt_id: Target ID of remote device.
5749  * @lun_id: Lun number for the TMF
5750  * @task_mgmt_cmd: type of TMF to send
5751  *
5752  * This routine builds and sends a TMF (SCSI Task Mgmt Function) to
5753  * a remote port.
5754  *
5755  * Return Code:
5756  *   0x2003 - Error
5757  *   0x2002 - Success.
5758  **/
5759 static int
5760 lpfc_send_taskmgmt(struct lpfc_vport *vport, struct fc_rport *rport,
5761                    unsigned int tgt_id, uint64_t lun_id,
5762                    uint8_t task_mgmt_cmd)
5763 {
5764         struct lpfc_hba   *phba = vport->phba;
5765         struct lpfc_io_buf *lpfc_cmd;
5766         struct lpfc_iocbq *iocbq;
5767         struct lpfc_iocbq *iocbqrsp;
5768         struct lpfc_rport_data *rdata;
5769         struct lpfc_nodelist *pnode;
5770         int ret;
5771         int status;
5772
5773         rdata = rport->dd_data;
5774         if (!rdata || !rdata->pnode)
5775                 return FAILED;
5776         pnode = rdata->pnode;
5777
5778         lpfc_cmd = lpfc_get_scsi_buf(phba, rdata->pnode, NULL);
5779         if (lpfc_cmd == NULL)
5780                 return FAILED;
5781         lpfc_cmd->timeout = phba->cfg_task_mgmt_tmo;
5782         lpfc_cmd->rdata = rdata;
5783         lpfc_cmd->pCmd = NULL;
5784         lpfc_cmd->ndlp = pnode;
5785
5786         status = phba->lpfc_scsi_prep_task_mgmt_cmd(vport, lpfc_cmd, lun_id,
5787                                                     task_mgmt_cmd);
5788         if (!status) {
5789                 lpfc_release_scsi_buf(phba, lpfc_cmd);
5790                 return FAILED;
5791         }
5792
5793         iocbq = &lpfc_cmd->cur_iocbq;
5794         iocbqrsp = lpfc_sli_get_iocbq(phba);
5795         if (iocbqrsp == NULL) {
5796                 lpfc_release_scsi_buf(phba, lpfc_cmd);
5797                 return FAILED;
5798         }
5799         iocbq->cmd_cmpl = lpfc_tskmgmt_def_cmpl;
5800         iocbq->vport = vport;
5801
5802         lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
5803                          "0702 Issue %s to TGT %d LUN %llu "
5804                          "rpi x%x nlp_flag x%x Data: x%x x%x\n",
5805                          lpfc_taskmgmt_name(task_mgmt_cmd), tgt_id, lun_id,
5806                          pnode->nlp_rpi, pnode->nlp_flag, iocbq->sli4_xritag,
5807                          iocbq->cmd_flag);
5808
5809         status = lpfc_sli_issue_iocb_wait(phba, LPFC_FCP_RING,
5810                                           iocbq, iocbqrsp, lpfc_cmd->timeout);
5811         if ((status != IOCB_SUCCESS) ||
5812             (get_job_ulpstatus(phba, iocbqrsp) != IOSTAT_SUCCESS)) {
5813                 if (status != IOCB_SUCCESS ||
5814                     get_job_ulpstatus(phba, iocbqrsp) != IOSTAT_FCP_RSP_ERROR)
5815                         lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
5816                                          "0727 TMF %s to TGT %d LUN %llu "
5817                                          "failed (%d, %d) cmd_flag x%x\n",
5818                                          lpfc_taskmgmt_name(task_mgmt_cmd),
5819                                          tgt_id, lun_id,
5820                                          get_job_ulpstatus(phba, iocbqrsp),
5821                                          get_job_word4(phba, iocbqrsp),
5822                                          iocbq->cmd_flag);
5823                 /* if ulpStatus != IOCB_SUCCESS, then status == IOCB_SUCCESS */
5824                 if (status == IOCB_SUCCESS) {
5825                         if (get_job_ulpstatus(phba, iocbqrsp) ==
5826                             IOSTAT_FCP_RSP_ERROR)
5827                                 /* Something in the FCP_RSP was invalid.
5828                                  * Check conditions */
5829                                 ret = lpfc_check_fcp_rsp(vport, lpfc_cmd);
5830                         else
5831                                 ret = FAILED;
5832                 } else if ((status == IOCB_TIMEDOUT) ||
5833                            (status == IOCB_ABORTED)) {
5834                         ret = TIMEOUT_ERROR;
5835                 } else {
5836                         ret = FAILED;
5837                 }
5838         } else
5839                 ret = SUCCESS;
5840
5841         lpfc_sli_release_iocbq(phba, iocbqrsp);
5842
5843         if (status != IOCB_TIMEDOUT)
5844                 lpfc_release_scsi_buf(phba, lpfc_cmd);
5845
5846         return ret;
5847 }
5848
5849 /**
5850  * lpfc_chk_tgt_mapped -
5851  * @vport: The virtual port to check on
5852  * @rport: Pointer to fc_rport data structure.
5853  *
5854  * This routine delays until the scsi target (aka rport) for the
5855  * command exists (is present and logged in) or we declare it non-existent.
5856  *
5857  * Return code :
5858  *  0x2003 - Error
5859  *  0x2002 - Success
5860  **/
5861 static int
5862 lpfc_chk_tgt_mapped(struct lpfc_vport *vport, struct fc_rport *rport)
5863 {
5864         struct lpfc_rport_data *rdata;
5865         struct lpfc_nodelist *pnode = NULL;
5866         unsigned long later;
5867
5868         rdata = rport->dd_data;
5869         if (!rdata) {
5870                 lpfc_printf_vlog(vport, KERN_INFO, LOG_FCP,
5871                         "0797 Tgt Map rport failure: rdata x%px\n", rdata);
5872                 return FAILED;
5873         }
5874         pnode = rdata->pnode;
5875
5876         /*
5877          * If target is not in a MAPPED state, delay until
5878          * target is rediscovered or devloss timeout expires.
5879          */
5880         later = msecs_to_jiffies(2 * vport->cfg_devloss_tmo * 1000) + jiffies;
5881         while (time_after(later, jiffies)) {
5882                 if (!pnode)
5883                         return FAILED;
5884                 if (pnode->nlp_state == NLP_STE_MAPPED_NODE)
5885                         return SUCCESS;
5886                 schedule_timeout_uninterruptible(msecs_to_jiffies(500));
5887                 rdata = rport->dd_data;
5888                 if (!rdata)
5889                         return FAILED;
5890                 pnode = rdata->pnode;
5891         }
5892         if (!pnode || (pnode->nlp_state != NLP_STE_MAPPED_NODE))
5893                 return FAILED;
5894         return SUCCESS;
5895 }
5896
5897 /**
5898  * lpfc_reset_flush_io_context -
5899  * @vport: The virtual port (scsi_host) for the flush context
5900  * @tgt_id: If aborting by Target contect - specifies the target id
5901  * @lun_id: If aborting by Lun context - specifies the lun id
5902  * @context: specifies the context level to flush at.
5903  *
5904  * After a reset condition via TMF, we need to flush orphaned i/o
5905  * contexts from the adapter. This routine aborts any contexts
5906  * outstanding, then waits for their completions. The wait is
5907  * bounded by devloss_tmo though.
5908  *
5909  * Return code :
5910  *  0x2003 - Error
5911  *  0x2002 - Success
5912  **/
5913 static int
5914 lpfc_reset_flush_io_context(struct lpfc_vport *vport, uint16_t tgt_id,
5915                         uint64_t lun_id, lpfc_ctx_cmd context)
5916 {
5917         struct lpfc_hba   *phba = vport->phba;
5918         unsigned long later;
5919         int cnt;
5920
5921         cnt = lpfc_sli_sum_iocb(vport, tgt_id, lun_id, context);
5922         if (cnt)
5923                 lpfc_sli_abort_taskmgmt(vport,
5924                                         &phba->sli.sli3_ring[LPFC_FCP_RING],
5925                                         tgt_id, lun_id, context);
5926         later = msecs_to_jiffies(2 * vport->cfg_devloss_tmo * 1000) + jiffies;
5927         while (time_after(later, jiffies) && cnt) {
5928                 schedule_timeout_uninterruptible(msecs_to_jiffies(20));
5929                 cnt = lpfc_sli_sum_iocb(vport, tgt_id, lun_id, context);
5930         }
5931         if (cnt) {
5932                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
5933                         "0724 I/O flush failure for context %s : cnt x%x\n",
5934                         ((context == LPFC_CTX_LUN) ? "LUN" :
5935                          ((context == LPFC_CTX_TGT) ? "TGT" :
5936                           ((context == LPFC_CTX_HOST) ? "HOST" : "Unknown"))),
5937                         cnt);
5938                 return FAILED;
5939         }
5940         return SUCCESS;
5941 }
5942
5943 /**
5944  * lpfc_device_reset_handler - scsi_host_template eh_device_reset entry point
5945  * @cmnd: Pointer to scsi_cmnd data structure.
5946  *
5947  * This routine does a device reset by sending a LUN_RESET task management
5948  * command.
5949  *
5950  * Return code :
5951  *  0x2003 - Error
5952  *  0x2002 - Success
5953  **/
5954 static int
5955 lpfc_device_reset_handler(struct scsi_cmnd *cmnd)
5956 {
5957         struct Scsi_Host  *shost = cmnd->device->host;
5958         struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
5959         struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
5960         struct lpfc_rport_data *rdata;
5961         struct lpfc_nodelist *pnode;
5962         unsigned tgt_id = cmnd->device->id;
5963         uint64_t lun_id = cmnd->device->lun;
5964         struct lpfc_scsi_event_header scsi_event;
5965         int status;
5966         u32 logit = LOG_FCP;
5967
5968         if (!rport)
5969                 return FAILED;
5970
5971         rdata = rport->dd_data;
5972         if (!rdata || !rdata->pnode) {
5973                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
5974                                  "0798 Device Reset rdata failure: rdata x%px\n",
5975                                  rdata);
5976                 return FAILED;
5977         }
5978         pnode = rdata->pnode;
5979         status = fc_block_rport(rport);
5980         if (status != 0 && status != SUCCESS)
5981                 return status;
5982
5983         status = lpfc_chk_tgt_mapped(vport, rport);
5984         if (status == FAILED) {
5985                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
5986                         "0721 Device Reset rport failure: rdata x%px\n", rdata);
5987                 return FAILED;
5988         }
5989
5990         scsi_event.event_type = FC_REG_SCSI_EVENT;
5991         scsi_event.subcategory = LPFC_EVENT_LUNRESET;
5992         scsi_event.lun = lun_id;
5993         memcpy(scsi_event.wwpn, &pnode->nlp_portname, sizeof(struct lpfc_name));
5994         memcpy(scsi_event.wwnn, &pnode->nlp_nodename, sizeof(struct lpfc_name));
5995
5996         fc_host_post_vendor_event(shost, fc_get_event_number(),
5997                 sizeof(scsi_event), (char *)&scsi_event, LPFC_NL_VENDOR_ID);
5998
5999         status = lpfc_send_taskmgmt(vport, rport, tgt_id, lun_id,
6000                                                 FCP_LUN_RESET);
6001         if (status != SUCCESS)
6002                 logit =  LOG_TRACE_EVENT;
6003
6004         lpfc_printf_vlog(vport, KERN_ERR, logit,
6005                          "0713 SCSI layer issued Device Reset (%d, %llu) "
6006                          "return x%x\n", tgt_id, lun_id, status);
6007
6008         /*
6009          * We have to clean up i/o as : they may be orphaned by the TMF;
6010          * or if the TMF failed, they may be in an indeterminate state.
6011          * So, continue on.
6012          * We will report success if all the i/o aborts successfully.
6013          */
6014         if (status == SUCCESS)
6015                 status = lpfc_reset_flush_io_context(vport, tgt_id, lun_id,
6016                                                 LPFC_CTX_LUN);
6017
6018         return status;
6019 }
6020
6021 /**
6022  * lpfc_target_reset_handler - scsi_host_template eh_target_reset entry point
6023  * @cmnd: Pointer to scsi_cmnd data structure.
6024  *
6025  * This routine does a target reset by sending a TARGET_RESET task management
6026  * command.
6027  *
6028  * Return code :
6029  *  0x2003 - Error
6030  *  0x2002 - Success
6031  **/
6032 static int
6033 lpfc_target_reset_handler(struct scsi_cmnd *cmnd)
6034 {
6035         struct Scsi_Host  *shost = cmnd->device->host;
6036         struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
6037         struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
6038         struct lpfc_rport_data *rdata;
6039         struct lpfc_nodelist *pnode;
6040         unsigned tgt_id = cmnd->device->id;
6041         uint64_t lun_id = cmnd->device->lun;
6042         struct lpfc_scsi_event_header scsi_event;
6043         int status;
6044         u32 logit = LOG_FCP;
6045         u32 dev_loss_tmo = vport->cfg_devloss_tmo;
6046         unsigned long flags;
6047         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(waitq);
6048
6049         if (!rport)
6050                 return FAILED;
6051
6052         rdata = rport->dd_data;
6053         if (!rdata || !rdata->pnode) {
6054                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
6055                                  "0799 Target Reset rdata failure: rdata x%px\n",
6056                                  rdata);
6057                 return FAILED;
6058         }
6059         pnode = rdata->pnode;
6060         status = fc_block_rport(rport);
6061         if (status != 0 && status != SUCCESS)
6062                 return status;
6063
6064         status = lpfc_chk_tgt_mapped(vport, rport);
6065         if (status == FAILED) {
6066                 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
6067                         "0722 Target Reset rport failure: rdata x%px\n", rdata);
6068                 if (pnode) {
6069                         spin_lock_irqsave(&pnode->lock, flags);
6070                         pnode->nlp_flag &= ~NLP_NPR_ADISC;
6071                         pnode->nlp_fcp_info &= ~NLP_FCP_2_DEVICE;
6072                         spin_unlock_irqrestore(&pnode->lock, flags);
6073                 }
6074                 lpfc_reset_flush_io_context(vport, tgt_id, lun_id,
6075                                           LPFC_CTX_TGT);
6076                 return FAST_IO_FAIL;
6077         }
6078
6079         scsi_event.event_type = FC_REG_SCSI_EVENT;
6080         scsi_event.subcategory = LPFC_EVENT_TGTRESET;
6081         scsi_event.lun = 0;
6082         memcpy(scsi_event.wwpn, &pnode->nlp_portname, sizeof(struct lpfc_name));
6083         memcpy(scsi_event.wwnn, &pnode->nlp_nodename, sizeof(struct lpfc_name));
6084
6085         fc_host_post_vendor_event(shost, fc_get_event_number(),
6086                 sizeof(scsi_event), (char *)&scsi_event, LPFC_NL_VENDOR_ID);
6087
6088         status = lpfc_send_taskmgmt(vport, rport, tgt_id, lun_id,
6089                                         FCP_TARGET_RESET);
6090         if (status != SUCCESS) {
6091                 logit = LOG_TRACE_EVENT;
6092
6093                 /* Issue LOGO, if no LOGO is outstanding */
6094                 spin_lock_irqsave(&pnode->lock, flags);
6095                 if (!(pnode->save_flags & NLP_WAIT_FOR_LOGO) &&
6096                     !pnode->logo_waitq) {
6097                         pnode->logo_waitq = &waitq;
6098                         pnode->nlp_fcp_info &= ~NLP_FCP_2_DEVICE;
6099                         pnode->nlp_flag |= NLP_ISSUE_LOGO;
6100                         pnode->save_flags |= NLP_WAIT_FOR_LOGO;
6101                         spin_unlock_irqrestore(&pnode->lock, flags);
6102                         lpfc_unreg_rpi(vport, pnode);
6103                         wait_event_timeout(waitq,
6104                                            (!(pnode->save_flags &
6105                                               NLP_WAIT_FOR_LOGO)),
6106                                            msecs_to_jiffies(dev_loss_tmo *
6107                                                             1000));
6108
6109                         if (pnode->save_flags & NLP_WAIT_FOR_LOGO) {
6110                                 lpfc_printf_vlog(vport, KERN_ERR, logit,
6111                                                  "0725 SCSI layer TGTRST "
6112                                                  "failed & LOGO TMO (%d, %llu) "
6113                                                  "return x%x\n",
6114                                                  tgt_id, lun_id, status);
6115                                 spin_lock_irqsave(&pnode->lock, flags);
6116                                 pnode->save_flags &= ~NLP_WAIT_FOR_LOGO;
6117                         } else {
6118                                 spin_lock_irqsave(&pnode->lock, flags);
6119                         }
6120                         pnode->logo_waitq = NULL;
6121                         spin_unlock_irqrestore(&pnode->lock, flags);
6122                         status = SUCCESS;
6123
6124                 } else {
6125                         spin_unlock_irqrestore(&pnode->lock, flags);
6126                         status = FAILED;
6127                 }
6128         }
6129
6130         lpfc_printf_vlog(vport, KERN_ERR, logit,
6131                          "0723 SCSI layer issued Target Reset (%d, %llu) "
6132                          "return x%x\n", tgt_id, lun_id, status);
6133
6134         /*
6135          * We have to clean up i/o as : they may be orphaned by the TMF;
6136          * or if the TMF failed, they may be in an indeterminate state.
6137          * So, continue on.
6138          * We will report success if all the i/o aborts successfully.
6139          */
6140         if (status == SUCCESS)
6141                 status = lpfc_reset_flush_io_context(vport, tgt_id, lun_id,
6142                                           LPFC_CTX_TGT);
6143         return status;
6144 }
6145
6146 /**
6147  * lpfc_host_reset_handler - scsi_host_template eh_host_reset_handler entry pt
6148  * @cmnd: Pointer to scsi_cmnd data structure.
6149  *
6150  * This routine does host reset to the adaptor port. It brings the HBA
6151  * offline, performs a board restart, and then brings the board back online.
6152  * The lpfc_offline calls lpfc_sli_hba_down which will abort and local
6153  * reject all outstanding SCSI commands to the host and error returned
6154  * back to SCSI mid-level. As this will be SCSI mid-level's last resort
6155  * of error handling, it will only return error if resetting of the adapter
6156  * is not successful; in all other cases, will return success.
6157  *
6158  * Return code :
6159  *  0x2003 - Error
6160  *  0x2002 - Success
6161  **/
6162 static int
6163 lpfc_host_reset_handler(struct scsi_cmnd *cmnd)
6164 {
6165         struct Scsi_Host *shost = cmnd->device->host;
6166         struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
6167         struct lpfc_hba *phba = vport->phba;
6168         int rc, ret = SUCCESS;
6169
6170         lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
6171                          "3172 SCSI layer issued Host Reset Data:\n");
6172
6173         lpfc_offline_prep(phba, LPFC_MBX_WAIT);
6174         lpfc_offline(phba);
6175         rc = lpfc_sli_brdrestart(phba);
6176         if (rc)
6177                 goto error;
6178
6179         /* Wait for successful restart of adapter */
6180         if (phba->sli_rev < LPFC_SLI_REV4) {
6181                 rc = lpfc_sli_chipset_init(phba);
6182                 if (rc)
6183                         goto error;
6184         }
6185
6186         rc = lpfc_online(phba);
6187         if (rc)
6188                 goto error;
6189
6190         lpfc_unblock_mgmt_io(phba);
6191
6192         return ret;
6193 error:
6194         lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
6195                          "3323 Failed host reset\n");
6196         lpfc_unblock_mgmt_io(phba);
6197         return FAILED;
6198 }
6199
6200 /**
6201  * lpfc_slave_alloc - scsi_host_template slave_alloc entry point
6202  * @sdev: Pointer to scsi_device.
6203  *
6204  * This routine populates the cmds_per_lun count + 2 scsi_bufs into  this host's
6205  * globally available list of scsi buffers. This routine also makes sure scsi
6206  * buffer is not allocated more than HBA limit conveyed to midlayer. This list
6207  * of scsi buffer exists for the lifetime of the driver.
6208  *
6209  * Return codes:
6210  *   non-0 - Error
6211  *   0 - Success
6212  **/
6213 static int
6214 lpfc_slave_alloc(struct scsi_device *sdev)
6215 {
6216         struct lpfc_vport *vport = (struct lpfc_vport *) sdev->host->hostdata;
6217         struct lpfc_hba   *phba = vport->phba;
6218         struct fc_rport *rport = starget_to_rport(scsi_target(sdev));
6219         uint32_t total = 0;
6220         uint32_t num_to_alloc = 0;
6221         int num_allocated = 0;
6222         uint32_t sdev_cnt;
6223         struct lpfc_device_data *device_data;
6224         unsigned long flags;
6225         struct lpfc_name target_wwpn;
6226
6227         if (!rport || fc_remote_port_chkready(rport))
6228                 return -ENXIO;
6229
6230         if (phba->cfg_fof) {
6231
6232                 /*
6233                  * Check to see if the device data structure for the lun
6234                  * exists.  If not, create one.
6235                  */
6236
6237                 u64_to_wwn(rport->port_name, target_wwpn.u.wwn);
6238                 spin_lock_irqsave(&phba->devicelock, flags);
6239                 device_data = __lpfc_get_device_data(phba,
6240                                                      &phba->luns,
6241                                                      &vport->fc_portname,
6242                                                      &target_wwpn,
6243                                                      sdev->lun);
6244                 if (!device_data) {
6245                         spin_unlock_irqrestore(&phba->devicelock, flags);
6246                         device_data = lpfc_create_device_data(phba,
6247                                                         &vport->fc_portname,
6248                                                         &target_wwpn,
6249                                                         sdev->lun,
6250                                                         phba->cfg_XLanePriority,
6251                                                         true);
6252                         if (!device_data)
6253                                 return -ENOMEM;
6254                         spin_lock_irqsave(&phba->devicelock, flags);
6255                         list_add_tail(&device_data->listentry, &phba->luns);
6256                 }
6257                 device_data->rport_data = rport->dd_data;
6258                 device_data->available = true;
6259                 spin_unlock_irqrestore(&phba->devicelock, flags);
6260                 sdev->hostdata = device_data;
6261         } else {
6262                 sdev->hostdata = rport->dd_data;
6263         }
6264         sdev_cnt = atomic_inc_return(&phba->sdev_cnt);
6265
6266         /* For SLI4, all IO buffers are pre-allocated */
6267         if (phba->sli_rev == LPFC_SLI_REV4)
6268                 return 0;
6269
6270         /* This code path is now ONLY for SLI3 adapters */
6271
6272         /*
6273          * Populate the cmds_per_lun count scsi_bufs into this host's globally
6274          * available list of scsi buffers.  Don't allocate more than the
6275          * HBA limit conveyed to the midlayer via the host structure.  The
6276          * formula accounts for the lun_queue_depth + error handlers + 1
6277          * extra.  This list of scsi bufs exists for the lifetime of the driver.
6278          */
6279         total = phba->total_scsi_bufs;
6280         num_to_alloc = vport->cfg_lun_queue_depth + 2;
6281
6282         /* If allocated buffers are enough do nothing */
6283         if ((sdev_cnt * (vport->cfg_lun_queue_depth + 2)) < total)
6284                 return 0;
6285
6286         /* Allow some exchanges to be available always to complete discovery */
6287         if (total >= phba->cfg_hba_queue_depth - LPFC_DISC_IOCB_BUFF_COUNT ) {
6288                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
6289                                  "0704 At limitation of %d preallocated "
6290                                  "command buffers\n", total);
6291                 return 0;
6292         /* Allow some exchanges to be available always to complete discovery */
6293         } else if (total + num_to_alloc >
6294                 phba->cfg_hba_queue_depth - LPFC_DISC_IOCB_BUFF_COUNT ) {
6295                 lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
6296                                  "0705 Allocation request of %d "
6297                                  "command buffers will exceed max of %d.  "
6298                                  "Reducing allocation request to %d.\n",
6299                                  num_to_alloc, phba->cfg_hba_queue_depth,
6300                                  (phba->cfg_hba_queue_depth - total));
6301                 num_to_alloc = phba->cfg_hba_queue_depth - total;
6302         }
6303         num_allocated = lpfc_new_scsi_buf_s3(vport, num_to_alloc);
6304         if (num_to_alloc != num_allocated) {
6305                         lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
6306                                          "0708 Allocation request of %d "
6307                                          "command buffers did not succeed.  "
6308                                          "Allocated %d buffers.\n",
6309                                          num_to_alloc, num_allocated);
6310         }
6311         if (num_allocated > 0)
6312                 phba->total_scsi_bufs += num_allocated;
6313         return 0;
6314 }
6315
6316 /**
6317  * lpfc_slave_configure - scsi_host_template slave_configure entry point
6318  * @sdev: Pointer to scsi_device.
6319  *
6320  * This routine configures following items
6321  *   - Tag command queuing support for @sdev if supported.
6322  *   - Enable SLI polling for fcp ring if ENABLE_FCP_RING_POLLING flag is set.
6323  *
6324  * Return codes:
6325  *   0 - Success
6326  **/
6327 static int
6328 lpfc_slave_configure(struct scsi_device *sdev)
6329 {
6330         struct lpfc_vport *vport = (struct lpfc_vport *) sdev->host->hostdata;
6331         struct lpfc_hba   *phba = vport->phba;
6332
6333         scsi_change_queue_depth(sdev, vport->cfg_lun_queue_depth);
6334
6335         if (phba->cfg_poll & ENABLE_FCP_RING_POLLING) {
6336                 lpfc_sli_handle_fast_ring_event(phba,
6337                         &phba->sli.sli3_ring[LPFC_FCP_RING], HA_R0RE_REQ);
6338                 if (phba->cfg_poll & DISABLE_FCP_RING_INT)
6339                         lpfc_poll_rearm_timer(phba);
6340         }
6341
6342         return 0;
6343 }
6344
6345 /**
6346  * lpfc_slave_destroy - slave_destroy entry point of SHT data structure
6347  * @sdev: Pointer to scsi_device.
6348  *
6349  * This routine sets @sdev hostatdata filed to null.
6350  **/
6351 static void
6352 lpfc_slave_destroy(struct scsi_device *sdev)
6353 {
6354         struct lpfc_vport *vport = (struct lpfc_vport *) sdev->host->hostdata;
6355         struct lpfc_hba   *phba = vport->phba;
6356         unsigned long flags;
6357         struct lpfc_device_data *device_data = sdev->hostdata;
6358
6359         atomic_dec(&phba->sdev_cnt);
6360         if ((phba->cfg_fof) && (device_data)) {
6361                 spin_lock_irqsave(&phba->devicelock, flags);
6362                 device_data->available = false;
6363                 if (!device_data->oas_enabled)
6364                         lpfc_delete_device_data(phba, device_data);
6365                 spin_unlock_irqrestore(&phba->devicelock, flags);
6366         }
6367         sdev->hostdata = NULL;
6368         return;
6369 }
6370
6371 /**
6372  * lpfc_create_device_data - creates and initializes device data structure for OAS
6373  * @phba: Pointer to host bus adapter structure.
6374  * @vport_wwpn: Pointer to vport's wwpn information
6375  * @target_wwpn: Pointer to target's wwpn information
6376  * @lun: Lun on target
6377  * @pri: Priority
6378  * @atomic_create: Flag to indicate if memory should be allocated using the
6379  *                GFP_ATOMIC flag or not.
6380  *
6381  * This routine creates a device data structure which will contain identifying
6382  * information for the device (host wwpn, target wwpn, lun), state of OAS,
6383  * whether or not the corresponding lun is available by the system,
6384  * and pointer to the rport data.
6385  *
6386  * Return codes:
6387  *   NULL - Error
6388  *   Pointer to lpfc_device_data - Success
6389  **/
6390 struct lpfc_device_data*
6391 lpfc_create_device_data(struct lpfc_hba *phba, struct lpfc_name *vport_wwpn,
6392                         struct lpfc_name *target_wwpn, uint64_t lun,
6393                         uint32_t pri, bool atomic_create)
6394 {
6395
6396         struct lpfc_device_data *lun_info;
6397         int memory_flags;
6398
6399         if (unlikely(!phba) || !vport_wwpn || !target_wwpn  ||
6400             !(phba->cfg_fof))
6401                 return NULL;
6402
6403         /* Attempt to create the device data to contain lun info */
6404
6405         if (atomic_create)
6406                 memory_flags = GFP_ATOMIC;
6407         else
6408                 memory_flags = GFP_KERNEL;
6409         lun_info = mempool_alloc(phba->device_data_mem_pool, memory_flags);
6410         if (!lun_info)
6411                 return NULL;
6412         INIT_LIST_HEAD(&lun_info->listentry);
6413         lun_info->rport_data  = NULL;
6414         memcpy(&lun_info->device_id.vport_wwpn, vport_wwpn,
6415                sizeof(struct lpfc_name));
6416         memcpy(&lun_info->device_id.target_wwpn, target_wwpn,
6417                sizeof(struct lpfc_name));
6418         lun_info->device_id.lun = lun;
6419         lun_info->oas_enabled = false;
6420         lun_info->priority = pri;
6421         lun_info->available = false;
6422         return lun_info;
6423 }
6424
6425 /**
6426  * lpfc_delete_device_data - frees a device data structure for OAS
6427  * @phba: Pointer to host bus adapter structure.
6428  * @lun_info: Pointer to device data structure to free.
6429  *
6430  * This routine frees the previously allocated device data structure passed.
6431  *
6432  **/
6433 void
6434 lpfc_delete_device_data(struct lpfc_hba *phba,
6435                         struct lpfc_device_data *lun_info)
6436 {
6437
6438         if (unlikely(!phba) || !lun_info  ||
6439             !(phba->cfg_fof))
6440                 return;
6441
6442         if (!list_empty(&lun_info->listentry))
6443                 list_del(&lun_info->listentry);
6444         mempool_free(lun_info, phba->device_data_mem_pool);
6445         return;
6446 }
6447
6448 /**
6449  * __lpfc_get_device_data - returns the device data for the specified lun
6450  * @phba: Pointer to host bus adapter structure.
6451  * @list: Point to list to search.
6452  * @vport_wwpn: Pointer to vport's wwpn information
6453  * @target_wwpn: Pointer to target's wwpn information
6454  * @lun: Lun on target
6455  *
6456  * This routine searches the list passed for the specified lun's device data.
6457  * This function does not hold locks, it is the responsibility of the caller
6458  * to ensure the proper lock is held before calling the function.
6459  *
6460  * Return codes:
6461  *   NULL - Error
6462  *   Pointer to lpfc_device_data - Success
6463  **/
6464 struct lpfc_device_data*
6465 __lpfc_get_device_data(struct lpfc_hba *phba, struct list_head *list,
6466                        struct lpfc_name *vport_wwpn,
6467                        struct lpfc_name *target_wwpn, uint64_t lun)
6468 {
6469
6470         struct lpfc_device_data *lun_info;
6471
6472         if (unlikely(!phba) || !list || !vport_wwpn || !target_wwpn ||
6473             !phba->cfg_fof)
6474                 return NULL;
6475
6476         /* Check to see if the lun is already enabled for OAS. */
6477
6478         list_for_each_entry(lun_info, list, listentry) {
6479                 if ((memcmp(&lun_info->device_id.vport_wwpn, vport_wwpn,
6480                             sizeof(struct lpfc_name)) == 0) &&
6481                     (memcmp(&lun_info->device_id.target_wwpn, target_wwpn,
6482                             sizeof(struct lpfc_name)) == 0) &&
6483                     (lun_info->device_id.lun == lun))
6484                         return lun_info;
6485         }
6486
6487         return NULL;
6488 }
6489
6490 /**
6491  * lpfc_find_next_oas_lun - searches for the next oas lun
6492  * @phba: Pointer to host bus adapter structure.
6493  * @vport_wwpn: Pointer to vport's wwpn information
6494  * @target_wwpn: Pointer to target's wwpn information
6495  * @starting_lun: Pointer to the lun to start searching for
6496  * @found_vport_wwpn: Pointer to the found lun's vport wwpn information
6497  * @found_target_wwpn: Pointer to the found lun's target wwpn information
6498  * @found_lun: Pointer to the found lun.
6499  * @found_lun_status: Pointer to status of the found lun.
6500  * @found_lun_pri: Pointer to priority of the found lun.
6501  *
6502  * This routine searches the luns list for the specified lun
6503  * or the first lun for the vport/target.  If the vport wwpn contains
6504  * a zero value then a specific vport is not specified. In this case
6505  * any vport which contains the lun will be considered a match.  If the
6506  * target wwpn contains a zero value then a specific target is not specified.
6507  * In this case any target which contains the lun will be considered a
6508  * match.  If the lun is found, the lun, vport wwpn, target wwpn and lun status
6509  * are returned.  The function will also return the next lun if available.
6510  * If the next lun is not found, starting_lun parameter will be set to
6511  * NO_MORE_OAS_LUN.
6512  *
6513  * Return codes:
6514  *   non-0 - Error
6515  *   0 - Success
6516  **/
6517 bool
6518 lpfc_find_next_oas_lun(struct lpfc_hba *phba, struct lpfc_name *vport_wwpn,
6519                        struct lpfc_name *target_wwpn, uint64_t *starting_lun,
6520                        struct lpfc_name *found_vport_wwpn,
6521                        struct lpfc_name *found_target_wwpn,
6522                        uint64_t *found_lun,
6523                        uint32_t *found_lun_status,
6524                        uint32_t *found_lun_pri)
6525 {
6526
6527         unsigned long flags;
6528         struct lpfc_device_data *lun_info;
6529         struct lpfc_device_id *device_id;
6530         uint64_t lun;
6531         bool found = false;
6532
6533         if (unlikely(!phba) || !vport_wwpn || !target_wwpn ||
6534             !starting_lun || !found_vport_wwpn ||
6535             !found_target_wwpn || !found_lun || !found_lun_status ||
6536             (*starting_lun == NO_MORE_OAS_LUN) ||
6537             !phba->cfg_fof)
6538                 return false;
6539
6540         lun = *starting_lun;
6541         *found_lun = NO_MORE_OAS_LUN;
6542         *starting_lun = NO_MORE_OAS_LUN;
6543
6544         /* Search for lun or the lun closet in value */
6545
6546         spin_lock_irqsave(&phba->devicelock, flags);
6547         list_for_each_entry(lun_info, &phba->luns, listentry) {
6548                 if (((wwn_to_u64(vport_wwpn->u.wwn) == 0) ||
6549                      (memcmp(&lun_info->device_id.vport_wwpn, vport_wwpn,
6550                             sizeof(struct lpfc_name)) == 0)) &&
6551                     ((wwn_to_u64(target_wwpn->u.wwn) == 0) ||
6552                      (memcmp(&lun_info->device_id.target_wwpn, target_wwpn,
6553                             sizeof(struct lpfc_name)) == 0)) &&
6554                     (lun_info->oas_enabled)) {
6555                         device_id = &lun_info->device_id;
6556                         if ((!found) &&
6557                             ((lun == FIND_FIRST_OAS_LUN) ||
6558                              (device_id->lun == lun))) {
6559                                 *found_lun = device_id->lun;
6560                                 memcpy(found_vport_wwpn,
6561                                        &device_id->vport_wwpn,
6562                                        sizeof(struct lpfc_name));
6563                                 memcpy(found_target_wwpn,
6564                                        &device_id->target_wwpn,
6565                                        sizeof(struct lpfc_name));
6566                                 if (lun_info->available)
6567                                         *found_lun_status =
6568                                                 OAS_LUN_STATUS_EXISTS;
6569                                 else
6570                                         *found_lun_status = 0;
6571                                 *found_lun_pri = lun_info->priority;
6572                                 if (phba->cfg_oas_flags & OAS_FIND_ANY_VPORT)
6573                                         memset(vport_wwpn, 0x0,
6574                                                sizeof(struct lpfc_name));
6575                                 if (phba->cfg_oas_flags & OAS_FIND_ANY_TARGET)
6576                                         memset(target_wwpn, 0x0,
6577                                                sizeof(struct lpfc_name));
6578                                 found = true;
6579                         } else if (found) {
6580                                 *starting_lun = device_id->lun;
6581                                 memcpy(vport_wwpn, &device_id->vport_wwpn,
6582                                        sizeof(struct lpfc_name));
6583                                 memcpy(target_wwpn, &device_id->target_wwpn,
6584                                        sizeof(struct lpfc_name));
6585                                 break;
6586                         }
6587                 }
6588         }
6589         spin_unlock_irqrestore(&phba->devicelock, flags);
6590         return found;
6591 }
6592
6593 /**
6594  * lpfc_enable_oas_lun - enables a lun for OAS operations
6595  * @phba: Pointer to host bus adapter structure.
6596  * @vport_wwpn: Pointer to vport's wwpn information
6597  * @target_wwpn: Pointer to target's wwpn information
6598  * @lun: Lun
6599  * @pri: Priority
6600  *
6601  * This routine enables a lun for oas operations.  The routines does so by
6602  * doing the following :
6603  *
6604  *   1) Checks to see if the device data for the lun has been created.
6605  *   2) If found, sets the OAS enabled flag if not set and returns.
6606  *   3) Otherwise, creates a device data structure.
6607  *   4) If successfully created, indicates the device data is for an OAS lun,
6608  *   indicates the lun is not available and add to the list of luns.
6609  *
6610  * Return codes:
6611  *   false - Error
6612  *   true - Success
6613  **/
6614 bool
6615 lpfc_enable_oas_lun(struct lpfc_hba *phba, struct lpfc_name *vport_wwpn,
6616                     struct lpfc_name *target_wwpn, uint64_t lun, uint8_t pri)
6617 {
6618
6619         struct lpfc_device_data *lun_info;
6620         unsigned long flags;
6621
6622         if (unlikely(!phba) || !vport_wwpn || !target_wwpn ||
6623             !phba->cfg_fof)
6624                 return false;
6625
6626         spin_lock_irqsave(&phba->devicelock, flags);
6627
6628         /* Check to see if the device data for the lun has been created */
6629         lun_info = __lpfc_get_device_data(phba, &phba->luns, vport_wwpn,
6630                                           target_wwpn, lun);
6631         if (lun_info) {
6632                 if (!lun_info->oas_enabled)
6633                         lun_info->oas_enabled = true;
6634                 lun_info->priority = pri;
6635                 spin_unlock_irqrestore(&phba->devicelock, flags);
6636                 return true;
6637         }
6638
6639         /* Create an lun info structure and add to list of luns */
6640         lun_info = lpfc_create_device_data(phba, vport_wwpn, target_wwpn, lun,
6641                                            pri, true);
6642         if (lun_info) {
6643                 lun_info->oas_enabled = true;
6644                 lun_info->priority = pri;
6645                 lun_info->available = false;
6646                 list_add_tail(&lun_info->listentry, &phba->luns);
6647                 spin_unlock_irqrestore(&phba->devicelock, flags);
6648                 return true;
6649         }
6650         spin_unlock_irqrestore(&phba->devicelock, flags);
6651         return false;
6652 }
6653
6654 /**
6655  * lpfc_disable_oas_lun - disables a lun for OAS operations
6656  * @phba: Pointer to host bus adapter structure.
6657  * @vport_wwpn: Pointer to vport's wwpn information
6658  * @target_wwpn: Pointer to target's wwpn information
6659  * @lun: Lun
6660  * @pri: Priority
6661  *
6662  * This routine disables a lun for oas operations.  The routines does so by
6663  * doing the following :
6664  *
6665  *   1) Checks to see if the device data for the lun is created.
6666  *   2) If present, clears the flag indicating this lun is for OAS.
6667  *   3) If the lun is not available by the system, the device data is
6668  *   freed.
6669  *
6670  * Return codes:
6671  *   false - Error
6672  *   true - Success
6673  **/
6674 bool
6675 lpfc_disable_oas_lun(struct lpfc_hba *phba, struct lpfc_name *vport_wwpn,
6676                      struct lpfc_name *target_wwpn, uint64_t lun, uint8_t pri)
6677 {
6678
6679         struct lpfc_device_data *lun_info;
6680         unsigned long flags;
6681
6682         if (unlikely(!phba) || !vport_wwpn || !target_wwpn ||
6683             !phba->cfg_fof)
6684                 return false;
6685
6686         spin_lock_irqsave(&phba->devicelock, flags);
6687
6688         /* Check to see if the lun is available. */
6689         lun_info = __lpfc_get_device_data(phba,
6690                                           &phba->luns, vport_wwpn,
6691                                           target_wwpn, lun);
6692         if (lun_info) {
6693                 lun_info->oas_enabled = false;
6694                 lun_info->priority = pri;
6695                 if (!lun_info->available)
6696                         lpfc_delete_device_data(phba, lun_info);
6697                 spin_unlock_irqrestore(&phba->devicelock, flags);
6698                 return true;
6699         }
6700
6701         spin_unlock_irqrestore(&phba->devicelock, flags);
6702         return false;
6703 }
6704
6705 static int
6706 lpfc_no_command(struct Scsi_Host *shost, struct scsi_cmnd *cmnd)
6707 {
6708         return SCSI_MLQUEUE_HOST_BUSY;
6709 }
6710
6711 static int
6712 lpfc_no_slave(struct scsi_device *sdev)
6713 {
6714         return -ENODEV;
6715 }
6716
6717 struct scsi_host_template lpfc_template_nvme = {
6718         .module                 = THIS_MODULE,
6719         .name                   = LPFC_DRIVER_NAME,
6720         .proc_name              = LPFC_DRIVER_NAME,
6721         .info                   = lpfc_info,
6722         .queuecommand           = lpfc_no_command,
6723         .slave_alloc            = lpfc_no_slave,
6724         .slave_configure        = lpfc_no_slave,
6725         .scan_finished          = lpfc_scan_finished,
6726         .this_id                = -1,
6727         .sg_tablesize           = 1,
6728         .cmd_per_lun            = 1,
6729         .shost_groups           = lpfc_hba_groups,
6730         .max_sectors            = 0xFFFFFFFF,
6731         .vendor_id              = LPFC_NL_VENDOR_ID,
6732         .track_queue_depth      = 0,
6733 };
6734
6735 struct scsi_host_template lpfc_template = {
6736         .module                 = THIS_MODULE,
6737         .name                   = LPFC_DRIVER_NAME,
6738         .proc_name              = LPFC_DRIVER_NAME,
6739         .info                   = lpfc_info,
6740         .queuecommand           = lpfc_queuecommand,
6741         .eh_timed_out           = fc_eh_timed_out,
6742         .eh_should_retry_cmd    = fc_eh_should_retry_cmd,
6743         .eh_abort_handler       = lpfc_abort_handler,
6744         .eh_device_reset_handler = lpfc_device_reset_handler,
6745         .eh_target_reset_handler = lpfc_target_reset_handler,
6746         .eh_host_reset_handler  = lpfc_host_reset_handler,
6747         .slave_alloc            = lpfc_slave_alloc,
6748         .slave_configure        = lpfc_slave_configure,
6749         .slave_destroy          = lpfc_slave_destroy,
6750         .scan_finished          = lpfc_scan_finished,
6751         .this_id                = -1,
6752         .sg_tablesize           = LPFC_DEFAULT_SG_SEG_CNT,
6753         .cmd_per_lun            = LPFC_CMD_PER_LUN,
6754         .shost_groups           = lpfc_hba_groups,
6755         .max_sectors            = 0xFFFFFFFF,
6756         .vendor_id              = LPFC_NL_VENDOR_ID,
6757         .change_queue_depth     = scsi_change_queue_depth,
6758         .track_queue_depth      = 1,
6759 };
6760
6761 struct scsi_host_template lpfc_vport_template = {
6762         .module                 = THIS_MODULE,
6763         .name                   = LPFC_DRIVER_NAME,
6764         .proc_name              = LPFC_DRIVER_NAME,
6765         .info                   = lpfc_info,
6766         .queuecommand           = lpfc_queuecommand,
6767         .eh_timed_out           = fc_eh_timed_out,
6768         .eh_should_retry_cmd    = fc_eh_should_retry_cmd,
6769         .eh_abort_handler       = lpfc_abort_handler,
6770         .eh_device_reset_handler = lpfc_device_reset_handler,
6771         .eh_target_reset_handler = lpfc_target_reset_handler,
6772         .eh_bus_reset_handler   = NULL,
6773         .eh_host_reset_handler  = NULL,
6774         .slave_alloc            = lpfc_slave_alloc,
6775         .slave_configure        = lpfc_slave_configure,
6776         .slave_destroy          = lpfc_slave_destroy,
6777         .scan_finished          = lpfc_scan_finished,
6778         .this_id                = -1,
6779         .sg_tablesize           = LPFC_DEFAULT_SG_SEG_CNT,
6780         .cmd_per_lun            = LPFC_CMD_PER_LUN,
6781         .shost_groups           = lpfc_vport_groups,
6782         .max_sectors            = 0xFFFFFFFF,
6783         .vendor_id              = 0,
6784         .change_queue_depth     = scsi_change_queue_depth,
6785         .track_queue_depth      = 1,
6786 };