drivers/scsi/be2iscsi/be_main.c

   1 /**
   2  * Copyright (C) 2005 - 2016 Broadcom
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Written by: Jayamohan Kallickal (jayamohan.kallickal@broadcom.com)
  11  *
  12  * Contact Information:
  13  * linux-drivers@broadcom.com
  14  *
  15  * Emulex
  16  * 3333 Susan Street
  17  * Costa Mesa, CA 92626
  18  */
  19
  20 #include <linux/reboot.h>
  21 #include <linux/delay.h>
  22 #include <linux/slab.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/blkdev.h>
  25 #include <linux/pci.h>
  26 #include <linux/string.h>
  27 #include <linux/kernel.h>
  28 #include <linux/semaphore.h>
  29 #include <linux/iscsi_boot_sysfs.h>
  30 #include <linux/module.h>
  31 #include <linux/bsg-lib.h>
  32 #include <linux/irq_poll.h>
  33
  34 #include <scsi/libiscsi.h>
  35 #include <scsi/scsi_bsg_iscsi.h>
  36 #include <scsi/scsi_netlink.h>
  37 #include <scsi/scsi_transport_iscsi.h>
  38 #include <scsi/scsi_transport.h>
  39 #include <scsi/scsi_cmnd.h>
  40 #include <scsi/scsi_device.h>
  41 #include <scsi/scsi_host.h>
  42 #include <scsi/scsi.h>
  43 #include "be_main.h"
  44 #include "be_iscsi.h"
  45 #include "be_mgmt.h"
  46 #include "be_cmds.h"
  47
  48 static unsigned int be_iopoll_budget = 10;
  49 static unsigned int be_max_phys_size = 64;
  50 static unsigned int enable_msix = 1;
  51
  52 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
  53 MODULE_VERSION(BUILD_STR);
  54 MODULE_AUTHOR("Emulex Corporation");
  55 MODULE_LICENSE("GPL");
  56 module_param(be_iopoll_budget, int, 0);
  57 module_param(enable_msix, int, 0);
  58 module_param(be_max_phys_size, uint, S_IRUGO);
  59 MODULE_PARM_DESC(be_max_phys_size,
  60                 "Maximum Size (In Kilobytes) of physically contiguous "
  61                 "memory that can be allocated. Range is 16 - 128");
  62
  63 #define beiscsi_disp_param(_name)\
  64 static ssize_t  \
  65 beiscsi_##_name##_disp(struct device *dev,\
  66                         struct device_attribute *attrib, char *buf)     \
  67 {       \
  68         struct Scsi_Host *shost = class_to_shost(dev);\
  69         struct beiscsi_hba *phba = iscsi_host_priv(shost); \
  70         uint32_t param_val = 0; \
  71         param_val = phba->attr_##_name;\
  72         return snprintf(buf, PAGE_SIZE, "%d\n",\
  73                         phba->attr_##_name);\
  74 }
  75
  76 #define beiscsi_change_param(_name, _minval, _maxval, _defaval)\
  77 static int \
  78 beiscsi_##_name##_change(struct beiscsi_hba *phba, uint32_t val)\
  79 {\
  80         if (val >= _minval && val <= _maxval) {\
  81                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
  82                             "BA_%d : beiscsi_"#_name" updated "\
  83                             "from 0x%x ==> 0x%x\n",\
  84                             phba->attr_##_name, val); \
  85                 phba->attr_##_name = val;\
  86                 return 0;\
  87         } \
  88         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, \
  89                     "BA_%d beiscsi_"#_name" attribute "\
  90                     "cannot be updated to 0x%x, "\
  91                     "range allowed is ["#_minval" - "#_maxval"]\n", val);\
  92                 return -EINVAL;\
  93 }
  94
  95 #define beiscsi_store_param(_name)  \
  96 static ssize_t \
  97 beiscsi_##_name##_store(struct device *dev,\
  98                          struct device_attribute *attr, const char *buf,\
  99                          size_t count) \
 100 { \
 101         struct Scsi_Host  *shost = class_to_shost(dev);\
 102         struct beiscsi_hba *phba = iscsi_host_priv(shost);\
 103         uint32_t param_val = 0;\
 104         if (!isdigit(buf[0]))\
 105                 return -EINVAL;\
 106         if (sscanf(buf, "%i", &param_val) != 1)\
 107                 return -EINVAL;\
 108         if (beiscsi_##_name##_change(phba, param_val) == 0) \
 109                 return strlen(buf);\
 110         else \
 111                 return -EINVAL;\
 112 }
 113
 114 #define beiscsi_init_param(_name, _minval, _maxval, _defval) \
 115 static int \
 116 beiscsi_##_name##_init(struct beiscsi_hba *phba, uint32_t val) \
 117 { \
 118         if (val >= _minval && val <= _maxval) {\
 119                 phba->attr_##_name = val;\
 120                 return 0;\
 121         } \
 122         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
 123                     "BA_%d beiscsi_"#_name" attribute " \
 124                     "cannot be updated to 0x%x, "\
 125                     "range allowed is ["#_minval" - "#_maxval"]\n", val);\
 126         phba->attr_##_name = _defval;\
 127         return -EINVAL;\
 128 }
 129
 130 #define BEISCSI_RW_ATTR(_name, _minval, _maxval, _defval, _descp) \
 131 static uint beiscsi_##_name = _defval;\
 132 module_param(beiscsi_##_name, uint, S_IRUGO);\
 133 MODULE_PARM_DESC(beiscsi_##_name, _descp);\
 134 beiscsi_disp_param(_name)\
 135 beiscsi_change_param(_name, _minval, _maxval, _defval)\
 136 beiscsi_store_param(_name)\
 137 beiscsi_init_param(_name, _minval, _maxval, _defval)\
 138 DEVICE_ATTR(beiscsi_##_name, S_IRUGO | S_IWUSR,\
 139               beiscsi_##_name##_disp, beiscsi_##_name##_store)
 140
 141 /*
 142  * When new log level added update the
 143  * the MAX allowed value for log_enable
 144  */
 145 BEISCSI_RW_ATTR(log_enable, 0x00,
 146                 0xFF, 0x00, "Enable logging Bit Mask\n"
 147                 "\t\t\t\tInitialization Events  : 0x01\n"
 148                 "\t\t\t\tMailbox Events         : 0x02\n"
 149                 "\t\t\t\tMiscellaneous Events   : 0x04\n"
 150                 "\t\t\t\tError Handling         : 0x08\n"
 151                 "\t\t\t\tIO Path Events         : 0x10\n"
 152                 "\t\t\t\tConfiguration Path     : 0x20\n"
 153                 "\t\t\t\tiSCSI Protocol         : 0x40\n");
 154
 155 DEVICE_ATTR(beiscsi_drvr_ver, S_IRUGO, beiscsi_drvr_ver_disp, NULL);
 156 DEVICE_ATTR(beiscsi_adapter_family, S_IRUGO, beiscsi_adap_family_disp, NULL);
 157 DEVICE_ATTR(beiscsi_fw_ver, S_IRUGO, beiscsi_fw_ver_disp, NULL);
 158 DEVICE_ATTR(beiscsi_phys_port, S_IRUGO, beiscsi_phys_port_disp, NULL);
 159 DEVICE_ATTR(beiscsi_active_session_count, S_IRUGO,
 160              beiscsi_active_session_disp, NULL);
 161 DEVICE_ATTR(beiscsi_free_session_count, S_IRUGO,
 162              beiscsi_free_session_disp, NULL);
 163 struct device_attribute *beiscsi_attrs[] = {
 164         &dev_attr_beiscsi_log_enable,
 165         &dev_attr_beiscsi_drvr_ver,
 166         &dev_attr_beiscsi_adapter_family,
 167         &dev_attr_beiscsi_fw_ver,
 168         &dev_attr_beiscsi_active_session_count,
 169         &dev_attr_beiscsi_free_session_count,
 170         &dev_attr_beiscsi_phys_port,
 171         NULL,
 172 };
 173
 174 static char const *cqe_desc[] = {
 175         "RESERVED_DESC",
 176         "SOL_CMD_COMPLETE",
 177         "SOL_CMD_KILLED_DATA_DIGEST_ERR",
 178         "CXN_KILLED_PDU_SIZE_EXCEEDS_DSL",
 179         "CXN_KILLED_BURST_LEN_MISMATCH",
 180         "CXN_KILLED_AHS_RCVD",
 181         "CXN_KILLED_HDR_DIGEST_ERR",
 182         "CXN_KILLED_UNKNOWN_HDR",
 183         "CXN_KILLED_STALE_ITT_TTT_RCVD",
 184         "CXN_KILLED_INVALID_ITT_TTT_RCVD",
 185         "CXN_KILLED_RST_RCVD",
 186         "CXN_KILLED_TIMED_OUT",
 187         "CXN_KILLED_RST_SENT",
 188         "CXN_KILLED_FIN_RCVD",
 189         "CXN_KILLED_BAD_UNSOL_PDU_RCVD",
 190         "CXN_KILLED_BAD_WRB_INDEX_ERROR",
 191         "CXN_KILLED_OVER_RUN_RESIDUAL",
 192         "CXN_KILLED_UNDER_RUN_RESIDUAL",
 193         "CMD_KILLED_INVALID_STATSN_RCVD",
 194         "CMD_KILLED_INVALID_R2T_RCVD",
 195         "CMD_CXN_KILLED_LUN_INVALID",
 196         "CMD_CXN_KILLED_ICD_INVALID",
 197         "CMD_CXN_KILLED_ITT_INVALID",
 198         "CMD_CXN_KILLED_SEQ_OUTOFORDER",
 199         "CMD_CXN_KILLED_INVALID_DATASN_RCVD",
 200         "CXN_INVALIDATE_NOTIFY",
 201         "CXN_INVALIDATE_INDEX_NOTIFY",
 202         "CMD_INVALIDATED_NOTIFY",
 203         "UNSOL_HDR_NOTIFY",
 204         "UNSOL_DATA_NOTIFY",
 205         "UNSOL_DATA_DIGEST_ERROR_NOTIFY",
 206         "DRIVERMSG_NOTIFY",
 207         "CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN",
 208         "SOL_CMD_KILLED_DIF_ERR",
 209         "CXN_KILLED_SYN_RCVD",
 210         "CXN_KILLED_IMM_DATA_RCVD"
 211 };
 212
 213 static int beiscsi_slave_configure(struct scsi_device *sdev)
 214 {
 215         blk_queue_max_segment_size(sdev->request_queue, 65536);
 216         return 0;
 217 }
 218
 219 static int beiscsi_eh_abort(struct scsi_cmnd *sc)
 220 {
 221         struct iscsi_cls_session *cls_session;
 222         struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
 223         struct beiscsi_io_task *aborted_io_task;
 224         struct iscsi_conn *conn;
 225         struct beiscsi_conn *beiscsi_conn;
 226         struct beiscsi_hba *phba;
 227         struct iscsi_session *session;
 228         struct invalidate_command_table *inv_tbl;
 229         struct be_dma_mem nonemb_cmd;
 230         unsigned int cid, tag, num_invalidate;
 231         int rc;
 232
 233         cls_session = starget_to_session(scsi_target(sc->device));
 234         session = cls_session->dd_data;
 235
 236         spin_lock_bh(&session->frwd_lock);
 237         if (!aborted_task || !aborted_task->sc) {
 238                 /* we raced */
 239                 spin_unlock_bh(&session->frwd_lock);
 240                 return SUCCESS;
 241         }
 242
 243         aborted_io_task = aborted_task->dd_data;
 244         if (!aborted_io_task->scsi_cmnd) {
 245                 /* raced or invalid command */
 246                 spin_unlock_bh(&session->frwd_lock);
 247                 return SUCCESS;
 248         }
 249         spin_unlock_bh(&session->frwd_lock);
 250         /* Invalidate WRB Posted for this Task */
 251         AMAP_SET_BITS(struct amap_iscsi_wrb, invld,
 252                       aborted_io_task->pwrb_handle->pwrb,
 253                       1);
 254
 255         conn = aborted_task->conn;
 256         beiscsi_conn = conn->dd_data;
 257         phba = beiscsi_conn->phba;
 258
 259         /* invalidate iocb */
 260         cid = beiscsi_conn->beiscsi_conn_cid;
 261         inv_tbl = phba->inv_tbl;
 262         memset(inv_tbl, 0x0, sizeof(*inv_tbl));
 263         inv_tbl->cid = cid;
 264         inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
 265         num_invalidate = 1;
 266         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
 267                                 sizeof(struct invalidate_commands_params_in),
 268                                 &nonemb_cmd.dma);
 269         if (nonemb_cmd.va == NULL) {
 270                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
 271                             "BM_%d : Failed to allocate memory for"
 272                             "mgmt_invalidate_icds\n");
 273                 return FAILED;
 274         }
 275         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
 276
 277         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 278                                    cid, &nonemb_cmd);
 279         if (!tag) {
 280                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
 281                             "BM_%d : mgmt_invalidate_icds could not be"
 282                             "submitted\n");
 283                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 284                                     nonemb_cmd.va, nonemb_cmd.dma);
 285
 286                 return FAILED;
 287         }
 288
 289         rc = beiscsi_mccq_compl_wait(phba, tag, NULL, &nonemb_cmd);
 290         if (rc != -EBUSY)
 291                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 292                                     nonemb_cmd.va, nonemb_cmd.dma);
 293
 294         return iscsi_eh_abort(sc);
 295 }
 296
 297 static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
 298 {
 299         struct iscsi_task *abrt_task;
 300         struct beiscsi_io_task *abrt_io_task;
 301         struct iscsi_conn *conn;
 302         struct beiscsi_conn *beiscsi_conn;
 303         struct beiscsi_hba *phba;
 304         struct iscsi_session *session;
 305         struct iscsi_cls_session *cls_session;
 306         struct invalidate_command_table *inv_tbl;
 307         struct be_dma_mem nonemb_cmd;
 308         unsigned int cid, tag, i, num_invalidate;
 309         int rc;
 310
 311         /* invalidate iocbs */
 312         cls_session = starget_to_session(scsi_target(sc->device));
 313         session = cls_session->dd_data;
 314         spin_lock_bh(&session->frwd_lock);
 315         if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) {
 316                 spin_unlock_bh(&session->frwd_lock);
 317                 return FAILED;
 318         }
 319         conn = session->leadconn;
 320         beiscsi_conn = conn->dd_data;
 321         phba = beiscsi_conn->phba;
 322         cid = beiscsi_conn->beiscsi_conn_cid;
 323         inv_tbl = phba->inv_tbl;
 324         memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
 325         num_invalidate = 0;
 326         for (i = 0; i < conn->session->cmds_max; i++) {
 327                 abrt_task = conn->session->cmds[i];
 328                 abrt_io_task = abrt_task->dd_data;
 329                 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
 330                         continue;
 331
 332                 if (sc->device->lun != abrt_task->sc->device->lun)
 333                         continue;
 334
 335                 /* Invalidate WRB Posted for this Task */
 336                 AMAP_SET_BITS(struct amap_iscsi_wrb, invld,
 337                               abrt_io_task->pwrb_handle->pwrb,
 338                               1);
 339
 340                 inv_tbl->cid = cid;
 341                 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
 342                 num_invalidate++;
 343                 inv_tbl++;
 344         }
 345         spin_unlock_bh(&session->frwd_lock);
 346         inv_tbl = phba->inv_tbl;
 347
 348         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
 349                                 sizeof(struct invalidate_commands_params_in),
 350                                 &nonemb_cmd.dma);
 351         if (nonemb_cmd.va == NULL) {
 352                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
 353                             "BM_%d : Failed to allocate memory for"
 354                             "mgmt_invalidate_icds\n");
 355                 return FAILED;
 356         }
 357         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
 358         memset(nonemb_cmd.va, 0, nonemb_cmd.size);
 359         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 360                                    cid, &nonemb_cmd);
 361         if (!tag) {
 362                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
 363                             "BM_%d : mgmt_invalidate_icds could not be"
 364                             " submitted\n");
 365                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 366                                     nonemb_cmd.va, nonemb_cmd.dma);
 367                 return FAILED;
 368         }
 369
 370         rc = beiscsi_mccq_compl_wait(phba, tag, NULL, &nonemb_cmd);
 371         if (rc != -EBUSY)
 372                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 373                                     nonemb_cmd.va, nonemb_cmd.dma);
 374         return iscsi_eh_device_reset(sc);
 375 }
 376
 377 /*------------------- PCI Driver operations and data ----------------- */
 378 static const struct pci_device_id beiscsi_pci_id_table[] = {
 379         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
 380         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
 381         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
 382         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
 383         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
 384         { PCI_DEVICE(ELX_VENDOR_ID, OC_SKH_ID1) },
 385         { 0 }
 386 };
 387 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
 388
 389
 390 static struct scsi_host_template beiscsi_sht = {
 391         .module = THIS_MODULE,
 392         .name = "Emulex 10Gbe open-iscsi Initiator Driver",
 393         .proc_name = DRV_NAME,
 394         .queuecommand = iscsi_queuecommand,
 395         .change_queue_depth = scsi_change_queue_depth,
 396         .slave_configure = beiscsi_slave_configure,
 397         .target_alloc = iscsi_target_alloc,
 398         .eh_abort_handler = beiscsi_eh_abort,
 399         .eh_device_reset_handler = beiscsi_eh_device_reset,
 400         .eh_target_reset_handler = iscsi_eh_session_reset,
 401         .shost_attrs = beiscsi_attrs,
 402         .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
 403         .can_queue = BE2_IO_DEPTH,
 404         .this_id = -1,
 405         .max_sectors = BEISCSI_MAX_SECTORS,
 406         .cmd_per_lun = BEISCSI_CMD_PER_LUN,
 407         .use_clustering = ENABLE_CLUSTERING,
 408         .vendor_id = SCSI_NL_VID_TYPE_PCI | BE_VENDOR_ID,
 409         .track_queue_depth = 1,
 410 };
 411
 412 static struct scsi_transport_template *beiscsi_scsi_transport;
 413
 414 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
 415 {
 416         struct beiscsi_hba *phba;
 417         struct Scsi_Host *shost;
 418
 419         shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
 420         if (!shost) {
 421                 dev_err(&pcidev->dev,
 422                         "beiscsi_hba_alloc - iscsi_host_alloc failed\n");
 423                 return NULL;
 424         }
 425         shost->max_id = BE2_MAX_SESSIONS;
 426         shost->max_channel = 0;
 427         shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
 428         shost->max_lun = BEISCSI_NUM_MAX_LUN;
 429         shost->transportt = beiscsi_scsi_transport;
 430         phba = iscsi_host_priv(shost);
 431         memset(phba, 0, sizeof(*phba));
 432         phba->shost = shost;
 433         phba->pcidev = pci_dev_get(pcidev);
 434         pci_set_drvdata(pcidev, phba);
 435         phba->interface_handle = 0xFFFFFFFF;
 436
 437         return phba;
 438 }
 439
 440 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
 441 {
 442         if (phba->csr_va) {
 443                 iounmap(phba->csr_va);
 444                 phba->csr_va = NULL;
 445         }
 446         if (phba->db_va) {
 447                 iounmap(phba->db_va);
 448                 phba->db_va = NULL;
 449         }
 450         if (phba->pci_va) {
 451                 iounmap(phba->pci_va);
 452                 phba->pci_va = NULL;
 453         }
 454 }
 455
 456 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
 457                                 struct pci_dev *pcidev)
 458 {
 459         u8 __iomem *addr;
 460         int pcicfg_reg;
 461
 462         addr = ioremap_nocache(pci_resource_start(pcidev, 2),
 463                                pci_resource_len(pcidev, 2));
 464         if (addr == NULL)
 465                 return -ENOMEM;
 466         phba->ctrl.csr = addr;
 467         phba->csr_va = addr;
 468         phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
 469
 470         addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
 471         if (addr == NULL)
 472                 goto pci_map_err;
 473         phba->ctrl.db = addr;
 474         phba->db_va = addr;
 475         phba->db_pa.u.a64.address =  pci_resource_start(pcidev, 4);
 476
 477         if (phba->generation == BE_GEN2)
 478                 pcicfg_reg = 1;
 479         else
 480                 pcicfg_reg = 0;
 481
 482         addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
 483                                pci_resource_len(pcidev, pcicfg_reg));
 484
 485         if (addr == NULL)
 486                 goto pci_map_err;
 487         phba->ctrl.pcicfg = addr;
 488         phba->pci_va = addr;
 489         phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
 490         return 0;
 491
 492 pci_map_err:
 493         beiscsi_unmap_pci_function(phba);
 494         return -ENOMEM;
 495 }
 496
 497 static int beiscsi_enable_pci(struct pci_dev *pcidev)
 498 {
 499         int ret;
 500
 501         ret = pci_enable_device(pcidev);
 502         if (ret) {
 503                 dev_err(&pcidev->dev,
 504                         "beiscsi_enable_pci - enable device failed\n");
 505                 return ret;
 506         }
 507
 508         ret = pci_request_regions(pcidev, DRV_NAME);
 509         if (ret) {
 510                 dev_err(&pcidev->dev,
 511                                 "beiscsi_enable_pci - request region failed\n");
 512                 goto pci_dev_disable;
 513         }
 514
 515         pci_set_master(pcidev);
 516         ret = pci_set_dma_mask(pcidev, DMA_BIT_MASK(64));
 517         if (ret) {
 518                 ret = pci_set_dma_mask(pcidev, DMA_BIT_MASK(32));
 519                 if (ret) {
 520                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
 521                         goto pci_region_release;
 522                 } else {
 523                         ret = pci_set_consistent_dma_mask(pcidev,
 524                                                           DMA_BIT_MASK(32));
 525                 }
 526         } else {
 527                 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64));
 528                 if (ret) {
 529                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
 530                         goto pci_region_release;
 531                 }
 532         }
 533         return 0;
 534
 535 pci_region_release:
 536         pci_release_regions(pcidev);
 537 pci_dev_disable:
 538         pci_disable_device(pcidev);
 539
 540         return ret;
 541 }
 542
 543 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
 544 {
 545         struct be_ctrl_info *ctrl = &phba->ctrl;
 546         struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
 547         struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
 548         int status = 0;
 549
 550         ctrl->pdev = pdev;
 551         status = beiscsi_map_pci_bars(phba, pdev);
 552         if (status)
 553                 return status;
 554         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
 555         mbox_mem_alloc->va = pci_alloc_consistent(pdev,
 556                                                   mbox_mem_alloc->size,
 557                                                   &mbox_mem_alloc->dma);
 558         if (!mbox_mem_alloc->va) {
 559                 beiscsi_unmap_pci_function(phba);
 560                 return -ENOMEM;
 561         }
 562
 563         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
 564         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
 565         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
 566         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
 567         mutex_init(&ctrl->mbox_lock);
 568         spin_lock_init(&phba->ctrl.mcc_lock);
 569
 570         return status;
 571 }
 572
 573 /**
 574  * beiscsi_get_params()- Set the config paramters
 575  * @phba: ptr  device priv structure
 576  **/
 577 static void beiscsi_get_params(struct beiscsi_hba *phba)
 578 {
 579         uint32_t total_cid_count = 0;
 580         uint32_t total_icd_count = 0;
 581         uint8_t ulp_num = 0;
 582
 583         total_cid_count = BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP0) +
 584                           BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP1);
 585
 586         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
 587                 uint32_t align_mask = 0;
 588                 uint32_t icd_post_per_page = 0;
 589                 uint32_t icd_count_unavailable = 0;
 590                 uint32_t icd_start = 0, icd_count = 0;
 591                 uint32_t icd_start_align = 0, icd_count_align = 0;
 592
 593                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
 594                         icd_start = phba->fw_config.iscsi_icd_start[ulp_num];
 595                         icd_count = phba->fw_config.iscsi_icd_count[ulp_num];
 596
 597                         /* Get ICD count that can be posted on each page */
 598                         icd_post_per_page = (PAGE_SIZE / (BE2_SGE *
 599                                              sizeof(struct iscsi_sge)));
 600                         align_mask = (icd_post_per_page - 1);
 601
 602                         /* Check if icd_start is aligned ICD per page posting */
 603                         if (icd_start % icd_post_per_page) {
 604                                 icd_start_align = ((icd_start +
 605                                                     icd_post_per_page) &
 606                                                     ~(align_mask));
 607                                 phba->fw_config.
 608                                         iscsi_icd_start[ulp_num] =
 609                                         icd_start_align;
 610                         }
 611
 612                         icd_count_align = (icd_count & ~align_mask);
 613
 614                         /* ICD discarded in the process of alignment */
 615                         if (icd_start_align)
 616                                 icd_count_unavailable = ((icd_start_align -
 617                                                           icd_start) +
 618                                                          (icd_count -
 619                                                           icd_count_align));
 620
 621                         /* Updated ICD count available */
 622                         phba->fw_config.iscsi_icd_count[ulp_num] = (icd_count -
 623                                         icd_count_unavailable);
 624
 625                         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
 626                                         "BM_%d : Aligned ICD values\n"
 627                                         "\t ICD Start : %d\n"
 628                                         "\t ICD Count : %d\n"
 629                                         "\t ICD Discarded : %d\n",
 630                                         phba->fw_config.
 631                                         iscsi_icd_start[ulp_num],
 632                                         phba->fw_config.
 633                                         iscsi_icd_count[ulp_num],
 634                                         icd_count_unavailable);
 635                         break;
 636                 }
 637         }
 638
 639         total_icd_count = phba->fw_config.iscsi_icd_count[ulp_num];
 640         phba->params.ios_per_ctrl = (total_icd_count -
 641                                     (total_cid_count +
 642                                      BE2_TMFS + BE2_NOPOUT_REQ));
 643         phba->params.cxns_per_ctrl = total_cid_count;
 644         phba->params.asyncpdus_per_ctrl = total_cid_count;
 645         phba->params.icds_per_ctrl = total_icd_count;
 646         phba->params.num_sge_per_io = BE2_SGE;
 647         phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
 648         phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
 649         phba->params.eq_timer = 64;
 650         phba->params.num_eq_entries = 1024;
 651         phba->params.num_cq_entries = 1024;
 652         phba->params.wrbs_per_cxn = 256;
 653 }
 654
 655 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
 656                            unsigned int id, unsigned int clr_interrupt,
 657                            unsigned int num_processed,
 658                            unsigned char rearm, unsigned char event)
 659 {
 660         u32 val = 0;
 661
 662         if (rearm)
 663                 val |= 1 << DB_EQ_REARM_SHIFT;
 664         if (clr_interrupt)
 665                 val |= 1 << DB_EQ_CLR_SHIFT;
 666         if (event)
 667                 val |= 1 << DB_EQ_EVNT_SHIFT;
 668
 669         val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
 670         /* Setting lower order EQ_ID Bits */
 671         val |= (id & DB_EQ_RING_ID_LOW_MASK);
 672
 673         /* Setting Higher order EQ_ID Bits */
 674         val |= (((id >> DB_EQ_HIGH_FEILD_SHIFT) &
 675                   DB_EQ_RING_ID_HIGH_MASK)
 676                   << DB_EQ_HIGH_SET_SHIFT);
 677
 678         iowrite32(val, phba->db_va + DB_EQ_OFFSET);
 679 }
 680
 681 /**
 682  * be_isr_mcc - The isr routine of the driver.
 683  * @irq: Not used
 684  * @dev_id: Pointer to host adapter structure
 685  */
 686 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
 687 {
 688         struct beiscsi_hba *phba;
 689         struct be_eq_entry *eqe;
 690         struct be_queue_info *eq;
 691         struct be_queue_info *mcc;
 692         unsigned int mcc_events;
 693         struct be_eq_obj *pbe_eq;
 694
 695         pbe_eq = dev_id;
 696         eq = &pbe_eq->q;
 697         phba =  pbe_eq->phba;
 698         mcc = &phba->ctrl.mcc_obj.cq;
 699         eqe = queue_tail_node(eq);
 700
 701         mcc_events = 0;
 702         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 703                                 & EQE_VALID_MASK) {
 704                 if (((eqe->dw[offsetof(struct amap_eq_entry,
 705                      resource_id) / 32] &
 706                      EQE_RESID_MASK) >> 16) == mcc->id) {
 707                         mcc_events++;
 708                 }
 709                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 710                 queue_tail_inc(eq);
 711                 eqe = queue_tail_node(eq);
 712         }
 713
 714         if (mcc_events) {
 715                 queue_work(phba->wq, &pbe_eq->mcc_work);
 716                 hwi_ring_eq_db(phba, eq->id, 1, mcc_events, 1, 1);
 717         }
 718         return IRQ_HANDLED;
 719 }
 720
 721 /**
 722  * be_isr_msix - The isr routine of the driver.
 723  * @irq: Not used
 724  * @dev_id: Pointer to host adapter structure
 725  */
 726 static irqreturn_t be_isr_msix(int irq, void *dev_id)
 727 {
 728         struct beiscsi_hba *phba;
 729         struct be_queue_info *eq;
 730         struct be_eq_obj *pbe_eq;
 731
 732         pbe_eq = dev_id;
 733         eq = &pbe_eq->q;
 734
 735         phba = pbe_eq->phba;
 736         /* disable interrupt till iopoll completes */
 737         hwi_ring_eq_db(phba, eq->id, 1, 0, 0, 1);
 738         irq_poll_sched(&pbe_eq->iopoll);
 739
 740         return IRQ_HANDLED;
 741 }
 742
 743 /**
 744  * be_isr - The isr routine of the driver.
 745  * @irq: Not used
 746  * @dev_id: Pointer to host adapter structure
 747  */
 748 static irqreturn_t be_isr(int irq, void *dev_id)
 749 {
 750         struct beiscsi_hba *phba;
 751         struct hwi_controller *phwi_ctrlr;
 752         struct hwi_context_memory *phwi_context;
 753         struct be_eq_entry *eqe;
 754         struct be_queue_info *eq;
 755         struct be_queue_info *mcc;
 756         unsigned int mcc_events, io_events;
 757         struct be_ctrl_info *ctrl;
 758         struct be_eq_obj *pbe_eq;
 759         int isr, rearm;
 760
 761         phba = dev_id;
 762         ctrl = &phba->ctrl;
 763         isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
 764                        (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
 765         if (!isr)
 766                 return IRQ_NONE;
 767
 768         phwi_ctrlr = phba->phwi_ctrlr;
 769         phwi_context = phwi_ctrlr->phwi_ctxt;
 770         pbe_eq = &phwi_context->be_eq[0];
 771
 772         eq = &phwi_context->be_eq[0].q;
 773         mcc = &phba->ctrl.mcc_obj.cq;
 774         eqe = queue_tail_node(eq);
 775
 776         io_events = 0;
 777         mcc_events = 0;
 778         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 779                                 & EQE_VALID_MASK) {
 780                 if (((eqe->dw[offsetof(struct amap_eq_entry,
 781                       resource_id) / 32] & EQE_RESID_MASK) >> 16) == mcc->id)
 782                         mcc_events++;
 783                 else
 784                         io_events++;
 785                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 786                 queue_tail_inc(eq);
 787                 eqe = queue_tail_node(eq);
 788         }
 789         if (!io_events && !mcc_events)
 790                 return IRQ_NONE;
 791
 792         /* no need to rearm if interrupt is only for IOs */
 793         rearm = 0;
 794         if (mcc_events) {
 795                 queue_work(phba->wq, &pbe_eq->mcc_work);
 796                 /* rearm for MCCQ */
 797                 rearm = 1;
 798         }
 799         if (io_events)
 800                 irq_poll_sched(&pbe_eq->iopoll);
 801         hwi_ring_eq_db(phba, eq->id, 0, (io_events + mcc_events), rearm, 1);
 802         return IRQ_HANDLED;
 803 }
 804
 805
 806 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
 807 {
 808         struct pci_dev *pcidev = phba->pcidev;
 809         struct hwi_controller *phwi_ctrlr;
 810         struct hwi_context_memory *phwi_context;
 811         int ret, msix_vec, i, j;
 812
 813         phwi_ctrlr = phba->phwi_ctrlr;
 814         phwi_context = phwi_ctrlr->phwi_ctxt;
 815
 816         if (phba->msix_enabled) {
 817                 for (i = 0; i < phba->num_cpus; i++) {
 818                         phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME,
 819                                                     GFP_KERNEL);
 820                         if (!phba->msi_name[i]) {
 821                                 ret = -ENOMEM;
 822                                 goto free_msix_irqs;
 823                         }
 824
 825                         sprintf(phba->msi_name[i], "beiscsi_%02x_%02x",
 826                                 phba->shost->host_no, i);
 827                         msix_vec = phba->msix_entries[i].vector;
 828                         ret = request_irq(msix_vec, be_isr_msix, 0,
 829                                           phba->msi_name[i],
 830                                           &phwi_context->be_eq[i]);
 831                         if (ret) {
 832                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
 833                                             "BM_%d : beiscsi_init_irqs-Failed to"
 834                                             "register msix for i = %d\n",
 835                                             i);
 836                                 kfree(phba->msi_name[i]);
 837                                 goto free_msix_irqs;
 838                         }
 839                 }
 840                 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, GFP_KERNEL);
 841                 if (!phba->msi_name[i]) {
 842                         ret = -ENOMEM;
 843                         goto free_msix_irqs;
 844                 }
 845                 sprintf(phba->msi_name[i], "beiscsi_mcc_%02x",
 846                         phba->shost->host_no);
 847                 msix_vec = phba->msix_entries[i].vector;
 848                 ret = request_irq(msix_vec, be_isr_mcc, 0, phba->msi_name[i],
 849                                   &phwi_context->be_eq[i]);
 850                 if (ret) {
 851                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT ,
 852                                     "BM_%d : beiscsi_init_irqs-"
 853                                     "Failed to register beiscsi_msix_mcc\n");
 854                         kfree(phba->msi_name[i]);
 855                         goto free_msix_irqs;
 856                 }
 857
 858         } else {
 859                 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
 860                                   "beiscsi", phba);
 861                 if (ret) {
 862                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
 863                                     "BM_%d : beiscsi_init_irqs-"
 864                                     "Failed to register irq\\n");
 865                         return ret;
 866                 }
 867         }
 868         return 0;
 869 free_msix_irqs:
 870         for (j = i - 1; j >= 0; j--) {
 871                 kfree(phba->msi_name[j]);
 872                 msix_vec = phba->msix_entries[j].vector;
 873                 free_irq(msix_vec, &phwi_context->be_eq[j]);
 874         }
 875         return ret;
 876 }
 877
 878 void hwi_ring_cq_db(struct beiscsi_hba *phba,
 879                            unsigned int id, unsigned int num_processed,
 880                            unsigned char rearm)
 881 {
 882         u32 val = 0;
 883
 884         if (rearm)
 885                 val |= 1 << DB_CQ_REARM_SHIFT;
 886
 887         val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
 888
 889         /* Setting lower order CQ_ID Bits */
 890         val |= (id & DB_CQ_RING_ID_LOW_MASK);
 891
 892         /* Setting Higher order CQ_ID Bits */
 893         val |= (((id >> DB_CQ_HIGH_FEILD_SHIFT) &
 894                   DB_CQ_RING_ID_HIGH_MASK)
 895                   << DB_CQ_HIGH_SET_SHIFT);
 896
 897         iowrite32(val, phba->db_va + DB_CQ_OFFSET);
 898 }
 899
 900 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
 901 {
 902         struct sgl_handle *psgl_handle;
 903         unsigned long flags;
 904
 905         spin_lock_irqsave(&phba->io_sgl_lock, flags);
 906         if (phba->io_sgl_hndl_avbl) {
 907                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
 908                             "BM_%d : In alloc_io_sgl_handle,"
 909                             " io_sgl_alloc_index=%d\n",
 910                             phba->io_sgl_alloc_index);
 911
 912                 psgl_handle = phba->io_sgl_hndl_base[phba->
 913                                                 io_sgl_alloc_index];
 914                 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
 915                 phba->io_sgl_hndl_avbl--;
 916                 if (phba->io_sgl_alloc_index == (phba->params.
 917                                                  ios_per_ctrl - 1))
 918                         phba->io_sgl_alloc_index = 0;
 919                 else
 920                         phba->io_sgl_alloc_index++;
 921         } else
 922                 psgl_handle = NULL;
 923         spin_unlock_irqrestore(&phba->io_sgl_lock, flags);
 924         return psgl_handle;
 925 }
 926
 927 static void
 928 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
 929 {
 930         unsigned long flags;
 931
 932         spin_lock_irqsave(&phba->io_sgl_lock, flags);
 933         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
 934                     "BM_%d : In free_,io_sgl_free_index=%d\n",
 935                     phba->io_sgl_free_index);
 936
 937         if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
 938                 /*
 939                  * this can happen if clean_task is called on a task that
 940                  * failed in xmit_task or alloc_pdu.
 941                  */
 942                  beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
 943                              "BM_%d : Double Free in IO SGL io_sgl_free_index=%d,"
 944                              "value there=%p\n", phba->io_sgl_free_index,
 945                              phba->io_sgl_hndl_base
 946                              [phba->io_sgl_free_index]);
 947                  spin_unlock_irqrestore(&phba->io_sgl_lock, flags);
 948                 return;
 949         }
 950         phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
 951         phba->io_sgl_hndl_avbl++;
 952         if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
 953                 phba->io_sgl_free_index = 0;
 954         else
 955                 phba->io_sgl_free_index++;
 956         spin_unlock_irqrestore(&phba->io_sgl_lock, flags);
 957 }
 958
 959 static inline struct wrb_handle *
 960 beiscsi_get_wrb_handle(struct hwi_wrb_context *pwrb_context,
 961                        unsigned int wrbs_per_cxn)
 962 {
 963         struct wrb_handle *pwrb_handle;
 964         unsigned long flags;
 965
 966         spin_lock_irqsave(&pwrb_context->wrb_lock, flags);
 967         pwrb_handle = pwrb_context->pwrb_handle_base[pwrb_context->alloc_index];
 968         pwrb_context->wrb_handles_available--;
 969         if (pwrb_context->alloc_index == (wrbs_per_cxn - 1))
 970                 pwrb_context->alloc_index = 0;
 971         else
 972                 pwrb_context->alloc_index++;
 973         spin_unlock_irqrestore(&pwrb_context->wrb_lock, flags);
 974
 975         if (pwrb_handle)
 976                 memset(pwrb_handle->pwrb, 0, sizeof(*pwrb_handle->pwrb));
 977
 978         return pwrb_handle;
 979 }
 980
 981 /**
 982  * alloc_wrb_handle - To allocate a wrb handle
 983  * @phba: The hba pointer
 984  * @cid: The cid to use for allocation
 985  * @pwrb_context: ptr to ptr to wrb context
 986  *
 987  * This happens under session_lock until submission to chip
 988  */
 989 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid,
 990                                     struct hwi_wrb_context **pcontext)
 991 {
 992         struct hwi_wrb_context *pwrb_context;
 993         struct hwi_controller *phwi_ctrlr;
 994         uint16_t cri_index = BE_GET_CRI_FROM_CID(cid);
 995
 996         phwi_ctrlr = phba->phwi_ctrlr;
 997         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
 998         /* return the context address */
 999         *pcontext = pwrb_context;
1000         return beiscsi_get_wrb_handle(pwrb_context, phba->params.wrbs_per_cxn);
1001 }
1002
1003 static inline void
1004 beiscsi_put_wrb_handle(struct hwi_wrb_context *pwrb_context,
1005                        struct wrb_handle *pwrb_handle,
1006                        unsigned int wrbs_per_cxn)
1007 {
1008         unsigned long flags;
1009
1010         spin_lock_irqsave(&pwrb_context->wrb_lock, flags);
1011         pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
1012         pwrb_context->wrb_handles_available++;
1013         if (pwrb_context->free_index == (wrbs_per_cxn - 1))
1014                 pwrb_context->free_index = 0;
1015         else
1016                 pwrb_context->free_index++;
1017         spin_unlock_irqrestore(&pwrb_context->wrb_lock, flags);
1018 }
1019
1020 /**
1021  * free_wrb_handle - To free the wrb handle back to pool
1022  * @phba: The hba pointer
1023  * @pwrb_context: The context to free from
1024  * @pwrb_handle: The wrb_handle to free
1025  *
1026  * This happens under session_lock until submission to chip
1027  */
1028 static void
1029 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
1030                 struct wrb_handle *pwrb_handle)
1031 {
1032         beiscsi_put_wrb_handle(pwrb_context,
1033                                pwrb_handle,
1034                                phba->params.wrbs_per_cxn);
1035         beiscsi_log(phba, KERN_INFO,
1036                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1037                     "BM_%d : FREE WRB: pwrb_handle=%p free_index=0x%x"
1038                     "wrb_handles_available=%d\n",
1039                     pwrb_handle, pwrb_context->free_index,
1040                     pwrb_context->wrb_handles_available);
1041 }
1042
1043 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
1044 {
1045         struct sgl_handle *psgl_handle;
1046         unsigned long flags;
1047
1048         spin_lock_irqsave(&phba->mgmt_sgl_lock, flags);
1049         if (phba->eh_sgl_hndl_avbl) {
1050                 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
1051                 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
1052                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1053                             "BM_%d : mgmt_sgl_alloc_index=%d=0x%x\n",
1054                             phba->eh_sgl_alloc_index,
1055                             phba->eh_sgl_alloc_index);
1056
1057                 phba->eh_sgl_hndl_avbl--;
1058                 if (phba->eh_sgl_alloc_index ==
1059                     (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
1060                      1))
1061                         phba->eh_sgl_alloc_index = 0;
1062                 else
1063                         phba->eh_sgl_alloc_index++;
1064         } else
1065                 psgl_handle = NULL;
1066         spin_unlock_irqrestore(&phba->mgmt_sgl_lock, flags);
1067         return psgl_handle;
1068 }
1069
1070 void
1071 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1072 {
1073         unsigned long flags;
1074
1075         spin_lock_irqsave(&phba->mgmt_sgl_lock, flags);
1076         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1077                     "BM_%d : In  free_mgmt_sgl_handle,"
1078                     "eh_sgl_free_index=%d\n",
1079                     phba->eh_sgl_free_index);
1080
1081         if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
1082                 /*
1083                  * this can happen if clean_task is called on a task that
1084                  * failed in xmit_task or alloc_pdu.
1085                  */
1086                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
1087                             "BM_%d : Double Free in eh SGL ,"
1088                             "eh_sgl_free_index=%d\n",
1089                             phba->eh_sgl_free_index);
1090                 spin_unlock_irqrestore(&phba->mgmt_sgl_lock, flags);
1091                 return;
1092         }
1093         phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
1094         phba->eh_sgl_hndl_avbl++;
1095         if (phba->eh_sgl_free_index ==
1096             (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
1097                 phba->eh_sgl_free_index = 0;
1098         else
1099                 phba->eh_sgl_free_index++;
1100         spin_unlock_irqrestore(&phba->mgmt_sgl_lock, flags);
1101 }
1102
1103 static void
1104 be_complete_io(struct beiscsi_conn *beiscsi_conn,
1105                 struct iscsi_task *task,
1106                 struct common_sol_cqe *csol_cqe)
1107 {
1108         struct beiscsi_io_task *io_task = task->dd_data;
1109         struct be_status_bhs *sts_bhs =
1110                                 (struct be_status_bhs *)io_task->cmd_bhs;
1111         struct iscsi_conn *conn = beiscsi_conn->conn;
1112         unsigned char *sense;
1113         u32 resid = 0, exp_cmdsn, max_cmdsn;
1114         u8 rsp, status, flags;
1115
1116         exp_cmdsn = csol_cqe->exp_cmdsn;
1117         max_cmdsn = (csol_cqe->exp_cmdsn +
1118                      csol_cqe->cmd_wnd - 1);
1119         rsp = csol_cqe->i_resp;
1120         status = csol_cqe->i_sts;
1121         flags = csol_cqe->i_flags;
1122         resid = csol_cqe->res_cnt;
1123
1124         if (!task->sc) {
1125                 if (io_task->scsi_cmnd) {
1126                         scsi_dma_unmap(io_task->scsi_cmnd);
1127                         io_task->scsi_cmnd = NULL;
1128                 }
1129
1130                 return;
1131         }
1132         task->sc->result = (DID_OK << 16) | status;
1133         if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
1134                 task->sc->result = DID_ERROR << 16;
1135                 goto unmap;
1136         }
1137
1138         /* bidi not initially supported */
1139         if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
1140                 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
1141                         task->sc->result = DID_ERROR << 16;
1142
1143                 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
1144                         scsi_set_resid(task->sc, resid);
1145                         if (!status && (scsi_bufflen(task->sc) - resid <
1146                             task->sc->underflow))
1147                                 task->sc->result = DID_ERROR << 16;
1148                 }
1149         }
1150
1151         if (status == SAM_STAT_CHECK_CONDITION) {
1152                 u16 sense_len;
1153                 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
1154
1155                 sense = sts_bhs->sense_info + sizeof(unsigned short);
1156                 sense_len = be16_to_cpu(*slen);
1157                 memcpy(task->sc->sense_buffer, sense,
1158                        min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
1159         }
1160
1161         if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ)
1162                 conn->rxdata_octets += resid;
1163 unmap:
1164         if (io_task->scsi_cmnd) {
1165                 scsi_dma_unmap(io_task->scsi_cmnd);
1166                 io_task->scsi_cmnd = NULL;
1167         }
1168         iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
1169 }
1170
1171 static void
1172 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
1173                     struct iscsi_task *task,
1174                     struct common_sol_cqe *csol_cqe)
1175 {
1176         struct iscsi_logout_rsp *hdr;
1177         struct beiscsi_io_task *io_task = task->dd_data;
1178         struct iscsi_conn *conn = beiscsi_conn->conn;
1179
1180         hdr = (struct iscsi_logout_rsp *)task->hdr;
1181         hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1182         hdr->t2wait = 5;
1183         hdr->t2retain = 0;
1184         hdr->flags = csol_cqe->i_flags;
1185         hdr->response = csol_cqe->i_resp;
1186         hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
1187         hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +
1188                                      csol_cqe->cmd_wnd - 1);
1189
1190         hdr->dlength[0] = 0;
1191         hdr->dlength[1] = 0;
1192         hdr->dlength[2] = 0;
1193         hdr->hlength = 0;
1194         hdr->itt = io_task->libiscsi_itt;
1195         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1196 }
1197
1198 static void
1199 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
1200                  struct iscsi_task *task,
1201                  struct common_sol_cqe *csol_cqe)
1202 {
1203         struct iscsi_tm_rsp *hdr;
1204         struct iscsi_conn *conn = beiscsi_conn->conn;
1205         struct beiscsi_io_task *io_task = task->dd_data;
1206
1207         hdr = (struct iscsi_tm_rsp *)task->hdr;
1208         hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
1209         hdr->flags = csol_cqe->i_flags;
1210         hdr->response = csol_cqe->i_resp;
1211         hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
1212         hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +
1213                                      csol_cqe->cmd_wnd - 1);
1214
1215         hdr->itt = io_task->libiscsi_itt;
1216         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1217 }
1218
1219 static void
1220 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
1221                        struct beiscsi_hba *phba, struct sol_cqe *psol)
1222 {
1223         struct hwi_wrb_context *pwrb_context;
1224         uint16_t wrb_index, cid, cri_index;
1225         struct hwi_controller *phwi_ctrlr;
1226         struct wrb_handle *pwrb_handle;
1227         struct iscsi_task *task;
1228
1229         phwi_ctrlr = phba->phwi_ctrlr;
1230         if (is_chip_be2_be3r(phba)) {
1231                 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe,
1232                                           wrb_idx, psol);
1233                 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe,
1234                                     cid, psol);
1235         } else {
1236                 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2,
1237                                           wrb_idx, psol);
1238                 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2,
1239                                     cid, psol);
1240         }
1241
1242         cri_index = BE_GET_CRI_FROM_CID(cid);
1243         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
1244         pwrb_handle = pwrb_context->pwrb_handle_basestd[wrb_index];
1245         task = pwrb_handle->pio_handle;
1246         iscsi_put_task(task);
1247 }
1248
1249 static void
1250 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
1251                         struct iscsi_task *task,
1252                         struct common_sol_cqe *csol_cqe)
1253 {
1254         struct iscsi_nopin *hdr;
1255         struct iscsi_conn *conn = beiscsi_conn->conn;
1256         struct beiscsi_io_task *io_task = task->dd_data;
1257
1258         hdr = (struct iscsi_nopin *)task->hdr;
1259         hdr->flags = csol_cqe->i_flags;
1260         hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
1261         hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +
1262                                      csol_cqe->cmd_wnd - 1);
1263
1264         hdr->opcode = ISCSI_OP_NOOP_IN;
1265         hdr->itt = io_task->libiscsi_itt;
1266         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1267 }
1268
1269 static void adapter_get_sol_cqe(struct beiscsi_hba *phba,
1270                 struct sol_cqe *psol,
1271                 struct common_sol_cqe *csol_cqe)
1272 {
1273         if (is_chip_be2_be3r(phba)) {
1274                 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe,
1275                                                     i_exp_cmd_sn, psol);
1276                 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe,
1277                                                   i_res_cnt, psol);
1278                 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe,
1279                                                   i_cmd_wnd, psol);
1280                 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe,
1281                                                     wrb_index, psol);
1282                 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe,
1283                                               cid, psol);
1284                 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe,
1285                                                  hw_sts, psol);
1286                 csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe,
1287                                                  i_resp, psol);
1288                 csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe,
1289                                                 i_sts, psol);
1290                 csol_cqe->i_flags = AMAP_GET_BITS(struct amap_sol_cqe,
1291                                                   i_flags, psol);
1292         } else {
1293                 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1294                                                     i_exp_cmd_sn, psol);
1295                 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1296                                                   i_res_cnt, psol);
1297                 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1298                                                     wrb_index, psol);
1299                 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1300                                               cid, psol);
1301                 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1302                                                  hw_sts, psol);
1303                 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1304                                                   i_cmd_wnd, psol);
1305                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1306                                   cmd_cmpl, psol))
1307                         csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1308                                                         i_sts, psol);
1309                 else
1310                         csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1311                                                          i_sts, psol);
1312                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1313                                   u, psol))
1314                         csol_cqe->i_flags = ISCSI_FLAG_CMD_UNDERFLOW;
1315
1316                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1317                                   o, psol))
1318                         csol_cqe->i_flags |= ISCSI_FLAG_CMD_OVERFLOW;
1319         }
1320 }
1321
1322
1323 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1324                              struct beiscsi_hba *phba, struct sol_cqe *psol)
1325 {
1326         struct hwi_wrb_context *pwrb_context;
1327         struct wrb_handle *pwrb_handle;
1328         struct iscsi_wrb *pwrb = NULL;
1329         struct hwi_controller *phwi_ctrlr;
1330         struct iscsi_task *task;
1331         unsigned int type;
1332         struct iscsi_conn *conn = beiscsi_conn->conn;
1333         struct iscsi_session *session = conn->session;
1334         struct common_sol_cqe csol_cqe = {0};
1335         uint16_t cri_index = 0;
1336
1337         phwi_ctrlr = phba->phwi_ctrlr;
1338
1339         /* Copy the elements to a common structure */
1340         adapter_get_sol_cqe(phba, psol, &csol_cqe);
1341
1342         cri_index = BE_GET_CRI_FROM_CID(csol_cqe.cid);
1343         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
1344
1345         pwrb_handle = pwrb_context->pwrb_handle_basestd[
1346                       csol_cqe.wrb_index];
1347
1348         task = pwrb_handle->pio_handle;
1349         pwrb = pwrb_handle->pwrb;
1350         type = ((struct beiscsi_io_task *)task->dd_data)->wrb_type;
1351
1352         spin_lock_bh(&session->back_lock);
1353         switch (type) {
1354         case HWH_TYPE_IO:
1355         case HWH_TYPE_IO_RD:
1356                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
1357                      ISCSI_OP_NOOP_OUT)
1358                         be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe);
1359                 else
1360                         be_complete_io(beiscsi_conn, task, &csol_cqe);
1361                 break;
1362
1363         case HWH_TYPE_LOGOUT:
1364                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
1365                         be_complete_logout(beiscsi_conn, task, &csol_cqe);
1366                 else
1367                         be_complete_tmf(beiscsi_conn, task, &csol_cqe);
1368                 break;
1369
1370         case HWH_TYPE_LOGIN:
1371                 beiscsi_log(phba, KERN_ERR,
1372                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1373                             "BM_%d :\t\t No HWH_TYPE_LOGIN Expected in"
1374                             " hwi_complete_cmd- Solicited path\n");
1375                 break;
1376
1377         case HWH_TYPE_NOP:
1378                 be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe);
1379                 break;
1380
1381         default:
1382                 beiscsi_log(phba, KERN_WARNING,
1383                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1384                             "BM_%d : In hwi_complete_cmd, unknown type = %d"
1385                             "wrb_index 0x%x CID 0x%x\n", type,
1386                             csol_cqe.wrb_index,
1387                             csol_cqe.cid);
1388                 break;
1389         }
1390
1391         spin_unlock_bh(&session->back_lock);
1392 }
1393
1394 /**
1395  * ASYNC PDUs include
1396  * a. Unsolicited NOP-In (target initiated NOP-In)
1397  * b. ASYNC Messages
1398  * c. Reject PDU
1399  * d. Login response
1400  * These headers arrive unprocessed by the EP firmware.
1401  * iSCSI layer processes them.
1402  */
1403 static unsigned int
1404 beiscsi_complete_pdu(struct beiscsi_conn *beiscsi_conn,
1405                 struct pdu_base *phdr, void *pdata, unsigned int dlen)
1406 {
1407         struct beiscsi_hba *phba = beiscsi_conn->phba;
1408         struct iscsi_conn *conn = beiscsi_conn->conn;
1409         struct beiscsi_io_task *io_task;
1410         struct iscsi_hdr *login_hdr;
1411         struct iscsi_task *task;
1412         u8 code;
1413
1414         code = AMAP_GET_BITS(struct amap_pdu_base, opcode, phdr);
1415         switch (code) {
1416         case ISCSI_OP_NOOP_IN:
1417                 pdata = NULL;
1418                 dlen = 0;
1419                 break;
1420         case ISCSI_OP_ASYNC_EVENT:
1421                 break;
1422         case ISCSI_OP_REJECT:
1423                 WARN_ON(!pdata);
1424                 WARN_ON(!(dlen == 48));
1425                 beiscsi_log(phba, KERN_ERR,
1426                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1427                             "BM_%d : In ISCSI_OP_REJECT\n");
1428                 break;
1429         case ISCSI_OP_LOGIN_RSP:
1430         case ISCSI_OP_TEXT_RSP:
1431                 task = conn->login_task;
1432                 io_task = task->dd_data;
1433                 login_hdr = (struct iscsi_hdr *)phdr;
1434                 login_hdr->itt = io_task->libiscsi_itt;
1435                 break;
1436         default:
1437                 beiscsi_log(phba, KERN_WARNING,
1438                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1439                             "BM_%d : unrecognized async PDU opcode 0x%x\n",
1440                             code);
1441                 return 1;
1442         }
1443         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)phdr, pdata, dlen);
1444         return 0;
1445 }
1446
1447 static inline void
1448 beiscsi_hdl_put_handle(struct hd_async_context *pasync_ctx,
1449                          struct hd_async_handle *pasync_handle)
1450 {
1451         if (pasync_handle->is_header) {
1452                 list_add_tail(&pasync_handle->link,
1453                                 &pasync_ctx->async_header.free_list);
1454                 pasync_ctx->async_header.free_entries++;
1455         } else {
1456                 list_add_tail(&pasync_handle->link,
1457                                 &pasync_ctx->async_data.free_list);
1458                 pasync_ctx->async_data.free_entries++;
1459         }
1460 }
1461
1462 static struct hd_async_handle *
1463 beiscsi_hdl_get_handle(struct beiscsi_conn *beiscsi_conn,
1464                        struct hd_async_context *pasync_ctx,
1465                        struct i_t_dpdu_cqe *pdpdu_cqe)
1466 {
1467         struct beiscsi_hba *phba = beiscsi_conn->phba;
1468         struct hd_async_handle *pasync_handle;
1469         struct be_bus_address phys_addr;
1470         u8 final, error = 0;
1471         u16 cid, code, ci;
1472         u32 dpl;
1473
1474         cid = beiscsi_conn->beiscsi_conn_cid;
1475         /**
1476          * This function is invoked to get the right async_handle structure
1477          * from a given DEF PDU CQ entry.
1478          *
1479          * - index in CQ entry gives the vertical index
1480          * - address in CQ entry is the offset where the DMA last ended
1481          * - final - no more notifications for this PDU
1482          */
1483         if (is_chip_be2_be3r(phba)) {
1484                 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1485                                     dpl, pdpdu_cqe);
1486                 ci = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1487                                       index, pdpdu_cqe);
1488                 final = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1489                                       final, pdpdu_cqe);
1490         } else {
1491                 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
1492                                     dpl, pdpdu_cqe);
1493                 ci = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
1494                                       index, pdpdu_cqe);
1495                 final = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
1496                                       final, pdpdu_cqe);
1497         }
1498
1499         /**
1500          * DB addr Hi/Lo is same for BE and SKH.
1501          * Subtract the dataplacementlength to get to the base.
1502          */
1503         phys_addr.u.a32.address_lo = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1504                                                    db_addr_lo, pdpdu_cqe);
1505         phys_addr.u.a32.address_lo -= dpl;
1506         phys_addr.u.a32.address_hi = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1507                                                    db_addr_hi, pdpdu_cqe);
1508
1509         code = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe, code, pdpdu_cqe);
1510         switch (code) {
1511         case UNSOL_HDR_NOTIFY:
1512                 pasync_handle = pasync_ctx->async_entry[ci].header;
1513                 break;
1514         case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1515                 error = 1;
1516         case UNSOL_DATA_NOTIFY:
1517                 pasync_handle = pasync_ctx->async_entry[ci].data;
1518                 break;
1519         /* called only for above codes */
1520         default:
1521                 pasync_handle = NULL;
1522                 break;
1523         }
1524
1525         if (!pasync_handle) {
1526                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI,
1527                             "BM_%d : cid %d async PDU handle not found - code %d ci %d addr %llx\n",
1528                             cid, code, ci, phys_addr.u.a64.address);
1529                 return pasync_handle;
1530         }
1531
1532         if (pasync_handle->pa.u.a64.address != phys_addr.u.a64.address ||
1533             pasync_handle->index != ci) {
1534                 /* driver bug - if ci does not match async handle index */
1535                 error = 1;
1536                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI,
1537                             "BM_%d : cid %u async PDU handle mismatch - addr in %cQE %llx at %u:addr in CQE %llx ci %u\n",
1538                             cid, pasync_handle->is_header ? 'H' : 'D',
1539                             pasync_handle->pa.u.a64.address,
1540                             pasync_handle->index,
1541                             phys_addr.u.a64.address, ci);
1542                 /* FW has stale address - attempt continuing by dropping */
1543         }
1544
1545         /**
1546          * Each CID is associated with unique CRI.
1547          * ASYNC_CRI_FROM_CID mapping and CRI_FROM_CID are totaly different.
1548          **/
1549         pasync_handle->cri = BE_GET_ASYNC_CRI_FROM_CID(cid);
1550         pasync_handle->is_final = final;
1551         pasync_handle->buffer_len = dpl;
1552         /* empty the slot */
1553         if (pasync_handle->is_header)
1554                 pasync_ctx->async_entry[ci].header = NULL;
1555         else
1556                 pasync_ctx->async_entry[ci].data = NULL;
1557
1558         /**
1559          * DEF PDU header and data buffers with errors should be simply
1560          * dropped as there are no consumers for it.
1561          */
1562         if (error) {
1563                 beiscsi_hdl_put_handle(pasync_ctx, pasync_handle);
1564                 pasync_handle = NULL;
1565         }
1566         return pasync_handle;
1567 }
1568
1569 static void
1570 beiscsi_hdl_purge_handles(struct beiscsi_hba *phba,
1571                           struct hd_async_context *pasync_ctx,
1572                           u16 cri)
1573 {
1574         struct hd_async_handle *pasync_handle, *tmp_handle;
1575         struct list_head *plist;
1576
1577         plist  = &pasync_ctx->async_entry[cri].wq.list;
1578         list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1579                 list_del(&pasync_handle->link);
1580                 beiscsi_hdl_put_handle(pasync_ctx, pasync_handle);
1581         }
1582
1583         INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wq.list);
1584         pasync_ctx->async_entry[cri].wq.hdr_len = 0;
1585         pasync_ctx->async_entry[cri].wq.bytes_received = 0;
1586         pasync_ctx->async_entry[cri].wq.bytes_needed = 0;
1587 }
1588
1589 static unsigned int
1590 beiscsi_hdl_fwd_pdu(struct beiscsi_conn *beiscsi_conn,
1591                     struct hd_async_context *pasync_ctx,
1592                     u16 cri)
1593 {
1594         struct iscsi_session *session = beiscsi_conn->conn->session;
1595         struct hd_async_handle *pasync_handle, *plast_handle;
1596         struct beiscsi_hba *phba = beiscsi_conn->phba;
1597         void *phdr = NULL, *pdata = NULL;
1598         u32 dlen = 0, status = 0;
1599         struct list_head *plist;
1600
1601         plist = &pasync_ctx->async_entry[cri].wq.list;
1602         plast_handle = NULL;
1603         list_for_each_entry(pasync_handle, plist, link) {
1604                 plast_handle = pasync_handle;
1605                 /* get the header, the first entry */
1606                 if (!phdr) {
1607                         phdr = pasync_handle->pbuffer;
1608                         continue;
1609                 }
1610                 /* use first buffer to collect all the data */
1611                 if (!pdata) {
1612                         pdata = pasync_handle->pbuffer;
1613                         dlen = pasync_handle->buffer_len;
1614                         continue;
1615                 }
1616                 memcpy(pdata + dlen, pasync_handle->pbuffer,
1617                        pasync_handle->buffer_len);
1618                 dlen += pasync_handle->buffer_len;
1619         }
1620
1621         if (!plast_handle->is_final) {
1622                 /* last handle should have final PDU notification from FW */
1623                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI,
1624                             "BM_%d : cid %u %p fwd async PDU with last handle missing - HL%u:DN%u:DR%u\n",
1625                             beiscsi_conn->beiscsi_conn_cid, plast_handle,
1626                             pasync_ctx->async_entry[cri].wq.hdr_len,
1627                             pasync_ctx->async_entry[cri].wq.bytes_needed,
1628                             pasync_ctx->async_entry[cri].wq.bytes_received);
1629         }
1630         spin_lock_bh(&session->back_lock);
1631         status = beiscsi_complete_pdu(beiscsi_conn, phdr, pdata, dlen);
1632         spin_unlock_bh(&session->back_lock);
1633         beiscsi_hdl_purge_handles(phba, pasync_ctx, cri);
1634         return status;
1635 }
1636
1637 static unsigned int
1638 beiscsi_hdl_gather_pdu(struct beiscsi_conn *beiscsi_conn,
1639                        struct hd_async_context *pasync_ctx,
1640                        struct hd_async_handle *pasync_handle)
1641 {
1642         unsigned int bytes_needed = 0, status = 0;
1643         u16 cri = pasync_handle->cri;
1644         struct cri_wait_queue *wq;
1645         struct beiscsi_hba *phba;
1646         struct pdu_base *ppdu;
1647         char *err = "";
1648
1649         phba = beiscsi_conn->phba;
1650         wq = &pasync_ctx->async_entry[cri].wq;
1651         if (pasync_handle->is_header) {
1652                 /* check if PDU hdr is rcv'd when old hdr not completed */
1653                 if (wq->hdr_len) {
1654                         err = "incomplete";
1655                         goto drop_pdu;
1656                 }
1657                 ppdu = pasync_handle->pbuffer;
1658                 bytes_needed = AMAP_GET_BITS(struct amap_pdu_base,
1659                                              data_len_hi, ppdu);
1660                 bytes_needed <<= 16;
1661                 bytes_needed |= be16_to_cpu(AMAP_GET_BITS(struct amap_pdu_base,
1662                                                           data_len_lo, ppdu));
1663                 wq->hdr_len = pasync_handle->buffer_len;
1664                 wq->bytes_received = 0;
1665                 wq->bytes_needed = bytes_needed;
1666                 list_add_tail(&pasync_handle->link, &wq->list);
1667                 if (!bytes_needed)
1668                         status = beiscsi_hdl_fwd_pdu(beiscsi_conn,
1669                                                      pasync_ctx, cri);
1670         } else {
1671                 /* check if data received has header and is needed */
1672                 if (!wq->hdr_len || !wq->bytes_needed) {
1673                         err = "header less";
1674                         goto drop_pdu;
1675                 }
1676                 wq->bytes_received += pasync_handle->buffer_len;
1677                 /* Something got overwritten? Better catch it here. */
1678                 if (wq->bytes_received > wq->bytes_needed) {
1679                         err = "overflow";
1680                         goto drop_pdu;
1681                 }
1682                 list_add_tail(&pasync_handle->link, &wq->list);
1683                 if (wq->bytes_received == wq->bytes_needed)
1684                         status = beiscsi_hdl_fwd_pdu(beiscsi_conn,
1685                                                      pasync_ctx, cri);
1686         }
1687         return status;
1688
1689 drop_pdu:
1690         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI,
1691                     "BM_%d : cid %u async PDU %s - def-%c:HL%u:DN%u:DR%u\n",
1692                     beiscsi_conn->beiscsi_conn_cid, err,
1693                     pasync_handle->is_header ? 'H' : 'D',
1694                     wq->hdr_len, wq->bytes_needed,
1695                     pasync_handle->buffer_len);
1696         /* discard this handle */
1697         beiscsi_hdl_put_handle(pasync_ctx, pasync_handle);
1698         /* free all the other handles in cri_wait_queue */
1699         beiscsi_hdl_purge_handles(phba, pasync_ctx, cri);
1700         /* try continuing */
1701         return status;
1702 }
1703
1704 static void
1705 beiscsi_hdq_post_handles(struct beiscsi_hba *phba,
1706                          u8 header, u8 ulp_num)
1707 {
1708         struct hd_async_handle *pasync_handle, *tmp, **slot;
1709         struct hd_async_context *pasync_ctx;
1710         struct hwi_controller *phwi_ctrlr;
1711         struct list_head *hfree_list;
1712         struct phys_addr *pasync_sge;
1713         u32 ring_id, doorbell = 0;
1714         u16 index, num_entries;
1715         u32 doorbell_offset;
1716         u16 prod = 0, cons;
1717
1718         phwi_ctrlr = phba->phwi_ctrlr;
1719         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr, ulp_num);
1720         num_entries = pasync_ctx->num_entries;
1721         if (header) {
1722                 cons = pasync_ctx->async_header.free_entries;
1723                 hfree_list = &pasync_ctx->async_header.free_list;
1724                 ring_id = phwi_ctrlr->default_pdu_hdr[ulp_num].id;
1725                 doorbell_offset = phwi_ctrlr->default_pdu_hdr[ulp_num].
1726                                         doorbell_offset;
1727         } else {
1728                 cons = pasync_ctx->async_data.free_entries;
1729                 hfree_list = &pasync_ctx->async_data.free_list;
1730                 ring_id = phwi_ctrlr->default_pdu_data[ulp_num].id;
1731                 doorbell_offset = phwi_ctrlr->default_pdu_data[ulp_num].
1732                                         doorbell_offset;
1733         }
1734         /* number of entries posted must be in multiples of 8 */
1735         if (cons % 8)
1736                 return;
1737
1738         list_for_each_entry_safe(pasync_handle, tmp, hfree_list, link) {
1739                 list_del_init(&pasync_handle->link);
1740                 pasync_handle->is_final = 0;
1741                 pasync_handle->buffer_len = 0;
1742
1743                 /* handles can be consumed out of order, use index in handle */
1744                 index = pasync_handle->index;
1745                 WARN_ON(pasync_handle->is_header != header);
1746                 if (header)
1747                         slot = &pasync_ctx->async_entry[index].header;
1748                 else
1749                         slot = &pasync_ctx->async_entry[index].data;
1750                 /**
1751                  * The slot just tracks handle's hold and release, so
1752                  * overwriting at the same index won't do any harm but
1753                  * needs to be caught.
1754                  */
1755                 if (*slot != NULL) {
1756                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI,
1757                                     "BM_%d : async PDU %s slot at %u not empty\n",
1758                                     header ? "header" : "data", index);
1759                 }
1760                 /**
1761                  * We use same freed index as in completion to post so this
1762                  * operation is not required for refills. Its required only
1763                  * for ring creation.
1764                  */
1765                 if (header)
1766                         pasync_sge = pasync_ctx->async_header.ring_base;
1767                 else
1768                         pasync_sge = pasync_ctx->async_data.ring_base;
1769                 pasync_sge += index;
1770                 /* if its a refill then address is same; hi is lo */
1771                 WARN_ON(pasync_sge->hi &&
1772                         pasync_sge->hi != pasync_handle->pa.u.a32.address_lo);
1773                 WARN_ON(pasync_sge->lo &&
1774                         pasync_sge->lo != pasync_handle->pa.u.a32.address_hi);
1775                 pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1776                 pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1777
1778                 *slot = pasync_handle;
1779                 if (++prod == cons)
1780                         break;
1781         }
1782         if (header)
1783                 pasync_ctx->async_header.free_entries -= prod;
1784         else
1785                 pasync_ctx->async_data.free_entries -= prod;
1786
1787         doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1788         doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1789         doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1790         doorbell |= (prod & DB_DEF_PDU_CQPROC_MASK) << DB_DEF_PDU_CQPROC_SHIFT;
1791         iowrite32(doorbell, phba->db_va + doorbell_offset);
1792 }
1793
1794 static void
1795 beiscsi_hdq_process_compl(struct beiscsi_conn *beiscsi_conn,
1796                           struct i_t_dpdu_cqe *pdpdu_cqe)
1797 {
1798         struct beiscsi_hba *phba = beiscsi_conn->phba;
1799         struct hd_async_handle *pasync_handle = NULL;
1800         struct hd_async_context *pasync_ctx;
1801         struct hwi_controller *phwi_ctrlr;
1802         u16 cid_cri;
1803         u8 ulp_num;
1804
1805         phwi_ctrlr = phba->phwi_ctrlr;
1806         cid_cri = BE_GET_CRI_FROM_CID(beiscsi_conn->beiscsi_conn_cid);
1807         ulp_num = BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr, cid_cri);
1808         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr, ulp_num);
1809         pasync_handle = beiscsi_hdl_get_handle(beiscsi_conn, pasync_ctx,
1810                                                pdpdu_cqe);
1811         if (!pasync_handle)
1812                 return;
1813
1814         beiscsi_hdl_gather_pdu(beiscsi_conn, pasync_ctx, pasync_handle);
1815         beiscsi_hdq_post_handles(phba, pasync_handle->is_header, ulp_num);
1816 }
1817
1818 void beiscsi_process_mcc_cq(struct beiscsi_hba *phba)
1819 {
1820         struct be_queue_info *mcc_cq;
1821         struct  be_mcc_compl *mcc_compl;
1822         unsigned int num_processed = 0;
1823
1824         mcc_cq = &phba->ctrl.mcc_obj.cq;
1825         mcc_compl = queue_tail_node(mcc_cq);
1826         mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1827         while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
1828                 if (beiscsi_hba_in_error(phba))
1829                         return;
1830
1831                 if (num_processed >= 32) {
1832                         hwi_ring_cq_db(phba, mcc_cq->id,
1833                                         num_processed, 0);
1834                         num_processed = 0;
1835                 }
1836                 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
1837                         beiscsi_process_async_event(phba, mcc_compl);
1838                 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
1839                         beiscsi_process_mcc_compl(&phba->ctrl, mcc_compl);
1840                 }
1841
1842                 mcc_compl->flags = 0;
1843                 queue_tail_inc(mcc_cq);
1844                 mcc_compl = queue_tail_node(mcc_cq);
1845                 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1846                 num_processed++;
1847         }
1848
1849         if (num_processed > 0)
1850                 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1);
1851 }
1852
1853 static void beiscsi_mcc_work(struct work_struct *work)
1854 {
1855         struct be_eq_obj *pbe_eq;
1856         struct beiscsi_hba *phba;
1857
1858         pbe_eq = container_of(work, struct be_eq_obj, mcc_work);
1859         phba = pbe_eq->phba;
1860         beiscsi_process_mcc_cq(phba);
1861         /* rearm EQ for further interrupts */
1862         if (!beiscsi_hba_in_error(phba))
1863                 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
1864 }
1865
1866 /**
1867  * beiscsi_process_cq()- Process the Completion Queue
1868  * @pbe_eq: Event Q on which the Completion has come
1869  * @budget: Max number of events to processed
1870  *
1871  * return
1872  *     Number of Completion Entries processed.
1873  **/
1874 unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq, int budget)
1875 {
1876         struct be_queue_info *cq;
1877         struct sol_cqe *sol;
1878         struct dmsg_cqe *dmsg;
1879         unsigned int total = 0;
1880         unsigned int num_processed = 0;
1881         unsigned short code = 0, cid = 0;
1882         uint16_t cri_index = 0;
1883         struct beiscsi_conn *beiscsi_conn;
1884         struct beiscsi_endpoint *beiscsi_ep;
1885         struct iscsi_endpoint *ep;
1886         struct beiscsi_hba *phba;
1887
1888         cq = pbe_eq->cq;
1889         sol = queue_tail_node(cq);
1890         phba = pbe_eq->phba;
1891
1892         while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1893                CQE_VALID_MASK) {
1894                 if (beiscsi_hba_in_error(phba))
1895                         return 0;
1896
1897                 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1898
1899                  code = (sol->dw[offsetof(struct amap_sol_cqe, code) /
1900                          32] & CQE_CODE_MASK);
1901
1902                  /* Get the CID */
1903                 if (is_chip_be2_be3r(phba)) {
1904                         cid = AMAP_GET_BITS(struct amap_sol_cqe, cid, sol);
1905                 } else {
1906                         if ((code == DRIVERMSG_NOTIFY) ||
1907                             (code == UNSOL_HDR_NOTIFY) ||
1908                             (code == UNSOL_DATA_NOTIFY))
1909                                 cid = AMAP_GET_BITS(
1910                                                     struct amap_i_t_dpdu_cqe_v2,
1911                                                     cid, sol);
1912                          else
1913                                  cid = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1914                                                      cid, sol);
1915                 }
1916
1917                 cri_index = BE_GET_CRI_FROM_CID(cid);
1918                 ep = phba->ep_array[cri_index];
1919
1920                 if (ep == NULL) {
1921                         /* connection has already been freed
1922                          * just move on to next one
1923                          */
1924                         beiscsi_log(phba, KERN_WARNING,
1925                                     BEISCSI_LOG_INIT,
1926                                     "BM_%d : proc cqe of disconn ep: cid %d\n",
1927                                     cid);
1928                         goto proc_next_cqe;
1929                 }
1930
1931                 beiscsi_ep = ep->dd_data;
1932                 beiscsi_conn = beiscsi_ep->conn;
1933
1934                 /* replenish cq */
1935                 if (num_processed == 32) {
1936                         hwi_ring_cq_db(phba, cq->id, 32, 0);
1937                         num_processed = 0;
1938                 }
1939                 total++;
1940
1941                 switch (code) {
1942                 case SOL_CMD_COMPLETE:
1943                         hwi_complete_cmd(beiscsi_conn, phba, sol);
1944                         break;
1945                 case DRIVERMSG_NOTIFY:
1946                         beiscsi_log(phba, KERN_INFO,
1947                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1948                                     "BM_%d : Received %s[%d] on CID : %d\n",
1949                                     cqe_desc[code], code, cid);
1950
1951                         dmsg = (struct dmsg_cqe *)sol;
1952                         hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1953                         break;
1954                 case UNSOL_HDR_NOTIFY:
1955                         beiscsi_log(phba, KERN_INFO,
1956                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1957                                     "BM_%d : Received %s[%d] on CID : %d\n",
1958                                     cqe_desc[code], code, cid);
1959
1960                         spin_lock_bh(&phba->async_pdu_lock);
1961                         beiscsi_hdq_process_compl(beiscsi_conn,
1962                                                   (struct i_t_dpdu_cqe *)sol);
1963                         spin_unlock_bh(&phba->async_pdu_lock);
1964                         break;
1965                 case UNSOL_DATA_NOTIFY:
1966                         beiscsi_log(phba, KERN_INFO,
1967                                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1968                                     "BM_%d : Received %s[%d] on CID : %d\n",
1969                                     cqe_desc[code], code, cid);
1970
1971                         spin_lock_bh(&phba->async_pdu_lock);
1972                         beiscsi_hdq_process_compl(beiscsi_conn,
1973                                                   (struct i_t_dpdu_cqe *)sol);
1974                         spin_unlock_bh(&phba->async_pdu_lock);
1975                         break;
1976                 case CXN_INVALIDATE_INDEX_NOTIFY:
1977                 case CMD_INVALIDATED_NOTIFY:
1978                 case CXN_INVALIDATE_NOTIFY:
1979                         beiscsi_log(phba, KERN_ERR,
1980                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1981                                     "BM_%d : Ignoring %s[%d] on CID : %d\n",
1982                                     cqe_desc[code], code, cid);
1983                         break;
1984                 case CXN_KILLED_HDR_DIGEST_ERR:
1985                 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1986                         beiscsi_log(phba, KERN_ERR,
1987                                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1988                                     "BM_%d : Cmd Notification %s[%d] on CID : %d\n",
1989                                     cqe_desc[code], code,  cid);
1990                         break;
1991                 case CMD_KILLED_INVALID_STATSN_RCVD:
1992                 case CMD_KILLED_INVALID_R2T_RCVD:
1993                 case CMD_CXN_KILLED_LUN_INVALID:
1994                 case CMD_CXN_KILLED_ICD_INVALID:
1995                 case CMD_CXN_KILLED_ITT_INVALID:
1996                 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1997                 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
1998                         beiscsi_log(phba, KERN_ERR,
1999                                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2000                                     "BM_%d : Cmd Notification %s[%d] on CID : %d\n",
2001                                     cqe_desc[code], code,  cid);
2002                         break;
2003                 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
2004                         beiscsi_log(phba, KERN_ERR,
2005                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2006                                     "BM_%d :  Dropping %s[%d] on DPDU ring on CID : %d\n",
2007                                     cqe_desc[code], code, cid);
2008                         spin_lock_bh(&phba->async_pdu_lock);
2009                         /* driver consumes the entry and drops the contents */
2010                         beiscsi_hdq_process_compl(beiscsi_conn,
2011                                                   (struct i_t_dpdu_cqe *)sol);
2012                         spin_unlock_bh(&phba->async_pdu_lock);
2013                         break;
2014                 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
2015                 case CXN_KILLED_BURST_LEN_MISMATCH:
2016                 case CXN_KILLED_AHS_RCVD:
2017                 case CXN_KILLED_UNKNOWN_HDR:
2018                 case CXN_KILLED_STALE_ITT_TTT_RCVD:
2019                 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
2020                 case CXN_KILLED_TIMED_OUT:
2021                 case CXN_KILLED_FIN_RCVD:
2022                 case CXN_KILLED_RST_SENT:
2023                 case CXN_KILLED_RST_RCVD:
2024                 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
2025                 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
2026                 case CXN_KILLED_OVER_RUN_RESIDUAL:
2027                 case CXN_KILLED_UNDER_RUN_RESIDUAL:
2028                 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
2029                         beiscsi_log(phba, KERN_ERR,
2030                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2031                                     "BM_%d : Event %s[%d] received on CID : %d\n",
2032                                     cqe_desc[code], code, cid);
2033                         if (beiscsi_conn)
2034                                 iscsi_conn_failure(beiscsi_conn->conn,
2035                                                    ISCSI_ERR_CONN_FAILED);
2036                         break;
2037                 default:
2038                         beiscsi_log(phba, KERN_ERR,
2039                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2040                                     "BM_%d : Invalid CQE Event Received Code : %d"
2041                                     "CID 0x%x...\n",
2042                                     code, cid);
2043                         break;
2044                 }
2045
2046 proc_next_cqe:
2047                 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
2048                 queue_tail_inc(cq);
2049                 sol = queue_tail_node(cq);
2050                 num_processed++;
2051                 if (total == budget)
2052                         break;
2053         }
2054
2055         hwi_ring_cq_db(phba, cq->id, num_processed, 1);
2056         return total;
2057 }
2058
2059 static int be_iopoll(struct irq_poll *iop, int budget)
2060 {
2061         unsigned int ret, io_events;
2062         struct beiscsi_hba *phba;
2063         struct be_eq_obj *pbe_eq;
2064         struct be_eq_entry *eqe = NULL;
2065         struct be_queue_info *eq;
2066
2067         pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
2068         phba = pbe_eq->phba;
2069         if (beiscsi_hba_in_error(phba)) {
2070                 irq_poll_complete(iop);
2071                 return 0;
2072         }
2073
2074         io_events = 0;
2075         eq = &pbe_eq->q;
2076         eqe = queue_tail_node(eq);
2077         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] &
2078                         EQE_VALID_MASK) {
2079                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
2080                 queue_tail_inc(eq);
2081                 eqe = queue_tail_node(eq);
2082                 io_events++;
2083         }
2084         hwi_ring_eq_db(phba, eq->id, 1, io_events, 0, 1);
2085
2086         ret = beiscsi_process_cq(pbe_eq, budget);
2087         pbe_eq->cq_count += ret;
2088         if (ret < budget) {
2089                 irq_poll_complete(iop);
2090                 beiscsi_log(phba, KERN_INFO,
2091                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2092                             "BM_%d : rearm pbe_eq->q.id =%d ret %d\n",
2093                             pbe_eq->q.id, ret);
2094                 if (!beiscsi_hba_in_error(phba))
2095                         hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
2096         }
2097         return ret;
2098 }
2099
2100 static void
2101 hwi_write_sgl_v2(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2102                   unsigned int num_sg, struct beiscsi_io_task *io_task)
2103 {
2104         struct iscsi_sge *psgl;
2105         unsigned int sg_len, index;
2106         unsigned int sge_len = 0;
2107         unsigned long long addr;
2108         struct scatterlist *l_sg;
2109         unsigned int offset;
2110
2111         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_lo, pwrb,
2112                       io_task->bhs_pa.u.a32.address_lo);
2113         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_hi, pwrb,
2114                       io_task->bhs_pa.u.a32.address_hi);
2115
2116         l_sg = sg;
2117         for (index = 0; (index < num_sg) && (index < 2); index++,
2118                         sg = sg_next(sg)) {
2119                 if (index == 0) {
2120                         sg_len = sg_dma_len(sg);
2121                         addr = (u64) sg_dma_address(sg);
2122                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2123                                       sge0_addr_lo, pwrb,
2124                                       lower_32_bits(addr));
2125                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2126                                       sge0_addr_hi, pwrb,
2127                                       upper_32_bits(addr));
2128                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2129                                       sge0_len, pwrb,
2130                                       sg_len);
2131                         sge_len = sg_len;
2132                 } else {
2133                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_r2t_offset,
2134                                       pwrb, sge_len);
2135                         sg_len = sg_dma_len(sg);
2136                         addr = (u64) sg_dma_address(sg);
2137                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2138                                       sge1_addr_lo, pwrb,
2139                                       lower_32_bits(addr));
2140                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2141                                       sge1_addr_hi, pwrb,
2142                                       upper_32_bits(addr));
2143                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2144                                       sge1_len, pwrb,
2145                                       sg_len);
2146                 }
2147         }
2148         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2149         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2150
2151         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2152
2153         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2154                       io_task->bhs_pa.u.a32.address_hi);
2155         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2156                       io_task->bhs_pa.u.a32.address_lo);
2157
2158         if (num_sg == 1) {
2159                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2160                               1);
2161                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2162                               0);
2163         } else if (num_sg == 2) {
2164                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2165                               0);
2166                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2167                               1);
2168         } else {
2169                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2170                               0);
2171                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2172                               0);
2173         }
2174
2175         sg = l_sg;
2176         psgl++;
2177         psgl++;
2178         offset = 0;
2179         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2180                 sg_len = sg_dma_len(sg);
2181                 addr = (u64) sg_dma_address(sg);
2182                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2183                               lower_32_bits(addr));
2184                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2185                               upper_32_bits(addr));
2186                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2187                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2188                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2189                 offset += sg_len;
2190         }
2191         psgl--;
2192         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2193 }
2194
2195 static void
2196 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2197               unsigned int num_sg, struct beiscsi_io_task *io_task)
2198 {
2199         struct iscsi_sge *psgl;
2200         unsigned int sg_len, index;
2201         unsigned int sge_len = 0;
2202         unsigned long long addr;
2203         struct scatterlist *l_sg;
2204         unsigned int offset;
2205
2206         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2207                                       io_task->bhs_pa.u.a32.address_lo);
2208         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2209                                       io_task->bhs_pa.u.a32.address_hi);
2210
2211         l_sg = sg;
2212         for (index = 0; (index < num_sg) && (index < 2); index++,
2213                                                          sg = sg_next(sg)) {
2214                 if (index == 0) {
2215                         sg_len = sg_dma_len(sg);
2216                         addr = (u64) sg_dma_address(sg);
2217                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2218                                                 ((u32)(addr & 0xFFFFFFFF)));
2219                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2220                                                         ((u32)(addr >> 32)));
2221                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2222                                                         sg_len);
2223                         sge_len = sg_len;
2224                 } else {
2225                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
2226                                                         pwrb, sge_len);
2227                         sg_len = sg_dma_len(sg);
2228                         addr = (u64) sg_dma_address(sg);
2229                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
2230                                                 ((u32)(addr & 0xFFFFFFFF)));
2231                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
2232                                                         ((u32)(addr >> 32)));
2233                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
2234                                                         sg_len);
2235                 }
2236         }
2237         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2238         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2239
2240         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2241
2242         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2243                         io_task->bhs_pa.u.a32.address_hi);
2244         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2245                         io_task->bhs_pa.u.a32.address_lo);
2246
2247         if (num_sg == 1) {
2248                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2249                                                                 1);
2250                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2251                                                                 0);
2252         } else if (num_sg == 2) {
2253                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2254                                                                 0);
2255                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2256                                                                 1);
2257         } else {
2258                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2259                                                                 0);
2260                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2261                                                                 0);
2262         }
2263         sg = l_sg;
2264         psgl++;
2265         psgl++;
2266         offset = 0;
2267         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2268                 sg_len = sg_dma_len(sg);
2269                 addr = (u64) sg_dma_address(sg);
2270                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2271                                                 (addr & 0xFFFFFFFF));
2272                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2273                                                 (addr >> 32));
2274                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2275                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2276                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2277                 offset += sg_len;
2278         }
2279         psgl--;
2280         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2281 }
2282
2283 /**
2284  * hwi_write_buffer()- Populate the WRB with task info
2285  * @pwrb: ptr to the WRB entry
2286  * @task: iscsi task which is to be executed
2287  **/
2288 static int hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
2289 {
2290         struct iscsi_sge *psgl;
2291         struct beiscsi_io_task *io_task = task->dd_data;
2292         struct beiscsi_conn *beiscsi_conn = io_task->conn;
2293         struct beiscsi_hba *phba = beiscsi_conn->phba;
2294         uint8_t dsp_value = 0;
2295
2296         io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
2297         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2298                                 io_task->bhs_pa.u.a32.address_lo);
2299         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2300                                 io_task->bhs_pa.u.a32.address_hi);
2301
2302         if (task->data) {
2303
2304                 /* Check for the data_count */
2305                 dsp_value = (task->data_count) ? 1 : 0;
2306
2307                 if (is_chip_be2_be3r(phba))
2308                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp,
2309                                       pwrb, dsp_value);
2310                 else
2311                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp,
2312                                       pwrb, dsp_value);
2313
2314                 /* Map addr only if there is data_count */
2315                 if (dsp_value) {
2316                         io_task->mtask_addr = pci_map_single(phba->pcidev,
2317                                                              task->data,
2318                                                              task->data_count,
2319                                                              PCI_DMA_TODEVICE);
2320                         if (pci_dma_mapping_error(phba->pcidev,
2321                                                   io_task->mtask_addr))
2322                                 return -ENOMEM;
2323                         io_task->mtask_data_count = task->data_count;
2324                 } else
2325                         io_task->mtask_addr = 0;
2326
2327                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2328                               lower_32_bits(io_task->mtask_addr));
2329                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2330                               upper_32_bits(io_task->mtask_addr));
2331                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2332                                                 task->data_count);
2333
2334                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
2335         } else {
2336                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2337                 io_task->mtask_addr = 0;
2338         }
2339
2340         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2341
2342         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
2343
2344         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2345                       io_task->bhs_pa.u.a32.address_hi);
2346         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2347                       io_task->bhs_pa.u.a32.address_lo);
2348         if (task->data) {
2349                 psgl++;
2350                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
2351                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
2352                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
2353                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
2354                 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
2355                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2356
2357                 psgl++;
2358                 if (task->data) {
2359                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2360                                       lower_32_bits(io_task->mtask_addr));
2361                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2362                                       upper_32_bits(io_task->mtask_addr));
2363                 }
2364                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
2365         }
2366         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2367         return 0;
2368 }
2369
2370 /**
2371  * beiscsi_find_mem_req()- Find mem needed
2372  * @phba: ptr to HBA struct
2373  **/
2374 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
2375 {
2376         uint8_t mem_descr_index, ulp_num;
2377         unsigned int num_cq_pages, num_async_pdu_buf_pages;
2378         unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
2379         unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
2380
2381         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2382                                       sizeof(struct sol_cqe));
2383
2384         phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
2385
2386         phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
2387                                                  BE_ISCSI_PDU_HEADER_SIZE;
2388         phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
2389                                             sizeof(struct hwi_context_memory);
2390
2391
2392         phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
2393             * (phba->params.wrbs_per_cxn)
2394             * phba->params.cxns_per_ctrl;
2395         wrb_sz_per_cxn =  sizeof(struct wrb_handle) *
2396                                  (phba->params.wrbs_per_cxn);
2397         phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
2398                                 phba->params.cxns_per_ctrl);
2399
2400         phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
2401                 phba->params.icds_per_ctrl;
2402         phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
2403                 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
2404         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
2405                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
2406
2407                         num_async_pdu_buf_sgl_pages =
2408                                 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT(
2409                                                phba, ulp_num) *
2410                                                sizeof(struct phys_addr));
2411
2412                         num_async_pdu_buf_pages =
2413                                 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT(
2414                                                phba, ulp_num) *
2415                                                phba->params.defpdu_hdr_sz);
2416
2417                         num_async_pdu_data_pages =
2418                                 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT(
2419                                                phba, ulp_num) *
2420                                                phba->params.defpdu_data_sz);
2421
2422                         num_async_pdu_data_sgl_pages =
2423                                 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT(
2424                                                phba, ulp_num) *
2425                                                sizeof(struct phys_addr));
2426
2427                         mem_descr_index = (HWI_MEM_TEMPLATE_HDR_ULP0 +
2428                                           (ulp_num * MEM_DESCR_OFFSET));
2429                         phba->mem_req[mem_descr_index] =
2430                                         BEISCSI_GET_CID_COUNT(phba, ulp_num) *
2431                                         BEISCSI_TEMPLATE_HDR_PER_CXN_SIZE;
2432
2433                         mem_descr_index = (HWI_MEM_ASYNC_HEADER_BUF_ULP0 +
2434                                           (ulp_num * MEM_DESCR_OFFSET));
2435                         phba->mem_req[mem_descr_index] =
2436                                           num_async_pdu_buf_pages *
2437                                           PAGE_SIZE;
2438
2439                         mem_descr_index = (HWI_MEM_ASYNC_DATA_BUF_ULP0 +
2440                                           (ulp_num * MEM_DESCR_OFFSET));
2441                         phba->mem_req[mem_descr_index] =
2442                                           num_async_pdu_data_pages *
2443                                           PAGE_SIZE;
2444
2445                         mem_descr_index = (HWI_MEM_ASYNC_HEADER_RING_ULP0 +
2446                                           (ulp_num * MEM_DESCR_OFFSET));
2447                         phba->mem_req[mem_descr_index] =
2448                                           num_async_pdu_buf_sgl_pages *
2449                                           PAGE_SIZE;
2450
2451                         mem_descr_index = (HWI_MEM_ASYNC_DATA_RING_ULP0 +
2452                                           (ulp_num * MEM_DESCR_OFFSET));
2453                         phba->mem_req[mem_descr_index] =
2454                                           num_async_pdu_data_sgl_pages *
2455                                           PAGE_SIZE;
2456
2457                         mem_descr_index = (HWI_MEM_ASYNC_HEADER_HANDLE_ULP0 +
2458                                           (ulp_num * MEM_DESCR_OFFSET));
2459                         phba->mem_req[mem_descr_index] =
2460                                           BEISCSI_GET_CID_COUNT(phba, ulp_num) *
2461                                           sizeof(struct hd_async_handle);
2462
2463                         mem_descr_index = (HWI_MEM_ASYNC_DATA_HANDLE_ULP0 +
2464                                           (ulp_num * MEM_DESCR_OFFSET));
2465                         phba->mem_req[mem_descr_index] =
2466                                           BEISCSI_GET_CID_COUNT(phba, ulp_num) *
2467                                           sizeof(struct hd_async_handle);
2468
2469                         mem_descr_index = (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0 +
2470                                           (ulp_num * MEM_DESCR_OFFSET));
2471                         phba->mem_req[mem_descr_index] =
2472                                           sizeof(struct hd_async_context) +
2473                                          (BEISCSI_GET_CID_COUNT(phba, ulp_num) *
2474                                           sizeof(struct hd_async_entry));
2475                 }
2476         }
2477 }
2478
2479 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
2480 {
2481         dma_addr_t bus_add;
2482         struct hwi_controller *phwi_ctrlr;
2483         struct be_mem_descriptor *mem_descr;
2484         struct mem_array *mem_arr, *mem_arr_orig;
2485         unsigned int i, j, alloc_size, curr_alloc_size;
2486
2487         phba->phwi_ctrlr = kzalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
2488         if (!phba->phwi_ctrlr)
2489                 return -ENOMEM;
2490
2491         /* Allocate memory for wrb_context */
2492         phwi_ctrlr = phba->phwi_ctrlr;
2493         phwi_ctrlr->wrb_context = kzalloc(sizeof(struct hwi_wrb_context) *
2494                                           phba->params.cxns_per_ctrl,
2495                                           GFP_KERNEL);
2496         if (!phwi_ctrlr->wrb_context) {
2497                 kfree(phba->phwi_ctrlr);
2498                 return -ENOMEM;
2499         }
2500
2501         phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
2502                                  GFP_KERNEL);
2503         if (!phba->init_mem) {
2504                 kfree(phwi_ctrlr->wrb_context);
2505                 kfree(phba->phwi_ctrlr);
2506                 return -ENOMEM;
2507         }
2508
2509         mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
2510                                GFP_KERNEL);
2511         if (!mem_arr_orig) {
2512                 kfree(phba->init_mem);
2513                 kfree(phwi_ctrlr->wrb_context);
2514                 kfree(phba->phwi_ctrlr);
2515                 return -ENOMEM;
2516         }
2517
2518         mem_descr = phba->init_mem;
2519         for (i = 0; i < SE_MEM_MAX; i++) {
2520                 if (!phba->mem_req[i]) {
2521                         mem_descr->mem_array = NULL;
2522                         mem_descr++;
2523                         continue;
2524                 }
2525
2526                 j = 0;
2527                 mem_arr = mem_arr_orig;
2528                 alloc_size = phba->mem_req[i];
2529                 memset(mem_arr, 0, sizeof(struct mem_array) *
2530                        BEISCSI_MAX_FRAGS_INIT);
2531                 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2532                 do {
2533                         mem_arr->virtual_address = pci_alloc_consistent(
2534                                                         phba->pcidev,
2535                                                         curr_alloc_size,
2536                                                         &bus_add);
2537                         if (!mem_arr->virtual_address) {
2538                                 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2539                                         goto free_mem;
2540                                 if (curr_alloc_size -
2541                                         rounddown_pow_of_two(curr_alloc_size))
2542                                         curr_alloc_size = rounddown_pow_of_two
2543                                                              (curr_alloc_size);
2544                                 else
2545                                         curr_alloc_size = curr_alloc_size / 2;
2546                         } else {
2547                                 mem_arr->bus_address.u.
2548                                     a64.address = (__u64) bus_add;
2549                                 mem_arr->size = curr_alloc_size;
2550                                 alloc_size -= curr_alloc_size;
2551                                 curr_alloc_size = min(be_max_phys_size *
2552                                                       1024, alloc_size);
2553                                 j++;
2554                                 mem_arr++;
2555                         }
2556                 } while (alloc_size);
2557                 mem_descr->num_elements = j;
2558                 mem_descr->size_in_bytes = phba->mem_req[i];
2559                 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
2560                                                GFP_KERNEL);
2561                 if (!mem_descr->mem_array)
2562                         goto free_mem;
2563
2564                 memcpy(mem_descr->mem_array, mem_arr_orig,
2565                        sizeof(struct mem_array) * j);
2566                 mem_descr++;
2567         }
2568         kfree(mem_arr_orig);
2569         return 0;
2570 free_mem:
2571         mem_descr->num_elements = j;
2572         while ((i) || (j)) {
2573                 for (j = mem_descr->num_elements; j > 0; j--) {
2574                         pci_free_consistent(phba->pcidev,
2575                                             mem_descr->mem_array[j - 1].size,
2576                                             mem_descr->mem_array[j - 1].
2577                                             virtual_address,
2578                                             (unsigned long)mem_descr->
2579                                             mem_array[j - 1].
2580                                             bus_address.u.a64.address);
2581                 }
2582                 if (i) {
2583                         i--;
2584                         kfree(mem_descr->mem_array);
2585                         mem_descr--;
2586                 }
2587         }
2588         kfree(mem_arr_orig);
2589         kfree(phba->init_mem);
2590         kfree(phba->phwi_ctrlr->wrb_context);
2591         kfree(phba->phwi_ctrlr);
2592         return -ENOMEM;
2593 }
2594
2595 static int beiscsi_get_memory(struct beiscsi_hba *phba)
2596 {
2597         beiscsi_find_mem_req(phba);
2598         return beiscsi_alloc_mem(phba);
2599 }
2600
2601 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2602 {
2603         struct pdu_data_out *pdata_out;
2604         struct pdu_nop_out *pnop_out;
2605         struct be_mem_descriptor *mem_descr;
2606
2607         mem_descr = phba->init_mem;
2608         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2609         pdata_out =
2610             (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2611         memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2612
2613         AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2614                       IIOC_SCSI_DATA);
2615
2616         pnop_out =
2617             (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2618                                    virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2619
2620         memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2621         AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2622         AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2623         AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2624 }
2625
2626 static int beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
2627 {
2628         struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
2629         struct hwi_context_memory *phwi_ctxt;
2630         struct wrb_handle *pwrb_handle = NULL;
2631         struct hwi_controller *phwi_ctrlr;
2632         struct hwi_wrb_context *pwrb_context;
2633         struct iscsi_wrb *pwrb = NULL;
2634         unsigned int num_cxn_wrbh = 0;
2635         unsigned int num_cxn_wrb = 0, j, idx = 0, index;
2636
2637         mem_descr_wrbh = phba->init_mem;
2638         mem_descr_wrbh += HWI_MEM_WRBH;
2639
2640         mem_descr_wrb = phba->init_mem;
2641         mem_descr_wrb += HWI_MEM_WRB;
2642         phwi_ctrlr = phba->phwi_ctrlr;
2643
2644         /* Allocate memory for WRBQ */
2645         phwi_ctxt = phwi_ctrlr->phwi_ctxt;
2646         phwi_ctxt->be_wrbq = kzalloc(sizeof(struct be_queue_info) *
2647                                      phba->params.cxns_per_ctrl,
2648                                      GFP_KERNEL);
2649         if (!phwi_ctxt->be_wrbq) {
2650                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2651                             "BM_%d : WRBQ Mem Alloc Failed\n");
2652                 return -ENOMEM;
2653         }
2654
2655         for (index = 0; index < phba->params.cxns_per_ctrl; index++) {
2656                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2657                 pwrb_context->pwrb_handle_base =
2658                                 kzalloc(sizeof(struct wrb_handle *) *
2659                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2660                 if (!pwrb_context->pwrb_handle_base) {
2661                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2662                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
2663                         goto init_wrb_hndl_failed;
2664                 }
2665                 pwrb_context->pwrb_handle_basestd =
2666                                 kzalloc(sizeof(struct wrb_handle *) *
2667                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2668                 if (!pwrb_context->pwrb_handle_basestd) {
2669                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2670                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
2671                         goto init_wrb_hndl_failed;
2672                 }
2673                 if (!num_cxn_wrbh) {
2674                         pwrb_handle =
2675                                 mem_descr_wrbh->mem_array[idx].virtual_address;
2676                         num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2677                                         ((sizeof(struct wrb_handle)) *
2678                                          phba->params.wrbs_per_cxn));
2679                         idx++;
2680                 }
2681                 pwrb_context->alloc_index = 0;
2682                 pwrb_context->wrb_handles_available = 0;
2683                 pwrb_context->free_index = 0;
2684
2685                 if (num_cxn_wrbh) {
2686                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2687                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2688                                 pwrb_context->pwrb_handle_basestd[j] =
2689                                                                 pwrb_handle;
2690                                 pwrb_context->wrb_handles_available++;
2691                                 pwrb_handle->wrb_index = j;
2692                                 pwrb_handle++;
2693                         }
2694                         num_cxn_wrbh--;
2695                 }
2696                 spin_lock_init(&pwrb_context->wrb_lock);
2697         }
2698         idx = 0;
2699         for (index = 0; index < phba->params.cxns_per_ctrl; index++) {
2700                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2701                 if (!num_cxn_wrb) {
2702                         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2703                         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2704                                 ((sizeof(struct iscsi_wrb) *
2705                                   phba->params.wrbs_per_cxn));
2706                         idx++;
2707                 }
2708
2709                 if (num_cxn_wrb) {
2710                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2711                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2712                                 pwrb_handle->pwrb = pwrb;
2713                                 pwrb++;
2714                         }
2715                         num_cxn_wrb--;
2716                 }
2717         }
2718         return 0;
2719 init_wrb_hndl_failed:
2720         for (j = index; j > 0; j--) {
2721                 pwrb_context = &phwi_ctrlr->wrb_context[j];
2722                 kfree(pwrb_context->pwrb_handle_base);
2723                 kfree(pwrb_context->pwrb_handle_basestd);
2724         }
2725         return -ENOMEM;
2726 }
2727
2728 static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2729 {
2730         uint8_t ulp_num;
2731         struct hwi_controller *phwi_ctrlr;
2732         struct hba_parameters *p = &phba->params;
2733         struct hd_async_context *pasync_ctx;
2734         struct hd_async_handle *pasync_header_h, *pasync_data_h;
2735         unsigned int index, idx, num_per_mem, num_async_data;
2736         struct be_mem_descriptor *mem_descr;
2737
2738         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
2739                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
2740                          /* get async_ctx for each ULP */
2741                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2742                         mem_descr += (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0 +
2743                                      (ulp_num * MEM_DESCR_OFFSET));
2744
2745                         phwi_ctrlr = phba->phwi_ctrlr;
2746                         phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num] =
2747                                 (struct hd_async_context *)
2748                                  mem_descr->mem_array[0].virtual_address;
2749
2750                         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num];
2751                         memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2752
2753                         pasync_ctx->async_entry =
2754                                         (struct hd_async_entry *)
2755                                         ((long unsigned int)pasync_ctx +
2756                                         sizeof(struct hd_async_context));
2757
2758                         pasync_ctx->num_entries = BEISCSI_GET_CID_COUNT(phba,
2759                                                   ulp_num);
2760                         /* setup header buffers */
2761                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2762                         mem_descr += HWI_MEM_ASYNC_HEADER_BUF_ULP0 +
2763                                 (ulp_num * MEM_DESCR_OFFSET);
2764                         if (mem_descr->mem_array[0].virtual_address) {
2765                                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2766                                             "BM_%d : hwi_init_async_pdu_ctx"
2767                                             " HWI_MEM_ASYNC_HEADER_BUF_ULP%d va=%p\n",
2768                                             ulp_num,
2769                                             mem_descr->mem_array[0].
2770                                             virtual_address);
2771                         } else
2772                                 beiscsi_log(phba, KERN_WARNING,
2773                                             BEISCSI_LOG_INIT,
2774                                             "BM_%d : No Virtual address for ULP : %d\n",
2775                                             ulp_num);
2776
2777                         pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz;
2778                         pasync_ctx->async_header.va_base =
2779                                 mem_descr->mem_array[0].virtual_address;
2780
2781                         pasync_ctx->async_header.pa_base.u.a64.address =
2782                                 mem_descr->mem_array[0].
2783                                 bus_address.u.a64.address;
2784
2785                         /* setup header buffer sgls */
2786                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2787                         mem_descr += HWI_MEM_ASYNC_HEADER_RING_ULP0 +
2788                                      (ulp_num * MEM_DESCR_OFFSET);
2789                         if (mem_descr->mem_array[0].virtual_address) {
2790                                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2791                                             "BM_%d : hwi_init_async_pdu_ctx"
2792                                             " HWI_MEM_ASYNC_HEADER_RING_ULP%d va=%p\n",
2793                                             ulp_num,
2794                                             mem_descr->mem_array[0].
2795                                             virtual_address);
2796                         } else
2797                                 beiscsi_log(phba, KERN_WARNING,
2798                                             BEISCSI_LOG_INIT,
2799                                             "BM_%d : No Virtual address for ULP : %d\n",
2800                                             ulp_num);
2801
2802                         pasync_ctx->async_header.ring_base =
2803                                 mem_descr->mem_array[0].virtual_address;
2804
2805                         /* setup header buffer handles */
2806                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2807                         mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE_ULP0 +
2808                                      (ulp_num * MEM_DESCR_OFFSET);
2809                         if (mem_descr->mem_array[0].virtual_address) {
2810                                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2811                                             "BM_%d : hwi_init_async_pdu_ctx"
2812                                             " HWI_MEM_ASYNC_HEADER_HANDLE_ULP%d va=%p\n",
2813                                             ulp_num,
2814                                             mem_descr->mem_array[0].
2815                                             virtual_address);
2816                         } else
2817                                 beiscsi_log(phba, KERN_WARNING,
2818                                             BEISCSI_LOG_INIT,
2819                                             "BM_%d : No Virtual address for ULP : %d\n",
2820                                             ulp_num);
2821
2822                         pasync_ctx->async_header.handle_base =
2823                                 mem_descr->mem_array[0].virtual_address;
2824                         INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2825
2826                         /* setup data buffer sgls */
2827                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2828                         mem_descr += HWI_MEM_ASYNC_DATA_RING_ULP0 +
2829                                      (ulp_num * MEM_DESCR_OFFSET);
2830                         if (mem_descr->mem_array[0].virtual_address) {
2831                                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2832                                             "BM_%d : hwi_init_async_pdu_ctx"
2833                                             " HWI_MEM_ASYNC_DATA_RING_ULP%d va=%p\n",
2834                                             ulp_num,
2835                                             mem_descr->mem_array[0].
2836                                             virtual_address);
2837                         } else
2838                                 beiscsi_log(phba, KERN_WARNING,
2839                                             BEISCSI_LOG_INIT,
2840                                             "BM_%d : No Virtual address for ULP : %d\n",
2841                                             ulp_num);
2842
2843                         pasync_ctx->async_data.ring_base =
2844                                 mem_descr->mem_array[0].virtual_address;
2845
2846                         /* setup data buffer handles */
2847                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2848                         mem_descr += HWI_MEM_ASYNC_DATA_HANDLE_ULP0 +
2849                                      (ulp_num * MEM_DESCR_OFFSET);
2850                         if (!mem_descr->mem_array[0].virtual_address)
2851                                 beiscsi_log(phba, KERN_WARNING,
2852                                             BEISCSI_LOG_INIT,
2853                                             "BM_%d : No Virtual address for ULP : %d\n",
2854                                             ulp_num);
2855
2856                         pasync_ctx->async_data.handle_base =
2857                                 mem_descr->mem_array[0].virtual_address;
2858                         INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2859
2860                         pasync_header_h =
2861                                 (struct hd_async_handle *)
2862                                 pasync_ctx->async_header.handle_base;
2863                         pasync_data_h =
2864                                 (struct hd_async_handle *)
2865                                 pasync_ctx->async_data.handle_base;
2866
2867                         /* setup data buffers */
2868                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2869                         mem_descr += HWI_MEM_ASYNC_DATA_BUF_ULP0 +
2870                                      (ulp_num * MEM_DESCR_OFFSET);
2871                         if (mem_descr->mem_array[0].virtual_address) {
2872                                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2873                                             "BM_%d : hwi_init_async_pdu_ctx"
2874                                             " HWI_MEM_ASYNC_DATA_BUF_ULP%d va=%p\n",
2875                                             ulp_num,
2876                                             mem_descr->mem_array[0].
2877                                             virtual_address);
2878                         } else
2879                                 beiscsi_log(phba, KERN_WARNING,
2880                                             BEISCSI_LOG_INIT,
2881                                             "BM_%d : No Virtual address for ULP : %d\n",
2882                                             ulp_num);
2883
2884                         idx = 0;
2885                         pasync_ctx->async_data.buffer_size = p->defpdu_data_sz;
2886                         pasync_ctx->async_data.va_base =
2887                                 mem_descr->mem_array[idx].virtual_address;
2888                         pasync_ctx->async_data.pa_base.u.a64.address =
2889                                 mem_descr->mem_array[idx].
2890                                 bus_address.u.a64.address;
2891
2892                         num_async_data = ((mem_descr->mem_array[idx].size) /
2893                                         phba->params.defpdu_data_sz);
2894                         num_per_mem = 0;
2895
2896                         for (index = 0; index < BEISCSI_GET_CID_COUNT
2897                                         (phba, ulp_num); index++) {
2898                                 pasync_header_h->cri = -1;
2899                                 pasync_header_h->is_header = 1;
2900                                 pasync_header_h->index = index;
2901                                 INIT_LIST_HEAD(&pasync_header_h->link);
2902                                 pasync_header_h->pbuffer =
2903                                         (void *)((unsigned long)
2904                                                  (pasync_ctx->
2905                                                   async_header.va_base) +
2906                                                  (p->defpdu_hdr_sz * index));
2907
2908                                 pasync_header_h->pa.u.a64.address =
2909                                         pasync_ctx->async_header.pa_base.u.a64.
2910                                         address + (p->defpdu_hdr_sz * index);
2911
2912                                 list_add_tail(&pasync_header_h->link,
2913                                               &pasync_ctx->async_header.
2914                                               free_list);
2915                                 pasync_header_h++;
2916                                 pasync_ctx->async_header.free_entries++;
2917                                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2918                                                 wq.list);
2919                                 pasync_ctx->async_entry[index].header = NULL;
2920
2921                                 pasync_data_h->cri = -1;
2922                                 pasync_data_h->is_header = 0;
2923                                 pasync_data_h->index = index;
2924                                 INIT_LIST_HEAD(&pasync_data_h->link);
2925
2926                                 if (!num_async_data) {
2927                                         num_per_mem = 0;
2928                                         idx++;
2929                                         pasync_ctx->async_data.va_base =
2930                                                 mem_descr->mem_array[idx].
2931                                                 virtual_address;
2932                                         pasync_ctx->async_data.pa_base.u.
2933                                                 a64.address =
2934                                                 mem_descr->mem_array[idx].
2935                                                 bus_address.u.a64.address;
2936                                         num_async_data =
2937                                                 ((mem_descr->mem_array[idx].
2938                                                   size) /
2939                                                  phba->params.defpdu_data_sz);
2940                                 }
2941                                 pasync_data_h->pbuffer =
2942                                         (void *)((unsigned long)
2943                                         (pasync_ctx->async_data.va_base) +
2944                                         (p->defpdu_data_sz * num_per_mem));
2945
2946                                 pasync_data_h->pa.u.a64.address =
2947                                         pasync_ctx->async_data.pa_base.u.a64.
2948                                         address + (p->defpdu_data_sz *
2949                                         num_per_mem);
2950                                 num_per_mem++;
2951                                 num_async_data--;
2952
2953                                 list_add_tail(&pasync_data_h->link,
2954                                               &pasync_ctx->async_data.
2955                                               free_list);
2956                                 pasync_data_h++;
2957                                 pasync_ctx->async_data.free_entries++;
2958                                 pasync_ctx->async_entry[index].data = NULL;
2959                         }
2960                 }
2961         }
2962
2963         return 0;
2964 }
2965
2966 static int
2967 be_sgl_create_contiguous(void *virtual_address,
2968                          u64 physical_address, u32 length,
2969                          struct be_dma_mem *sgl)
2970 {
2971         WARN_ON(!virtual_address);
2972         WARN_ON(!physical_address);
2973         WARN_ON(!length);
2974         WARN_ON(!sgl);
2975
2976         sgl->va = virtual_address;
2977         sgl->dma = (unsigned long)physical_address;
2978         sgl->size = length;
2979
2980         return 0;
2981 }
2982
2983 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2984 {
2985         memset(sgl, 0, sizeof(*sgl));
2986 }
2987
2988 static void
2989 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2990                      struct mem_array *pmem, struct be_dma_mem *sgl)
2991 {
2992         if (sgl->va)
2993                 be_sgl_destroy_contiguous(sgl);
2994
2995         be_sgl_create_contiguous(pmem->virtual_address,
2996                                  pmem->bus_address.u.a64.address,
2997                                  pmem->size, sgl);
2998 }
2999
3000 static void
3001 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
3002                            struct mem_array *pmem, struct be_dma_mem *sgl)
3003 {
3004         if (sgl->va)
3005                 be_sgl_destroy_contiguous(sgl);
3006
3007         be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
3008                                  pmem->bus_address.u.a64.address,
3009                                  pmem->size, sgl);
3010 }
3011
3012 static int be_fill_queue(struct be_queue_info *q,
3013                 u16 len, u16 entry_size, void *vaddress)
3014 {
3015         struct be_dma_mem *mem = &q->dma_mem;
3016
3017         memset(q, 0, sizeof(*q));
3018         q->len = len;
3019         q->entry_size = entry_size;
3020         mem->size = len * entry_size;
3021         mem->va = vaddress;
3022         if (!mem->va)
3023                 return -ENOMEM;
3024         memset(mem->va, 0, mem->size);
3025         return 0;
3026 }
3027
3028 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
3029                              struct hwi_context_memory *phwi_context)
3030 {
3031         int ret = -ENOMEM, eq_for_mcc;
3032         unsigned int i, num_eq_pages;
3033         struct be_queue_info *eq;
3034         struct be_dma_mem *mem;
3035         void *eq_vaddress;
3036         dma_addr_t paddr;
3037
3038         num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
3039                                       sizeof(struct be_eq_entry));
3040
3041         if (phba->msix_enabled)
3042                 eq_for_mcc = 1;
3043         else
3044                 eq_for_mcc = 0;
3045         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
3046                 eq = &phwi_context->be_eq[i].q;
3047                 mem = &eq->dma_mem;
3048                 phwi_context->be_eq[i].phba = phba;
3049                 eq_vaddress = pci_alloc_consistent(phba->pcidev,
3050                                                    num_eq_pages * PAGE_SIZE,
3051                                                    &paddr);
3052                 if (!eq_vaddress)
3053                         goto create_eq_error;
3054
3055                 mem->va = eq_vaddress;
3056                 ret = be_fill_queue(eq, phba->params.num_eq_entries,
3057                                     sizeof(struct be_eq_entry), eq_vaddress);
3058                 if (ret) {
3059                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3060                                     "BM_%d : be_fill_queue Failed for EQ\n");
3061                         goto create_eq_error;
3062                 }
3063
3064                 mem->dma = paddr;
3065                 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
3066                                             phwi_context->cur_eqd);
3067                 if (ret) {
3068                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3069                                     "BM_%d : beiscsi_cmd_eq_create"
3070                                     "Failed for EQ\n");
3071                         goto create_eq_error;
3072                 }
3073
3074                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3075                             "BM_%d : eqid = %d\n",
3076                             phwi_context->be_eq[i].q.id);
3077         }
3078         return 0;
3079
3080 create_eq_error:
3081         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
3082                 eq = &phwi_context->be_eq[i].q;
3083                 mem = &eq->dma_mem;
3084                 if (mem->va)
3085                         pci_free_consistent(phba->pcidev, num_eq_pages
3086                                             * PAGE_SIZE,
3087                                             mem->va, mem->dma);
3088         }
3089         return ret;
3090 }
3091
3092 static int beiscsi_create_cqs(struct beiscsi_hba *phba,
3093                              struct hwi_context_memory *phwi_context)
3094 {
3095         unsigned int i, num_cq_pages;
3096         struct be_queue_info *cq, *eq;
3097         struct be_dma_mem *mem;
3098         struct be_eq_obj *pbe_eq;
3099         void *cq_vaddress;
3100         int ret = -ENOMEM;
3101         dma_addr_t paddr;
3102
3103         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
3104                                       sizeof(struct sol_cqe));
3105
3106         for (i = 0; i < phba->num_cpus; i++) {
3107                 cq = &phwi_context->be_cq[i];
3108                 eq = &phwi_context->be_eq[i].q;
3109                 pbe_eq = &phwi_context->be_eq[i];
3110                 pbe_eq->cq = cq;
3111                 pbe_eq->phba = phba;
3112                 mem = &cq->dma_mem;
3113                 cq_vaddress = pci_alloc_consistent(phba->pcidev,
3114                                                    num_cq_pages * PAGE_SIZE,
3115                                                    &paddr);
3116                 if (!cq_vaddress)
3117                         goto create_cq_error;
3118
3119                 ret = be_fill_queue(cq, phba->params.num_cq_entries,
3120                                     sizeof(struct sol_cqe), cq_vaddress);
3121                 if (ret) {
3122                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3123                                     "BM_%d : be_fill_queue Failed "
3124                                     "for ISCSI CQ\n");
3125                         goto create_cq_error;
3126                 }
3127
3128                 mem->dma = paddr;
3129                 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
3130                                             false, 0);
3131                 if (ret) {
3132                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3133                                     "BM_%d : beiscsi_cmd_eq_create"
3134                                     "Failed for ISCSI CQ\n");
3135                         goto create_cq_error;
3136                 }
3137                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3138                             "BM_%d : iscsi cq_id is %d for eq_id %d\n"
3139                             "iSCSI CQ CREATED\n", cq->id, eq->id);
3140         }
3141         return 0;
3142
3143 create_cq_error:
3144         for (i = 0; i < phba->num_cpus; i++) {
3145                 cq = &phwi_context->be_cq[i];
3146                 mem = &cq->dma_mem;
3147                 if (mem->va)
3148                         pci_free_consistent(phba->pcidev, num_cq_pages
3149                                             * PAGE_SIZE,
3150                                             mem->va, mem->dma);
3151         }
3152         return ret;
3153 }
3154
3155 static int
3156 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
3157                        struct hwi_context_memory *phwi_context,
3158                        struct hwi_controller *phwi_ctrlr,
3159                        unsigned int def_pdu_ring_sz, uint8_t ulp_num)
3160 {
3161         unsigned int idx;
3162         int ret;
3163         struct be_queue_info *dq, *cq;
3164         struct be_dma_mem *mem;
3165         struct be_mem_descriptor *mem_descr;
3166         void *dq_vaddress;
3167
3168         idx = 0;
3169         dq = &phwi_context->be_def_hdrq[ulp_num];
3170         cq = &phwi_context->be_cq[0];
3171         mem = &dq->dma_mem;
3172         mem_descr = phba->init_mem;
3173         mem_descr += HWI_MEM_ASYNC_HEADER_RING_ULP0 +
3174                     (ulp_num * MEM_DESCR_OFFSET);
3175         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
3176         ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
3177                             sizeof(struct phys_addr),
3178                             sizeof(struct phys_addr), dq_vaddress);
3179         if (ret) {
3180                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3181                             "BM_%d : be_fill_queue Failed for DEF PDU HDR on ULP : %d\n",
3182                             ulp_num);
3183
3184                 return ret;
3185         }
3186         mem->dma = (unsigned long)mem_descr->mem_array[idx].
3187                                   bus_address.u.a64.address;
3188         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
3189                                               def_pdu_ring_sz,
3190                                               phba->params.defpdu_hdr_sz,
3191                                               BEISCSI_DEFQ_HDR, ulp_num);
3192         if (ret) {
3193                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3194                             "BM_%d : be_cmd_create_default_pdu_queue Failed DEFHDR on ULP : %d\n",
3195                             ulp_num);
3196
3197                 return ret;
3198         }
3199
3200         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3201                     "BM_%d : iscsi hdr def pdu id for ULP : %d is %d\n",
3202                     ulp_num,
3203                     phwi_context->be_def_hdrq[ulp_num].id);
3204         return 0;
3205 }
3206
3207 static int
3208 beiscsi_create_def_data(struct beiscsi_hba *phba,
3209                         struct hwi_context_memory *phwi_context,
3210                         struct hwi_controller *phwi_ctrlr,
3211                         unsigned int def_pdu_ring_sz, uint8_t ulp_num)
3212 {
3213         unsigned int idx;
3214         int ret;
3215         struct be_queue_info *dataq, *cq;
3216         struct be_dma_mem *mem;
3217         struct be_mem_descriptor *mem_descr;
3218         void *dq_vaddress;
3219
3220         idx = 0;
3221         dataq = &phwi_context->be_def_dataq[ulp_num];
3222         cq = &phwi_context->be_cq[0];
3223         mem = &dataq->dma_mem;
3224         mem_descr = phba->init_mem;
3225         mem_descr += HWI_MEM_ASYNC_DATA_RING_ULP0 +
3226                     (ulp_num * MEM_DESCR_OFFSET);
3227         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
3228         ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
3229                             sizeof(struct phys_addr),
3230                             sizeof(struct phys_addr), dq_vaddress);
3231         if (ret) {
3232                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3233                             "BM_%d : be_fill_queue Failed for DEF PDU "
3234                             "DATA on ULP : %d\n",
3235                             ulp_num);
3236
3237                 return ret;
3238         }
3239         mem->dma = (unsigned long)mem_descr->mem_array[idx].
3240                                   bus_address.u.a64.address;
3241         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
3242                                               def_pdu_ring_sz,
3243                                               phba->params.defpdu_data_sz,
3244                                               BEISCSI_DEFQ_DATA, ulp_num);
3245         if (ret) {
3246                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3247                             "BM_%d be_cmd_create_default_pdu_queue"
3248                             " Failed for DEF PDU DATA on ULP : %d\n",
3249                             ulp_num);
3250                 return ret;
3251         }
3252
3253         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3254                     "BM_%d : iscsi def data id on ULP : %d is  %d\n",
3255                     ulp_num,
3256                     phwi_context->be_def_dataq[ulp_num].id);
3257
3258         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3259                     "BM_%d : DEFAULT PDU DATA RING CREATED"
3260                     "on ULP : %d\n", ulp_num);
3261         return 0;
3262 }
3263
3264
3265 static int
3266 beiscsi_post_template_hdr(struct beiscsi_hba *phba)
3267 {
3268         struct be_mem_descriptor *mem_descr;
3269         struct mem_array *pm_arr;
3270         struct be_dma_mem sgl;
3271         int status, ulp_num;
3272
3273         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3274                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3275                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
3276                         mem_descr += HWI_MEM_TEMPLATE_HDR_ULP0 +
3277                                     (ulp_num * MEM_DESCR_OFFSET);
3278                         pm_arr = mem_descr->mem_array;
3279
3280                         hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
3281                         status = be_cmd_iscsi_post_template_hdr(
3282                                  &phba->ctrl, &sgl);
3283
3284                         if (status != 0) {
3285                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3286                                             "BM_%d : Post Template HDR Failed for"
3287                                             "ULP_%d\n", ulp_num);
3288                                 return status;
3289                         }
3290
3291                         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3292                                     "BM_%d : Template HDR Pages Posted for"
3293                                     "ULP_%d\n", ulp_num);
3294                 }
3295         }
3296         return 0;
3297 }
3298
3299 static int
3300 beiscsi_post_pages(struct beiscsi_hba *phba)
3301 {
3302         struct be_mem_descriptor *mem_descr;
3303         struct mem_array *pm_arr;
3304         unsigned int page_offset, i;
3305         struct be_dma_mem sgl;
3306         int status, ulp_num = 0;
3307
3308         mem_descr = phba->init_mem;
3309         mem_descr += HWI_MEM_SGE;
3310         pm_arr = mem_descr->mem_array;
3311
3312         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
3313                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported))
3314                         break;
3315
3316         page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
3317                         phba->fw_config.iscsi_icd_start[ulp_num]) / PAGE_SIZE;
3318         for (i = 0; i < mem_descr->num_elements; i++) {
3319                 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
3320                 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
3321                                                 page_offset,
3322                                                 (pm_arr->size / PAGE_SIZE));
3323                 page_offset += pm_arr->size / PAGE_SIZE;
3324                 if (status != 0) {
3325                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3326                                     "BM_%d : post sgl failed.\n");
3327                         return status;
3328                 }
3329                 pm_arr++;
3330         }
3331         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3332                     "BM_%d : POSTED PAGES\n");
3333         return 0;
3334 }
3335
3336 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
3337 {
3338         struct be_dma_mem *mem = &q->dma_mem;
3339         if (mem->va) {
3340                 pci_free_consistent(phba->pcidev, mem->size,
3341                         mem->va, mem->dma);
3342                 mem->va = NULL;
3343         }
3344 }
3345
3346 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
3347                 u16 len, u16 entry_size)
3348 {
3349         struct be_dma_mem *mem = &q->dma_mem;
3350
3351         memset(q, 0, sizeof(*q));
3352         q->len = len;
3353         q->entry_size = entry_size;
3354         mem->size = len * entry_size;
3355         mem->va = pci_zalloc_consistent(phba->pcidev, mem->size, &mem->dma);
3356         if (!mem->va)
3357                 return -ENOMEM;
3358         return 0;
3359 }
3360
3361 static int
3362 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
3363                          struct hwi_context_memory *phwi_context,
3364                          struct hwi_controller *phwi_ctrlr)
3365 {
3366         unsigned int wrb_mem_index, offset, size, num_wrb_rings;
3367         u64 pa_addr_lo;
3368         unsigned int idx, num, i, ulp_num;
3369         struct mem_array *pwrb_arr;
3370         void *wrb_vaddr;
3371         struct be_dma_mem sgl;
3372         struct be_mem_descriptor *mem_descr;
3373         struct hwi_wrb_context *pwrb_context;
3374         int status;
3375         uint8_t ulp_count = 0, ulp_base_num = 0;
3376         uint16_t cid_count_ulp[BEISCSI_ULP_COUNT] = { 0 };
3377
3378         idx = 0;
3379         mem_descr = phba->init_mem;
3380         mem_descr += HWI_MEM_WRB;
3381         pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
3382                            GFP_KERNEL);
3383         if (!pwrb_arr) {
3384                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3385                             "BM_%d : Memory alloc failed in create wrb ring.\n");
3386                 return -ENOMEM;
3387         }
3388         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3389         pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
3390         num_wrb_rings = mem_descr->mem_array[idx].size /
3391                 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
3392
3393         for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
3394                 if (num_wrb_rings) {
3395                         pwrb_arr[num].virtual_address = wrb_vaddr;
3396                         pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
3397                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3398                                             sizeof(struct iscsi_wrb);
3399                         wrb_vaddr += pwrb_arr[num].size;
3400                         pa_addr_lo += pwrb_arr[num].size;
3401                         num_wrb_rings--;
3402                 } else {
3403                         idx++;
3404                         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3405                         pa_addr_lo = mem_descr->mem_array[idx].\
3406                                         bus_address.u.a64.address;
3407                         num_wrb_rings = mem_descr->mem_array[idx].size /
3408                                         (phba->params.wrbs_per_cxn *
3409                                         sizeof(struct iscsi_wrb));
3410                         pwrb_arr[num].virtual_address = wrb_vaddr;
3411                         pwrb_arr[num].bus_address.u.a64.address\
3412                                                 = pa_addr_lo;
3413                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3414                                                  sizeof(struct iscsi_wrb);
3415                         wrb_vaddr += pwrb_arr[num].size;
3416                         pa_addr_lo   += pwrb_arr[num].size;
3417                         num_wrb_rings--;
3418                 }
3419         }
3420
3421         /* Get the ULP Count */
3422         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
3423                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3424                         ulp_count++;
3425                         ulp_base_num = ulp_num;
3426                         cid_count_ulp[ulp_num] =
3427                                 BEISCSI_GET_CID_COUNT(phba, ulp_num);
3428                 }
3429
3430         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3431                 wrb_mem_index = 0;
3432                 offset = 0;
3433                 size = 0;
3434
3435                 if (ulp_count > 1) {
3436                         ulp_base_num = (ulp_base_num + 1) % BEISCSI_ULP_COUNT;
3437
3438                         if (!cid_count_ulp[ulp_base_num])
3439                                 ulp_base_num = (ulp_base_num + 1) %
3440                                                 BEISCSI_ULP_COUNT;
3441
3442                         cid_count_ulp[ulp_base_num]--;
3443                 }
3444
3445
3446                 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
3447                 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
3448                                             &phwi_context->be_wrbq[i],
3449                                             &phwi_ctrlr->wrb_context[i],
3450                                             ulp_base_num);
3451                 if (status != 0) {
3452                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3453                                     "BM_%d : wrbq create failed.");
3454                         kfree(pwrb_arr);
3455                         return status;
3456                 }
3457                 pwrb_context = &phwi_ctrlr->wrb_context[i];
3458                 BE_SET_CID_TO_CRI(i, pwrb_context->cid);
3459         }
3460         kfree(pwrb_arr);
3461         return 0;
3462 }
3463
3464 static void free_wrb_handles(struct beiscsi_hba *phba)
3465 {
3466         unsigned int index;
3467         struct hwi_controller *phwi_ctrlr;
3468         struct hwi_wrb_context *pwrb_context;
3469
3470         phwi_ctrlr = phba->phwi_ctrlr;
3471         for (index = 0; index < phba->params.cxns_per_ctrl; index++) {
3472                 pwrb_context = &phwi_ctrlr->wrb_context[index];
3473                 kfree(pwrb_context->pwrb_handle_base);
3474                 kfree(pwrb_context->pwrb_handle_basestd);
3475         }
3476 }
3477
3478 static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
3479 {
3480         struct be_ctrl_info *ctrl = &phba->ctrl;
3481         struct be_dma_mem *ptag_mem;
3482         struct be_queue_info *q;
3483         int i, tag;
3484
3485         q = &phba->ctrl.mcc_obj.q;
3486         for (i = 0; i < MAX_MCC_CMD; i++) {
3487                 tag = i + 1;
3488                 if (!test_bit(MCC_TAG_STATE_RUNNING,
3489                               &ctrl->ptag_state[tag].tag_state))
3490                         continue;
3491
3492                 if (test_bit(MCC_TAG_STATE_TIMEOUT,
3493                              &ctrl->ptag_state[tag].tag_state)) {
3494                         ptag_mem = &ctrl->ptag_state[tag].tag_mem_state;
3495                         if (ptag_mem->size) {
3496                                 pci_free_consistent(ctrl->pdev,
3497                                                     ptag_mem->size,
3498                                                     ptag_mem->va,
3499                                                     ptag_mem->dma);
3500                                 ptag_mem->size = 0;
3501                         }
3502                         continue;
3503                 }
3504                 /**
3505                  * If MCC is still active and waiting then wake up the process.
3506                  * We are here only because port is going offline. The process
3507                  * sees that (BEISCSI_HBA_ONLINE is cleared) and EIO error is
3508                  * returned for the operation and allocated memory cleaned up.
3509                  */
3510                 if (waitqueue_active(&ctrl->mcc_wait[tag])) {
3511                         ctrl->mcc_tag_status[tag] = MCC_STATUS_FAILED;
3512                         ctrl->mcc_tag_status[tag] |= CQE_VALID_MASK;
3513                         wake_up_interruptible(&ctrl->mcc_wait[tag]);
3514                         /*
3515                          * Control tag info gets reinitialized in enable
3516                          * so wait for the process to clear running state.
3517                          */
3518                         while (test_bit(MCC_TAG_STATE_RUNNING,
3519                                         &ctrl->ptag_state[tag].tag_state))
3520                                 schedule_timeout_uninterruptible(HZ);
3521                 }
3522                 /**
3523                  * For MCC with tag_states MCC_TAG_STATE_ASYNC and
3524                  * MCC_TAG_STATE_IGNORE nothing needs to done.
3525                  */
3526         }
3527         if (q->created) {
3528                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
3529                 be_queue_free(phba, q);
3530         }
3531
3532         q = &phba->ctrl.mcc_obj.cq;
3533         if (q->created) {
3534                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3535                 be_queue_free(phba, q);
3536         }
3537 }
3538
3539 static int be_mcc_queues_create(struct beiscsi_hba *phba,
3540                                 struct hwi_context_memory *phwi_context)
3541 {
3542         struct be_queue_info *q, *cq;
3543         struct be_ctrl_info *ctrl = &phba->ctrl;
3544
3545         /* Alloc MCC compl queue */
3546         cq = &phba->ctrl.mcc_obj.cq;
3547         if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
3548                         sizeof(struct be_mcc_compl)))
3549                 goto err;
3550         /* Ask BE to create MCC compl queue; */
3551         if (phba->msix_enabled) {
3552                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
3553                                          [phba->num_cpus].q, false, true, 0))
3554                 goto mcc_cq_free;
3555         } else {
3556                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
3557                                           false, true, 0))
3558                 goto mcc_cq_free;
3559         }
3560
3561         /* Alloc MCC queue */
3562         q = &phba->ctrl.mcc_obj.q;
3563         if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
3564                 goto mcc_cq_destroy;
3565
3566         /* Ask BE to create MCC queue */
3567         if (beiscsi_cmd_mccq_create(phba, q, cq))
3568                 goto mcc_q_free;
3569
3570         return 0;
3571
3572 mcc_q_free:
3573         be_queue_free(phba, q);
3574 mcc_cq_destroy:
3575         beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
3576 mcc_cq_free:
3577         be_queue_free(phba, cq);
3578 err:
3579         return -ENOMEM;
3580 }
3581
3582 /**
3583  * find_num_cpus()- Get the CPU online count
3584  * @phba: ptr to priv structure
3585  *
3586  * CPU count is used for creating EQ.
3587  **/
3588 static void find_num_cpus(struct beiscsi_hba *phba)
3589 {
3590         int  num_cpus = 0;
3591
3592         num_cpus = num_online_cpus();
3593
3594         switch (phba->generation) {
3595         case BE_GEN2:
3596         case BE_GEN3:
3597                 phba->num_cpus = (num_cpus > BEISCSI_MAX_NUM_CPUS) ?
3598                                   BEISCSI_MAX_NUM_CPUS : num_cpus;
3599                 break;
3600         case BE_GEN4:
3601                 /*
3602                  * If eqid_count == 1 fall back to
3603                  * INTX mechanism
3604                  **/
3605                 if (phba->fw_config.eqid_count == 1) {
3606                         enable_msix = 0;
3607                         phba->num_cpus = 1;
3608                         return;
3609                 }
3610
3611                 phba->num_cpus =
3612                         (num_cpus > (phba->fw_config.eqid_count - 1)) ?
3613                         (phba->fw_config.eqid_count - 1) : num_cpus;
3614                 break;
3615         default:
3616                 phba->num_cpus = 1;
3617         }
3618 }
3619
3620 static void hwi_purge_eq(struct beiscsi_hba *phba)
3621 {
3622         struct hwi_controller *phwi_ctrlr;
3623         struct hwi_context_memory *phwi_context;
3624         struct be_queue_info *eq;
3625         struct be_eq_entry *eqe = NULL;
3626         int i, eq_msix;
3627         unsigned int num_processed;
3628
3629         if (beiscsi_hba_in_error(phba))
3630                 return;
3631
3632         phwi_ctrlr = phba->phwi_ctrlr;
3633         phwi_context = phwi_ctrlr->phwi_ctxt;
3634         if (phba->msix_enabled)
3635                 eq_msix = 1;
3636         else
3637                 eq_msix = 0;
3638
3639         for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
3640                 eq = &phwi_context->be_eq[i].q;
3641                 eqe = queue_tail_node(eq);
3642                 num_processed = 0;
3643                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
3644                                         & EQE_VALID_MASK) {
3645                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
3646                         queue_tail_inc(eq);
3647                         eqe = queue_tail_node(eq);
3648                         num_processed++;
3649                 }
3650
3651                 if (num_processed)
3652                         hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
3653         }
3654 }
3655
3656 static void hwi_cleanup_port(struct beiscsi_hba *phba)
3657 {
3658         struct be_queue_info *q;
3659         struct be_ctrl_info *ctrl = &phba->ctrl;
3660         struct hwi_controller *phwi_ctrlr;
3661         struct hwi_context_memory *phwi_context;
3662         struct hd_async_context *pasync_ctx;
3663         int i, eq_for_mcc, ulp_num;
3664
3665         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
3666                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported))
3667                         beiscsi_cmd_iscsi_cleanup(phba, ulp_num);
3668
3669         /**
3670          * Purge all EQ entries that may have been left out. This is to
3671          * workaround a problem we've seen occasionally where driver gets an
3672          * interrupt with EQ entry bit set after stopping the controller.
3673          */
3674         hwi_purge_eq(phba);
3675
3676         phwi_ctrlr = phba->phwi_ctrlr;
3677         phwi_context = phwi_ctrlr->phwi_ctxt;
3678
3679         be_cmd_iscsi_remove_template_hdr(ctrl);
3680
3681         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3682                 q = &phwi_context->be_wrbq[i];
3683                 if (q->created)
3684                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
3685         }
3686         kfree(phwi_context->be_wrbq);
3687         free_wrb_handles(phba);
3688
3689         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3690                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3691
3692                         q = &phwi_context->be_def_hdrq[ulp_num];
3693                         if (q->created)
3694                                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3695
3696                         q = &phwi_context->be_def_dataq[ulp_num];
3697                         if (q->created)
3698                                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3699
3700                         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num];
3701                 }
3702         }
3703
3704         beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
3705
3706         for (i = 0; i < (phba->num_cpus); i++) {
3707                 q = &phwi_context->be_cq[i];
3708                 if (q->created) {
3709                         be_queue_free(phba, q);
3710                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3711                 }
3712         }
3713
3714         be_mcc_queues_destroy(phba);
3715         if (phba->msix_enabled)
3716                 eq_for_mcc = 1;
3717         else
3718                 eq_for_mcc = 0;
3719         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
3720                 q = &phwi_context->be_eq[i].q;
3721                 if (q->created) {
3722                         be_queue_free(phba, q);
3723                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
3724                 }
3725         }
3726         /* this ensures complete FW cleanup */
3727         beiscsi_cmd_function_reset(phba);
3728         /* last communication, indicate driver is unloading */
3729         beiscsi_cmd_special_wrb(&phba->ctrl, 0);
3730 }
3731
3732 static int hwi_init_port(struct beiscsi_hba *phba)
3733 {
3734         struct hwi_controller *phwi_ctrlr;
3735         struct hwi_context_memory *phwi_context;
3736         unsigned int def_pdu_ring_sz;
3737         struct be_ctrl_info *ctrl = &phba->ctrl;
3738         int status, ulp_num;
3739
3740         phwi_ctrlr = phba->phwi_ctrlr;
3741         phwi_context = phwi_ctrlr->phwi_ctxt;
3742         phwi_context->max_eqd = 128;
3743         phwi_context->min_eqd = 0;
3744         phwi_context->cur_eqd = 32;
3745         /* set port optic state to unknown */
3746         phba->optic_state = 0xff;
3747
3748         status = beiscsi_create_eqs(phba, phwi_context);
3749         if (status != 0) {
3750                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3751                             "BM_%d : EQ not created\n");
3752                 goto error;
3753         }
3754
3755         status = be_mcc_queues_create(phba, phwi_context);
3756         if (status != 0)
3757                 goto error;
3758
3759         status = beiscsi_check_supported_fw(ctrl, phba);
3760         if (status != 0) {
3761                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3762                             "BM_%d : Unsupported fw version\n");
3763                 goto error;
3764         }
3765
3766         status = beiscsi_create_cqs(phba, phwi_context);
3767         if (status != 0) {
3768                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3769                             "BM_%d : CQ not created\n");
3770                 goto error;
3771         }
3772
3773         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3774                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3775                         def_pdu_ring_sz =
3776                                 BEISCSI_GET_CID_COUNT(phba, ulp_num) *
3777                                 sizeof(struct phys_addr);
3778
3779                         status = beiscsi_create_def_hdr(phba, phwi_context,
3780                                                         phwi_ctrlr,
3781                                                         def_pdu_ring_sz,
3782                                                         ulp_num);
3783                         if (status != 0) {
3784                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3785                                             "BM_%d : Default Header not created for ULP : %d\n",
3786                                             ulp_num);
3787                                 goto error;
3788                         }
3789
3790                         status = beiscsi_create_def_data(phba, phwi_context,
3791                                                          phwi_ctrlr,
3792                                                          def_pdu_ring_sz,
3793                                                          ulp_num);
3794                         if (status != 0) {
3795                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3796                                             "BM_%d : Default Data not created for ULP : %d\n",
3797                                             ulp_num);
3798                                 goto error;
3799                         }
3800                         /**
3801                          * Now that the default PDU rings have been created,
3802                          * let EP know about it.
3803                          * Call beiscsi_cmd_iscsi_cleanup before posting?
3804                          */
3805                         beiscsi_hdq_post_handles(phba, BEISCSI_DEFQ_HDR,
3806                                                  ulp_num);
3807                         beiscsi_hdq_post_handles(phba, BEISCSI_DEFQ_DATA,
3808                                                  ulp_num);
3809                 }
3810         }
3811
3812         status = beiscsi_post_pages(phba);
3813         if (status != 0) {
3814                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3815                             "BM_%d : Post SGL Pages Failed\n");
3816                 goto error;
3817         }
3818
3819         status = beiscsi_post_template_hdr(phba);
3820         if (status != 0) {
3821                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3822                             "BM_%d : Template HDR Posting for CXN Failed\n");
3823         }
3824
3825         status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
3826         if (status != 0) {
3827                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3828                             "BM_%d : WRB Rings not created\n");
3829                 goto error;
3830         }
3831
3832         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3833                 uint16_t async_arr_idx = 0;
3834
3835                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3836                         uint16_t cri = 0;
3837                         struct hd_async_context *pasync_ctx;
3838
3839                         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(
3840                                      phwi_ctrlr, ulp_num);
3841                         for (cri = 0; cri <
3842                              phba->params.cxns_per_ctrl; cri++) {
3843                                 if (ulp_num == BEISCSI_GET_ULP_FROM_CRI
3844                                                (phwi_ctrlr, cri))
3845                                         pasync_ctx->cid_to_async_cri_map[
3846                                         phwi_ctrlr->wrb_context[cri].cid] =
3847                                         async_arr_idx++;
3848                         }
3849                         /**
3850                          * Now that the default PDU rings have been created,
3851                          * let EP know about it.
3852                          */
3853                         beiscsi_hdq_post_handles(phba, BEISCSI_DEFQ_HDR,
3854                                                  ulp_num);
3855                         beiscsi_hdq_post_handles(phba, BEISCSI_DEFQ_DATA,
3856                                                  ulp_num);
3857                 }
3858         }
3859
3860         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3861                     "BM_%d : hwi_init_port success\n");
3862         return 0;
3863
3864 error:
3865         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3866                     "BM_%d : hwi_init_port failed");
3867         hwi_cleanup_port(phba);
3868         return status;
3869 }
3870
3871 static int hwi_init_controller(struct beiscsi_hba *phba)
3872 {
3873         struct hwi_controller *phwi_ctrlr;
3874
3875         phwi_ctrlr = phba->phwi_ctrlr;
3876         if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
3877                 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
3878                     init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
3879                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3880                             "BM_%d :  phwi_ctrlr->phwi_ctxt=%p\n",
3881                             phwi_ctrlr->phwi_ctxt);
3882         } else {
3883                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3884                             "BM_%d : HWI_MEM_ADDN_CONTEXT is more "
3885                             "than one element.Failing to load\n");
3886                 return -ENOMEM;
3887         }
3888
3889         iscsi_init_global_templates(phba);
3890         if (beiscsi_init_wrb_handle(phba))
3891                 return -ENOMEM;
3892
3893         if (hwi_init_async_pdu_ctx(phba)) {
3894                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3895                             "BM_%d : hwi_init_async_pdu_ctx failed\n");
3896                 return -ENOMEM;
3897         }
3898
3899         if (hwi_init_port(phba) != 0) {
3900                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3901                             "BM_%d : hwi_init_controller failed\n");
3902
3903                 return -ENOMEM;
3904         }
3905         return 0;
3906 }
3907
3908 static void beiscsi_free_mem(struct beiscsi_hba *phba)
3909 {
3910         struct be_mem_descriptor *mem_descr;
3911         int i, j;
3912
3913         mem_descr = phba->init_mem;
3914         i = 0;
3915         j = 0;
3916         for (i = 0; i < SE_MEM_MAX; i++) {
3917                 for (j = mem_descr->num_elements; j > 0; j--) {
3918                         pci_free_consistent(phba->pcidev,
3919                           mem_descr->mem_array[j - 1].size,
3920                           mem_descr->mem_array[j - 1].virtual_address,
3921                           (unsigned long)mem_descr->mem_array[j - 1].
3922                           bus_address.u.a64.address);
3923                 }
3924
3925                 kfree(mem_descr->mem_array);
3926                 mem_descr++;
3927         }
3928         kfree(phba->init_mem);
3929         kfree(phba->phwi_ctrlr->wrb_context);
3930         kfree(phba->phwi_ctrlr);
3931 }
3932
3933 static int beiscsi_init_controller(struct beiscsi_hba *phba)
3934 {
3935         int ret = -ENOMEM;
3936
3937         ret = beiscsi_get_memory(phba);
3938         if (ret < 0) {
3939                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3940                             "BM_%d : beiscsi_dev_probe -"
3941                             "Failed in beiscsi_alloc_memory\n");
3942                 return ret;
3943         }
3944
3945         ret = hwi_init_controller(phba);
3946         if (ret)
3947                 goto free_init;
3948         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3949                     "BM_%d : Return success from beiscsi_init_controller");
3950
3951         return 0;
3952
3953 free_init:
3954         beiscsi_free_mem(phba);
3955         return ret;
3956 }
3957
3958 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
3959 {
3960         struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
3961         struct sgl_handle *psgl_handle;
3962         struct iscsi_sge *pfrag;
3963         unsigned int arr_index, i, idx;
3964         unsigned int ulp_icd_start, ulp_num = 0;
3965
3966         phba->io_sgl_hndl_avbl = 0;
3967         phba->eh_sgl_hndl_avbl = 0;
3968
3969         mem_descr_sglh = phba->init_mem;
3970         mem_descr_sglh += HWI_MEM_SGLH;
3971         if (1 == mem_descr_sglh->num_elements) {
3972                 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3973                                                  phba->params.ios_per_ctrl,
3974                                                  GFP_KERNEL);
3975                 if (!phba->io_sgl_hndl_base) {
3976                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3977                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
3978                         return -ENOMEM;
3979                 }
3980                 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3981                                                  (phba->params.icds_per_ctrl -
3982                                                  phba->params.ios_per_ctrl),
3983                                                  GFP_KERNEL);
3984                 if (!phba->eh_sgl_hndl_base) {
3985                         kfree(phba->io_sgl_hndl_base);
3986                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3987                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
3988                         return -ENOMEM;
3989                 }
3990         } else {
3991                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3992                             "BM_%d : HWI_MEM_SGLH is more than one element."
3993                             "Failing to load\n");
3994                 return -ENOMEM;
3995         }
3996
3997         arr_index = 0;
3998         idx = 0;
3999         while (idx < mem_descr_sglh->num_elements) {
4000                 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
4001
4002                 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
4003                       sizeof(struct sgl_handle)); i++) {
4004                         if (arr_index < phba->params.ios_per_ctrl) {
4005                                 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
4006                                 phba->io_sgl_hndl_avbl++;
4007                                 arr_index++;
4008                         } else {
4009                                 phba->eh_sgl_hndl_base[arr_index -
4010                                         phba->params.ios_per_ctrl] =
4011                                                                 psgl_handle;
4012                                 arr_index++;
4013                                 phba->eh_sgl_hndl_avbl++;
4014                         }
4015                         psgl_handle++;
4016                 }
4017                 idx++;
4018         }
4019         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4020                     "BM_%d : phba->io_sgl_hndl_avbl=%d"
4021                     "phba->eh_sgl_hndl_avbl=%d\n",
4022                     phba->io_sgl_hndl_avbl,
4023                     phba->eh_sgl_hndl_avbl);
4024
4025         mem_descr_sg = phba->init_mem;
4026         mem_descr_sg += HWI_MEM_SGE;
4027         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4028                     "\n BM_%d : mem_descr_sg->num_elements=%d\n",
4029                     mem_descr_sg->num_elements);
4030
4031         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
4032                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported))
4033                         break;
4034
4035         ulp_icd_start = phba->fw_config.iscsi_icd_start[ulp_num];
4036
4037         arr_index = 0;
4038         idx = 0;
4039         while (idx < mem_descr_sg->num_elements) {
4040                 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
4041
4042                 for (i = 0;
4043                      i < (mem_descr_sg->mem_array[idx].size) /
4044                      (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
4045                      i++) {
4046                         if (arr_index < phba->params.ios_per_ctrl)
4047                                 psgl_handle = phba->io_sgl_hndl_base[arr_index];
4048                         else
4049                                 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
4050                                                 phba->params.ios_per_ctrl];
4051                         psgl_handle->pfrag = pfrag;
4052                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
4053                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
4054                         pfrag += phba->params.num_sge_per_io;
4055                         psgl_handle->sgl_index = ulp_icd_start + arr_index++;
4056                 }
4057                 idx++;
4058         }
4059         phba->io_sgl_free_index = 0;
4060         phba->io_sgl_alloc_index = 0;
4061         phba->eh_sgl_free_index = 0;
4062         phba->eh_sgl_alloc_index = 0;
4063         return 0;
4064 }
4065
4066 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
4067 {
4068         int ret;
4069         uint16_t i, ulp_num;
4070         struct ulp_cid_info *ptr_cid_info = NULL;
4071
4072         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4073                 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4074                         ptr_cid_info = kzalloc(sizeof(struct ulp_cid_info),
4075                                                GFP_KERNEL);
4076
4077                         if (!ptr_cid_info) {
4078                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4079                                             "BM_%d : Failed to allocate memory"
4080                                             "for ULP_CID_INFO for ULP : %d\n",
4081                                             ulp_num);
4082                                 ret = -ENOMEM;
4083                                 goto free_memory;
4084
4085                         }
4086
4087                         /* Allocate memory for CID array */
4088                         ptr_cid_info->cid_array = kzalloc(sizeof(void *) *
4089                                                   BEISCSI_GET_CID_COUNT(phba,
4090                                                   ulp_num), GFP_KERNEL);
4091                         if (!ptr_cid_info->cid_array) {
4092                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4093                                             "BM_%d : Failed to allocate memory"
4094                                             "for CID_ARRAY for ULP : %d\n",
4095                                             ulp_num);
4096                                 kfree(ptr_cid_info);
4097                                 ptr_cid_info = NULL;
4098                                 ret = -ENOMEM;
4099
4100                                 goto free_memory;
4101                         }
4102                         ptr_cid_info->avlbl_cids = BEISCSI_GET_CID_COUNT(
4103                                                    phba, ulp_num);
4104
4105                         /* Save the cid_info_array ptr */
4106                         phba->cid_array_info[ulp_num] = ptr_cid_info;
4107                 }
4108         }
4109         phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
4110                                  phba->params.cxns_per_ctrl, GFP_KERNEL);
4111         if (!phba->ep_array) {
4112                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4113                             "BM_%d : Failed to allocate memory in "
4114                             "hba_setup_cid_tbls\n");
4115                 ret = -ENOMEM;
4116
4117                 goto free_memory;
4118         }
4119
4120         phba->conn_table = kzalloc(sizeof(struct beiscsi_conn *) *
4121                                    phba->params.cxns_per_ctrl, GFP_KERNEL);
4122         if (!phba->conn_table) {
4123                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4124                             "BM_%d : Failed to allocate memory in"
4125                             "hba_setup_cid_tbls\n");
4126
4127                 kfree(phba->ep_array);
4128                 phba->ep_array = NULL;
4129                 ret = -ENOMEM;
4130
4131                 goto free_memory;
4132         }
4133
4134         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
4135                 ulp_num = phba->phwi_ctrlr->wrb_context[i].ulp_num;
4136
4137                 ptr_cid_info = phba->cid_array_info[ulp_num];
4138                 ptr_cid_info->cid_array[ptr_cid_info->cid_alloc++] =
4139                         phba->phwi_ctrlr->wrb_context[i].cid;
4140
4141         }
4142
4143         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4144                 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4145                         ptr_cid_info = phba->cid_array_info[ulp_num];
4146
4147                         ptr_cid_info->cid_alloc = 0;
4148                         ptr_cid_info->cid_free = 0;
4149                 }
4150         }
4151         return 0;
4152
4153 free_memory:
4154         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4155                 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4156                         ptr_cid_info = phba->cid_array_info[ulp_num];
4157
4158                         if (ptr_cid_info) {
4159                                 kfree(ptr_cid_info->cid_array);
4160                                 kfree(ptr_cid_info);
4161                                 phba->cid_array_info[ulp_num] = NULL;
4162                         }
4163                 }
4164         }
4165
4166         return ret;
4167 }
4168
4169 static void hwi_enable_intr(struct beiscsi_hba *phba)
4170 {
4171         struct be_ctrl_info *ctrl = &phba->ctrl;
4172         struct hwi_controller *phwi_ctrlr;
4173         struct hwi_context_memory *phwi_context;
4174         struct be_queue_info *eq;
4175         u8 __iomem *addr;
4176         u32 reg, i;
4177         u32 enabled;
4178
4179         phwi_ctrlr = phba->phwi_ctrlr;
4180         phwi_context = phwi_ctrlr->phwi_ctxt;
4181
4182         addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
4183                         PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
4184         reg = ioread32(addr);
4185
4186         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4187         if (!enabled) {
4188                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4189                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4190                             "BM_%d : reg =x%08x addr=%p\n", reg, addr);
4191                 iowrite32(reg, addr);
4192         }
4193
4194         if (!phba->msix_enabled) {
4195                 eq = &phwi_context->be_eq[0].q;
4196                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4197                             "BM_%d : eq->id=%d\n", eq->id);
4198
4199                 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
4200         } else {
4201                 for (i = 0; i <= phba->num_cpus; i++) {
4202                         eq = &phwi_context->be_eq[i].q;
4203                         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4204                                     "BM_%d : eq->id=%d\n", eq->id);
4205                         hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
4206                 }
4207         }
4208 }
4209
4210 static void hwi_disable_intr(struct beiscsi_hba *phba)
4211 {
4212         struct be_ctrl_info *ctrl = &phba->ctrl;
4213
4214         u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
4215         u32 reg = ioread32(addr);
4216
4217         u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4218         if (enabled) {
4219                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4220                 iowrite32(reg, addr);
4221         } else
4222                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
4223                             "BM_%d : In hwi_disable_intr, Already Disabled\n");
4224 }
4225
4226 static int beiscsi_init_port(struct beiscsi_hba *phba)
4227 {
4228         int ret;
4229
4230         ret = beiscsi_init_controller(phba);
4231         if (ret < 0) {
4232                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4233                             "BM_%d : beiscsi_dev_probe - Failed in"
4234                             "beiscsi_init_controller\n");
4235                 return ret;
4236         }
4237         ret = beiscsi_init_sgl_handle(phba);
4238         if (ret < 0) {
4239                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4240                             "BM_%d : beiscsi_dev_probe - Failed in"
4241                             "beiscsi_init_sgl_handle\n");
4242                 goto do_cleanup_ctrlr;
4243         }
4244
4245         ret = hba_setup_cid_tbls(phba);
4246         if (ret < 0) {
4247                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4248                             "BM_%d : Failed in hba_setup_cid_tbls\n");
4249                 kfree(phba->io_sgl_hndl_base);
4250                 kfree(phba->eh_sgl_hndl_base);
4251                 goto do_cleanup_ctrlr;
4252         }
4253
4254         return ret;
4255
4256 do_cleanup_ctrlr:
4257         hwi_cleanup_port(phba);
4258         return ret;
4259 }
4260
4261 static void beiscsi_cleanup_port(struct beiscsi_hba *phba)
4262 {
4263         struct ulp_cid_info *ptr_cid_info = NULL;
4264         int ulp_num;
4265
4266         kfree(phba->io_sgl_hndl_base);
4267         kfree(phba->eh_sgl_hndl_base);
4268         kfree(phba->ep_array);
4269         kfree(phba->conn_table);
4270
4271         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4272                 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4273                         ptr_cid_info = phba->cid_array_info[ulp_num];
4274
4275                         if (ptr_cid_info) {
4276                                 kfree(ptr_cid_info->cid_array);
4277                                 kfree(ptr_cid_info);
4278                                 phba->cid_array_info[ulp_num] = NULL;
4279                         }
4280                 }
4281         }
4282 }
4283
4284 /**
4285  * beiscsi_free_mgmt_task_handles()- Free driver CXN resources
4286  * @beiscsi_conn: ptr to the conn to be cleaned up
4287  * @task: ptr to iscsi_task resource to be freed.
4288  *
4289  * Free driver mgmt resources binded to CXN.
4290  **/
4291 void
4292 beiscsi_free_mgmt_task_handles(struct beiscsi_conn *beiscsi_conn,
4293                                 struct iscsi_task *task)
4294 {
4295         struct beiscsi_io_task *io_task;
4296         struct beiscsi_hba *phba = beiscsi_conn->phba;
4297         struct hwi_wrb_context *pwrb_context;
4298         struct hwi_controller *phwi_ctrlr;
4299         uint16_t cri_index = BE_GET_CRI_FROM_CID(
4300                                 beiscsi_conn->beiscsi_conn_cid);
4301
4302         phwi_ctrlr = phba->phwi_ctrlr;
4303         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
4304
4305         io_task = task->dd_data;
4306
4307         if (io_task->pwrb_handle) {
4308                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
4309                 io_task->pwrb_handle = NULL;
4310         }
4311
4312         if (io_task->psgl_handle) {
4313                 free_mgmt_sgl_handle(phba, io_task->psgl_handle);
4314                 io_task->psgl_handle = NULL;
4315         }
4316
4317         if (io_task->mtask_addr) {
4318                 pci_unmap_single(phba->pcidev,
4319                                  io_task->mtask_addr,
4320                                  io_task->mtask_data_count,
4321                                  PCI_DMA_TODEVICE);
4322                 io_task->mtask_addr = 0;
4323         }
4324 }
4325
4326 /**
4327  * beiscsi_cleanup_task()- Free driver resources of the task
4328  * @task: ptr to the iscsi task
4329  *
4330  **/
4331 static void beiscsi_cleanup_task(struct iscsi_task *task)
4332 {
4333         struct beiscsi_io_task *io_task = task->dd_data;
4334         struct iscsi_conn *conn = task->conn;
4335         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4336         struct beiscsi_hba *phba = beiscsi_conn->phba;
4337         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
4338         struct hwi_wrb_context *pwrb_context;
4339         struct hwi_controller *phwi_ctrlr;
4340         uint16_t cri_index = BE_GET_CRI_FROM_CID(
4341                              beiscsi_conn->beiscsi_conn_cid);
4342
4343         phwi_ctrlr = phba->phwi_ctrlr;
4344         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
4345
4346         if (io_task->cmd_bhs) {
4347                 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
4348                               io_task->bhs_pa.u.a64.address);
4349                 io_task->cmd_bhs = NULL;
4350                 task->hdr = NULL;
4351         }
4352
4353         if (task->sc) {
4354                 if (io_task->pwrb_handle) {
4355                         free_wrb_handle(phba, pwrb_context,
4356                                         io_task->pwrb_handle);
4357                         io_task->pwrb_handle = NULL;
4358                 }
4359
4360                 if (io_task->psgl_handle) {
4361                         free_io_sgl_handle(phba, io_task->psgl_handle);
4362                         io_task->psgl_handle = NULL;
4363                 }
4364
4365                 if (io_task->scsi_cmnd) {
4366                         if (io_task->num_sg)
4367                                 scsi_dma_unmap(io_task->scsi_cmnd);
4368                         io_task->scsi_cmnd = NULL;
4369                 }
4370         } else {
4371                 if (!beiscsi_conn->login_in_progress)
4372                         beiscsi_free_mgmt_task_handles(beiscsi_conn, task);
4373         }
4374 }
4375
4376 void
4377 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
4378                            struct beiscsi_offload_params *params)
4379 {
4380         struct wrb_handle *pwrb_handle;
4381         struct hwi_wrb_context *pwrb_context = NULL;
4382         struct beiscsi_hba *phba = beiscsi_conn->phba;
4383         struct iscsi_task *task = beiscsi_conn->task;
4384         struct iscsi_session *session = task->conn->session;
4385         u32 doorbell = 0;
4386
4387         /*
4388          * We can always use 0 here because it is reserved by libiscsi for
4389          * login/startup related tasks.
4390          */
4391         beiscsi_conn->login_in_progress = 0;
4392         spin_lock_bh(&session->back_lock);
4393         beiscsi_cleanup_task(task);
4394         spin_unlock_bh(&session->back_lock);
4395
4396         pwrb_handle = alloc_wrb_handle(phba, beiscsi_conn->beiscsi_conn_cid,
4397                                        &pwrb_context);
4398
4399         /* Check for the adapter family */
4400         if (is_chip_be2_be3r(phba))
4401                 beiscsi_offload_cxn_v0(params, pwrb_handle,
4402                                        phba->init_mem,
4403                                        pwrb_context);
4404         else
4405                 beiscsi_offload_cxn_v2(params, pwrb_handle,
4406                                        pwrb_context);
4407
4408         be_dws_le_to_cpu(pwrb_handle->pwrb,
4409                          sizeof(struct iscsi_target_context_update_wrb));
4410
4411         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4412         doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
4413                              << DB_DEF_PDU_WRB_INDEX_SHIFT;
4414         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4415         iowrite32(doorbell, phba->db_va +
4416                   beiscsi_conn->doorbell_offset);
4417
4418         /*
4419          * There is no completion for CONTEXT_UPDATE. The completion of next
4420          * WRB posted guarantees FW's processing and DMA'ing of it.
4421          * Use beiscsi_put_wrb_handle to put it back in the pool which makes
4422          * sure zero'ing or reuse of the WRB only after wrbs_per_cxn.
4423          */
4424         beiscsi_put_wrb_handle(pwrb_context, pwrb_handle,
4425                                phba->params.wrbs_per_cxn);
4426         beiscsi_log(phba, KERN_INFO,
4427                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4428                     "BM_%d : put CONTEXT_UPDATE pwrb_handle=%p free_index=0x%x wrb_handles_available=%d\n",
4429                     pwrb_handle, pwrb_context->free_index,
4430                     pwrb_context->wrb_handles_available);
4431 }
4432
4433 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
4434                               int *index, int *age)
4435 {
4436         *index = (int)itt;
4437         if (age)
4438                 *age = conn->session->age;
4439 }
4440
4441 /**
4442  * beiscsi_alloc_pdu - allocates pdu and related resources
4443  * @task: libiscsi task
4444  * @opcode: opcode of pdu for task
4445  *
4446  * This is called with the session lock held. It will allocate
4447  * the wrb and sgl if needed for the command. And it will prep
4448  * the pdu's itt. beiscsi_parse_pdu will later translate
4449  * the pdu itt to the libiscsi task itt.
4450  */
4451 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
4452 {
4453         struct beiscsi_io_task *io_task = task->dd_data;
4454         struct iscsi_conn *conn = task->conn;
4455         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4456         struct beiscsi_hba *phba = beiscsi_conn->phba;
4457         struct hwi_wrb_context *pwrb_context;
4458         struct hwi_controller *phwi_ctrlr;
4459         itt_t itt;
4460         uint16_t cri_index = 0;
4461         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
4462         dma_addr_t paddr;
4463
4464         io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
4465                                           GFP_ATOMIC, &paddr);
4466         if (!io_task->cmd_bhs)
4467                 return -ENOMEM;
4468         io_task->bhs_pa.u.a64.address = paddr;
4469         io_task->libiscsi_itt = (itt_t)task->itt;
4470         io_task->conn = beiscsi_conn;
4471
4472         task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
4473         task->hdr_max = sizeof(struct be_cmd_bhs);
4474         io_task->psgl_handle = NULL;
4475         io_task->pwrb_handle = NULL;
4476
4477         if (task->sc) {
4478                 io_task->psgl_handle = alloc_io_sgl_handle(phba);
4479                 if (!io_task->psgl_handle) {
4480                         beiscsi_log(phba, KERN_ERR,
4481                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4482                                     "BM_%d : Alloc of IO_SGL_ICD Failed"
4483                                     "for the CID : %d\n",
4484                                     beiscsi_conn->beiscsi_conn_cid);
4485                         goto free_hndls;
4486                 }
4487                 io_task->pwrb_handle = alloc_wrb_handle(phba,
4488                                         beiscsi_conn->beiscsi_conn_cid,
4489                                         &io_task->pwrb_context);
4490                 if (!io_task->pwrb_handle) {
4491                         beiscsi_log(phba, KERN_ERR,
4492                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4493                                     "BM_%d : Alloc of WRB_HANDLE Failed"
4494                                     "for the CID : %d\n",
4495                                     beiscsi_conn->beiscsi_conn_cid);
4496                         goto free_io_hndls;
4497                 }
4498         } else {
4499                 io_task->scsi_cmnd = NULL;
4500                 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
4501                         beiscsi_conn->task = task;
4502                         if (!beiscsi_conn->login_in_progress) {
4503                                 io_task->psgl_handle = (struct sgl_handle *)
4504                                                 alloc_mgmt_sgl_handle(phba);
4505                                 if (!io_task->psgl_handle) {
4506                                         beiscsi_log(phba, KERN_ERR,
4507                                                     BEISCSI_LOG_IO |
4508                                                     BEISCSI_LOG_CONFIG,
4509                                                     "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4510                                                     "for the CID : %d\n",
4511                                                     beiscsi_conn->
4512                                                     beiscsi_conn_cid);
4513                                         goto free_hndls;
4514                                 }
4515
4516                                 beiscsi_conn->login_in_progress = 1;
4517                                 beiscsi_conn->plogin_sgl_handle =
4518                                                         io_task->psgl_handle;
4519                                 io_task->pwrb_handle =
4520                                         alloc_wrb_handle(phba,
4521                                         beiscsi_conn->beiscsi_conn_cid,
4522                                         &io_task->pwrb_context);
4523                                 if (!io_task->pwrb_handle) {
4524                                         beiscsi_log(phba, KERN_ERR,
4525                                                     BEISCSI_LOG_IO |
4526                                                     BEISCSI_LOG_CONFIG,
4527                                                     "BM_%d : Alloc of WRB_HANDLE Failed"
4528                                                     "for the CID : %d\n",
4529                                                     beiscsi_conn->
4530                                                     beiscsi_conn_cid);
4531                                         goto free_mgmt_hndls;
4532                                 }
4533                                 beiscsi_conn->plogin_wrb_handle =
4534                                                         io_task->pwrb_handle;
4535
4536                         } else {
4537                                 io_task->psgl_handle =
4538                                                 beiscsi_conn->plogin_sgl_handle;
4539                                 io_task->pwrb_handle =
4540                                                 beiscsi_conn->plogin_wrb_handle;
4541                         }
4542                 } else {
4543                         io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
4544                         if (!io_task->psgl_handle) {
4545                                 beiscsi_log(phba, KERN_ERR,
4546                                             BEISCSI_LOG_IO |
4547                                             BEISCSI_LOG_CONFIG,
4548                                             "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4549                                             "for the CID : %d\n",
4550                                             beiscsi_conn->
4551                                             beiscsi_conn_cid);
4552                                 goto free_hndls;
4553                         }
4554                         io_task->pwrb_handle =
4555                                         alloc_wrb_handle(phba,
4556                                         beiscsi_conn->beiscsi_conn_cid,
4557                                         &io_task->pwrb_context);
4558                         if (!io_task->pwrb_handle) {
4559                                 beiscsi_log(phba, KERN_ERR,
4560                                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4561                                             "BM_%d : Alloc of WRB_HANDLE Failed"
4562                                             "for the CID : %d\n",
4563                                             beiscsi_conn->beiscsi_conn_cid);
4564                                 goto free_mgmt_hndls;
4565                         }
4566
4567                 }
4568         }
4569         itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
4570                                  wrb_index << 16) | (unsigned int)
4571                                 (io_task->psgl_handle->sgl_index));
4572         io_task->pwrb_handle->pio_handle = task;
4573
4574         io_task->cmd_bhs->iscsi_hdr.itt = itt;
4575         return 0;
4576
4577 free_io_hndls:
4578         free_io_sgl_handle(phba, io_task->psgl_handle);
4579         goto free_hndls;
4580 free_mgmt_hndls:
4581         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
4582         io_task->psgl_handle = NULL;
4583 free_hndls:
4584         phwi_ctrlr = phba->phwi_ctrlr;
4585         cri_index = BE_GET_CRI_FROM_CID(
4586         beiscsi_conn->beiscsi_conn_cid);
4587         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
4588         if (io_task->pwrb_handle)
4589                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
4590         io_task->pwrb_handle = NULL;
4591         pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
4592                       io_task->bhs_pa.u.a64.address);
4593         io_task->cmd_bhs = NULL;
4594         return -ENOMEM;
4595 }
4596 static int beiscsi_iotask_v2(struct iscsi_task *task, struct scatterlist *sg,
4597                        unsigned int num_sg, unsigned int xferlen,
4598                        unsigned int writedir)
4599 {
4600
4601         struct beiscsi_io_task *io_task = task->dd_data;
4602         struct iscsi_conn *conn = task->conn;
4603         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4604         struct beiscsi_hba *phba = beiscsi_conn->phba;
4605         struct iscsi_wrb *pwrb = NULL;
4606         unsigned int doorbell = 0;
4607
4608         pwrb = io_task->pwrb_handle->pwrb;
4609
4610         io_task->bhs_len = sizeof(struct be_cmd_bhs);
4611
4612         if (writedir) {
4613                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb,
4614                               INI_WR_CMD);
4615                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 1);
4616         } else {
4617                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb,
4618                               INI_RD_CMD);
4619                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 0);
4620         }
4621
4622         io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb_v2,
4623                                           type, pwrb);
4624
4625         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, lun, pwrb,
4626                       cpu_to_be16(*(unsigned short *)
4627                       &io_task->cmd_bhs->iscsi_hdr.lun));
4628         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb, xferlen);
4629         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb,
4630                       io_task->pwrb_handle->wrb_index);
4631         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb,
4632                       be32_to_cpu(task->cmdsn));
4633         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb,
4634                       io_task->psgl_handle->sgl_index);
4635
4636         hwi_write_sgl_v2(pwrb, sg, num_sg, io_task);
4637         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb,
4638                       io_task->pwrb_handle->wrb_index);
4639         if (io_task->pwrb_context->plast_wrb)
4640                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb,
4641                               io_task->pwrb_context->plast_wrb,
4642                               io_task->pwrb_handle->wrb_index);
4643         io_task->pwrb_context->plast_wrb = pwrb;
4644
4645         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4646
4647         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4648         doorbell |= (io_task->pwrb_handle->wrb_index &
4649                      DB_DEF_PDU_WRB_INDEX_MASK) <<
4650                      DB_DEF_PDU_WRB_INDEX_SHIFT;
4651         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4652         iowrite32(doorbell, phba->db_va +
4653                   beiscsi_conn->doorbell_offset);
4654         return 0;
4655 }
4656
4657 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
4658                           unsigned int num_sg, unsigned int xferlen,
4659                           unsigned int writedir)
4660 {
4661
4662         struct beiscsi_io_task *io_task = task->dd_data;
4663         struct iscsi_conn *conn = task->conn;
4664         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4665         struct beiscsi_hba *phba = beiscsi_conn->phba;
4666         struct iscsi_wrb *pwrb = NULL;
4667         unsigned int doorbell = 0;
4668
4669         pwrb = io_task->pwrb_handle->pwrb;
4670         io_task->bhs_len = sizeof(struct be_cmd_bhs);
4671
4672         if (writedir) {
4673                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4674                               INI_WR_CMD);
4675                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
4676         } else {
4677                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4678                               INI_RD_CMD);
4679                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
4680         }
4681
4682         io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb,
4683                                           type, pwrb);
4684
4685         AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
4686                       cpu_to_be16(*(unsigned short *)
4687                                   &io_task->cmd_bhs->iscsi_hdr.lun));
4688         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
4689         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4690                       io_task->pwrb_handle->wrb_index);
4691         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4692                       be32_to_cpu(task->cmdsn));
4693         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4694                       io_task->psgl_handle->sgl_index);
4695
4696         hwi_write_sgl(pwrb, sg, num_sg, io_task);
4697
4698         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4699                       io_task->pwrb_handle->wrb_index);
4700         if (io_task->pwrb_context->plast_wrb)
4701                 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb,
4702                               io_task->pwrb_context->plast_wrb,
4703                               io_task->pwrb_handle->wrb_index);
4704         io_task->pwrb_context->plast_wrb = pwrb;
4705
4706         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4707
4708         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4709         doorbell |= (io_task->pwrb_handle->wrb_index &
4710                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4711         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4712
4713         iowrite32(doorbell, phba->db_va +
4714                   beiscsi_conn->doorbell_offset);
4715         return 0;
4716 }
4717
4718 static int beiscsi_mtask(struct iscsi_task *task)
4719 {
4720         struct beiscsi_io_task *io_task = task->dd_data;
4721         struct iscsi_conn *conn = task->conn;
4722         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4723         struct beiscsi_hba *phba = beiscsi_conn->phba;
4724         struct iscsi_wrb *pwrb = NULL;
4725         unsigned int doorbell = 0;
4726         unsigned int cid;
4727         unsigned int pwrb_typeoffset = 0;
4728         int ret = 0;
4729
4730         cid = beiscsi_conn->beiscsi_conn_cid;
4731         pwrb = io_task->pwrb_handle->pwrb;
4732
4733         if (is_chip_be2_be3r(phba)) {
4734                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4735                               be32_to_cpu(task->cmdsn));
4736                 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4737                               io_task->pwrb_handle->wrb_index);
4738                 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4739                               io_task->psgl_handle->sgl_index);
4740                 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
4741                               task->data_count);
4742                 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4743                               io_task->pwrb_handle->wrb_index);
4744                 if (io_task->pwrb_context->plast_wrb)
4745                         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb,
4746                                       io_task->pwrb_context->plast_wrb,
4747                                       io_task->pwrb_handle->wrb_index);
4748                 io_task->pwrb_context->plast_wrb = pwrb;
4749
4750                 pwrb_typeoffset = BE_WRB_TYPE_OFFSET;
4751         } else {
4752                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb,
4753                               be32_to_cpu(task->cmdsn));
4754                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb,
4755                               io_task->pwrb_handle->wrb_index);
4756                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb,
4757                               io_task->psgl_handle->sgl_index);
4758                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb,
4759                               task->data_count);
4760                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb,
4761                               io_task->pwrb_handle->wrb_index);
4762                 if (io_task->pwrb_context->plast_wrb)
4763                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb,
4764                                       io_task->pwrb_context->plast_wrb,
4765                                       io_task->pwrb_handle->wrb_index);
4766                 io_task->pwrb_context->plast_wrb = pwrb;
4767
4768                 pwrb_typeoffset = SKH_WRB_TYPE_OFFSET;
4769         }
4770
4771
4772         switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
4773         case ISCSI_OP_LOGIN:
4774                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
4775                 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
4776                 ret = hwi_write_buffer(pwrb, task);
4777                 break;
4778         case ISCSI_OP_NOOP_OUT:
4779                 if (task->hdr->ttt != ISCSI_RESERVED_TAG) {
4780                         ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
4781                         if (is_chip_be2_be3r(phba))
4782                                 AMAP_SET_BITS(struct amap_iscsi_wrb,
4783                                               dmsg, pwrb, 1);
4784                         else
4785                                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
4786                                               dmsg, pwrb, 1);
4787                 } else {
4788                         ADAPTER_SET_WRB_TYPE(pwrb, INI_RD_CMD, pwrb_typeoffset);
4789                         if (is_chip_be2_be3r(phba))
4790                                 AMAP_SET_BITS(struct amap_iscsi_wrb,
4791                                               dmsg, pwrb, 0);
4792                         else
4793                                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
4794                                               dmsg, pwrb, 0);
4795                 }
4796                 ret = hwi_write_buffer(pwrb, task);
4797                 break;
4798         case ISCSI_OP_TEXT:
4799                 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
4800                 ret = hwi_write_buffer(pwrb, task);
4801                 break;
4802         case ISCSI_OP_SCSI_TMFUNC:
4803                 ADAPTER_SET_WRB_TYPE(pwrb, INI_TMF_CMD, pwrb_typeoffset);
4804                 ret = hwi_write_buffer(pwrb, task);
4805                 break;
4806         case ISCSI_OP_LOGOUT:
4807                 ADAPTER_SET_WRB_TYPE(pwrb, HWH_TYPE_LOGOUT, pwrb_typeoffset);
4808                 ret = hwi_write_buffer(pwrb, task);
4809                 break;
4810
4811         default:
4812                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4813                             "BM_%d : opcode =%d Not supported\n",
4814                             task->hdr->opcode & ISCSI_OPCODE_MASK);
4815
4816                 return -EINVAL;
4817         }
4818
4819         if (ret)
4820                 return ret;
4821
4822         /* Set the task type */
4823         io_task->wrb_type = (is_chip_be2_be3r(phba)) ?
4824                 AMAP_GET_BITS(struct amap_iscsi_wrb, type, pwrb) :
4825                 AMAP_GET_BITS(struct amap_iscsi_wrb_v2, type, pwrb);
4826
4827         doorbell |= cid & DB_WRB_POST_CID_MASK;
4828         doorbell |= (io_task->pwrb_handle->wrb_index &
4829                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4830         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4831         iowrite32(doorbell, phba->db_va +
4832                   beiscsi_conn->doorbell_offset);
4833         return 0;
4834 }
4835
4836 static int beiscsi_task_xmit(struct iscsi_task *task)
4837 {
4838         struct beiscsi_io_task *io_task = task->dd_data;
4839         struct scsi_cmnd *sc = task->sc;
4840         struct beiscsi_hba *phba;
4841         struct scatterlist *sg;
4842         int num_sg;
4843         unsigned int  writedir = 0, xferlen = 0;
4844
4845         phba = io_task->conn->phba;
4846         /**
4847          * HBA in error includes BEISCSI_HBA_FW_TIMEOUT. IO path might be
4848          * operational if FW still gets heartbeat from EP FW. Is management
4849          * path really needed to continue further?
4850          */
4851         if (!beiscsi_hba_is_online(phba))
4852                 return -EIO;
4853
4854         if (!io_task->conn->login_in_progress)
4855                 task->hdr->exp_statsn = 0;
4856
4857         if (!sc)
4858                 return beiscsi_mtask(task);
4859
4860         io_task->scsi_cmnd = sc;
4861         io_task->num_sg = 0;
4862         num_sg = scsi_dma_map(sc);
4863         if (num_sg < 0) {
4864                 beiscsi_log(phba, KERN_ERR,
4865                             BEISCSI_LOG_IO | BEISCSI_LOG_ISCSI,
4866                             "BM_%d : scsi_dma_map Failed "
4867                             "Driver_ITT : 0x%x ITT : 0x%x Xferlen : 0x%x\n",
4868                             be32_to_cpu(io_task->cmd_bhs->iscsi_hdr.itt),
4869                             io_task->libiscsi_itt, scsi_bufflen(sc));
4870
4871                 return num_sg;
4872         }
4873         /**
4874          * For scsi cmd task, check num_sg before unmapping in cleanup_task.
4875          * For management task, cleanup_task checks mtask_addr before unmapping.
4876          */
4877         io_task->num_sg = num_sg;
4878         xferlen = scsi_bufflen(sc);
4879         sg = scsi_sglist(sc);
4880         if (sc->sc_data_direction == DMA_TO_DEVICE)
4881                 writedir = 1;
4882          else
4883                 writedir = 0;
4884
4885          return phba->iotask_fn(task, sg, num_sg, xferlen, writedir);
4886 }
4887
4888 /**
4889  * beiscsi_bsg_request - handle bsg request from ISCSI transport
4890  * @job: job to handle
4891  */
4892 static int beiscsi_bsg_request(struct bsg_job *job)
4893 {
4894         struct Scsi_Host *shost;
4895         struct beiscsi_hba *phba;
4896         struct iscsi_bsg_request *bsg_req = job->request;
4897         int rc = -EINVAL;
4898         unsigned int tag;
4899         struct be_dma_mem nonemb_cmd;
4900         struct be_cmd_resp_hdr *resp;
4901         struct iscsi_bsg_reply *bsg_reply = job->reply;
4902         unsigned short status, extd_status;
4903
4904         shost = iscsi_job_to_shost(job);
4905         phba = iscsi_host_priv(shost);
4906
4907         if (!beiscsi_hba_is_online(phba)) {
4908                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
4909                             "BM_%d : HBA in error 0x%lx\n", phba->state);
4910                 return -ENXIO;
4911         }
4912
4913         switch (bsg_req->msgcode) {
4914         case ISCSI_BSG_HST_VENDOR:
4915                 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
4916                                         job->request_payload.payload_len,
4917                                         &nonemb_cmd.dma);
4918                 if (nonemb_cmd.va == NULL) {
4919                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4920                                     "BM_%d : Failed to allocate memory for "
4921                                     "beiscsi_bsg_request\n");
4922                         return -ENOMEM;
4923                 }
4924                 tag = mgmt_vendor_specific_fw_cmd(&phba->ctrl, phba, job,
4925                                                   &nonemb_cmd);
4926                 if (!tag) {
4927                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4928                                     "BM_%d : MBX Tag Allocation Failed\n");
4929
4930                         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4931                                             nonemb_cmd.va, nonemb_cmd.dma);
4932                         return -EAGAIN;
4933                 }
4934
4935                 rc = wait_event_interruptible_timeout(
4936                                         phba->ctrl.mcc_wait[tag],
4937                                         phba->ctrl.mcc_tag_status[tag],
4938                                         msecs_to_jiffies(
4939                                         BEISCSI_HOST_MBX_TIMEOUT));
4940
4941                 if (!test_bit(BEISCSI_HBA_ONLINE, &phba->state)) {
4942                         clear_bit(MCC_TAG_STATE_RUNNING,
4943                                   &phba->ctrl.ptag_state[tag].tag_state);
4944                         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4945                                             nonemb_cmd.va, nonemb_cmd.dma);
4946                         return -EIO;
4947                 }
4948                 extd_status = (phba->ctrl.mcc_tag_status[tag] &
4949                                CQE_STATUS_ADDL_MASK) >> CQE_STATUS_ADDL_SHIFT;
4950                 status = phba->ctrl.mcc_tag_status[tag] & CQE_STATUS_MASK;
4951                 free_mcc_wrb(&phba->ctrl, tag);
4952                 resp = (struct be_cmd_resp_hdr *)nonemb_cmd.va;
4953                 sg_copy_from_buffer(job->reply_payload.sg_list,
4954                                     job->reply_payload.sg_cnt,
4955                                     nonemb_cmd.va, (resp->response_length
4956                                     + sizeof(*resp)));
4957                 bsg_reply->reply_payload_rcv_len = resp->response_length;
4958                 bsg_reply->result = status;
4959                 bsg_job_done(job, bsg_reply->result,
4960                              bsg_reply->reply_payload_rcv_len);
4961                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4962                                     nonemb_cmd.va, nonemb_cmd.dma);
4963                 if (status || extd_status) {
4964                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4965                                     "BM_%d : MBX Cmd Failed"
4966                                     " status = %d extd_status = %d\n",
4967                                     status, extd_status);
4968
4969                         return -EIO;
4970                 } else {
4971                         rc = 0;
4972                 }
4973                 break;
4974
4975         default:
4976                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4977                                 "BM_%d : Unsupported bsg command: 0x%x\n",
4978                                 bsg_req->msgcode);
4979                 break;
4980         }
4981
4982         return rc;
4983 }
4984
4985 static void beiscsi_hba_attrs_init(struct beiscsi_hba *phba)
4986 {
4987         /* Set the logging parameter */
4988         beiscsi_log_enable_init(phba, beiscsi_log_enable);
4989 }
4990
4991 void beiscsi_start_boot_work(struct beiscsi_hba *phba, unsigned int s_handle)
4992 {
4993         if (phba->boot_struct.boot_kset)
4994                 return;
4995
4996         /* skip if boot work is already in progress */
4997         if (test_and_set_bit(BEISCSI_HBA_BOOT_WORK, &phba->state))
4998                 return;
4999
5000         phba->boot_struct.retry = 3;
5001         phba->boot_struct.tag = 0;
5002         phba->boot_struct.s_handle = s_handle;
5003         phba->boot_struct.action = BEISCSI_BOOT_GET_SHANDLE;
5004         schedule_work(&phba->boot_work);
5005 }
5006
5007 static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
5008 {
5009         struct beiscsi_hba *phba = data;
5010         struct mgmt_session_info *boot_sess = &phba->boot_struct.boot_sess;
5011         struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
5012         char *str = buf;
5013         int rc = -EPERM;
5014
5015         switch (type) {
5016         case ISCSI_BOOT_TGT_NAME:
5017                 rc = sprintf(buf, "%.*s\n",
5018                             (int)strlen(boot_sess->target_name),
5019                             (char *)&boot_sess->target_name);
5020                 break;
5021         case ISCSI_BOOT_TGT_IP_ADDR:
5022                 if (boot_conn->dest_ipaddr.ip_type == BEISCSI_IP_TYPE_V4)
5023                         rc = sprintf(buf, "%pI4\n",
5024                                 (char *)&boot_conn->dest_ipaddr.addr);
5025                 else
5026                         rc = sprintf(str, "%pI6\n",
5027                                 (char *)&boot_conn->dest_ipaddr.addr);
5028                 break;
5029         case ISCSI_BOOT_TGT_PORT:
5030                 rc = sprintf(str, "%d\n", boot_conn->dest_port);
5031                 break;
5032
5033         case ISCSI_BOOT_TGT_CHAP_NAME:
5034                 rc = sprintf(str,  "%.*s\n",
5035                              boot_conn->negotiated_login_options.auth_data.chap.
5036                              target_chap_name_length,
5037                              (char *)&boot_conn->negotiated_login_options.
5038                              auth_data.chap.target_chap_name);
5039                 break;
5040         case ISCSI_BOOT_TGT_CHAP_SECRET:
5041                 rc = sprintf(str,  "%.*s\n",
5042                              boot_conn->negotiated_login_options.auth_data.chap.
5043                              target_secret_length,
5044                              (char *)&boot_conn->negotiated_login_options.
5045                              auth_data.chap.target_secret);
5046                 break;
5047         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
5048                 rc = sprintf(str,  "%.*s\n",
5049                              boot_conn->negotiated_login_options.auth_data.chap.
5050                              intr_chap_name_length,
5051                              (char *)&boot_conn->negotiated_login_options.
5052                              auth_data.chap.intr_chap_name);
5053                 break;
5054         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
5055                 rc = sprintf(str,  "%.*s\n",
5056                              boot_conn->negotiated_login_options.auth_data.chap.
5057                              intr_secret_length,
5058                              (char *)&boot_conn->negotiated_login_options.
5059                              auth_data.chap.intr_secret);
5060                 break;
5061         case ISCSI_BOOT_TGT_FLAGS:
5062                 rc = sprintf(str, "2\n");
5063                 break;
5064         case ISCSI_BOOT_TGT_NIC_ASSOC:
5065                 rc = sprintf(str, "0\n");
5066                 break;
5067         }
5068         return rc;
5069 }
5070
5071 static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
5072 {
5073         struct beiscsi_hba *phba = data;
5074         char *str = buf;
5075         int rc = -EPERM;
5076
5077         switch (type) {
5078         case ISCSI_BOOT_INI_INITIATOR_NAME:
5079                 rc = sprintf(str, "%s\n",
5080                              phba->boot_struct.boot_sess.initiator_iscsiname);
5081                 break;
5082         }
5083         return rc;
5084 }
5085
5086 static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
5087 {
5088         struct beiscsi_hba *phba = data;
5089         char *str = buf;
5090         int rc = -EPERM;
5091
5092         switch (type) {
5093         case ISCSI_BOOT_ETH_FLAGS:
5094                 rc = sprintf(str, "2\n");
5095                 break;
5096         case ISCSI_BOOT_ETH_INDEX:
5097                 rc = sprintf(str, "0\n");
5098                 break;
5099         case ISCSI_BOOT_ETH_MAC:
5100                 rc  = beiscsi_get_macaddr(str, phba);
5101                 break;
5102         }
5103         return rc;
5104 }
5105
5106 static umode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
5107 {
5108         umode_t rc = 0;
5109
5110         switch (type) {
5111         case ISCSI_BOOT_TGT_NAME:
5112         case ISCSI_BOOT_TGT_IP_ADDR:
5113         case ISCSI_BOOT_TGT_PORT:
5114         case ISCSI_BOOT_TGT_CHAP_NAME:
5115         case ISCSI_BOOT_TGT_CHAP_SECRET:
5116         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
5117         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
5118         case ISCSI_BOOT_TGT_NIC_ASSOC:
5119         case ISCSI_BOOT_TGT_FLAGS:
5120                 rc = S_IRUGO;
5121                 break;
5122         }
5123         return rc;
5124 }
5125
5126 static umode_t beiscsi_ini_get_attr_visibility(void *data, int type)
5127 {
5128         umode_t rc = 0;
5129
5130         switch (type) {
5131         case ISCSI_BOOT_INI_INITIATOR_NAME:
5132                 rc = S_IRUGO;
5133                 break;
5134         }
5135         return rc;
5136 }
5137
5138 static umode_t beiscsi_eth_get_attr_visibility(void *data, int type)
5139 {
5140         umode_t rc = 0;
5141
5142         switch (type) {
5143         case ISCSI_BOOT_ETH_FLAGS:
5144         case ISCSI_BOOT_ETH_MAC:
5145         case ISCSI_BOOT_ETH_INDEX:
5146                 rc = S_IRUGO;
5147                 break;
5148         }
5149         return rc;
5150 }
5151
5152 static void beiscsi_boot_kobj_release(void *data)
5153 {
5154         struct beiscsi_hba *phba = data;
5155
5156         scsi_host_put(phba->shost);
5157 }
5158
5159 static int beiscsi_boot_create_kset(struct beiscsi_hba *phba)
5160 {
5161         struct boot_struct *bs = &phba->boot_struct;
5162         struct iscsi_boot_kobj *boot_kobj;
5163
5164         if (bs->boot_kset) {
5165                 __beiscsi_log(phba, KERN_ERR,
5166                               "BM_%d: boot_kset already created\n");
5167                 return 0;
5168         }
5169
5170         bs->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);
5171         if (!bs->boot_kset) {
5172                 __beiscsi_log(phba, KERN_ERR,
5173                               "BM_%d: boot_kset alloc failed\n");
5174                 return -ENOMEM;
5175         }
5176
5177         /* get shost ref because the show function will refer phba */
5178         if (!scsi_host_get(phba->shost))
5179                 goto free_kset;
5180
5181         boot_kobj = iscsi_boot_create_target(bs->boot_kset, 0, phba,
5182                                              beiscsi_show_boot_tgt_info,
5183                                              beiscsi_tgt_get_attr_visibility,
5184                                              beiscsi_boot_kobj_release);
5185         if (!boot_kobj)
5186                 goto put_shost;
5187
5188         if (!scsi_host_get(phba->shost))
5189                 goto free_kset;
5190
5191         boot_kobj = iscsi_boot_create_initiator(bs->boot_kset, 0, phba,
5192                                                 beiscsi_show_boot_ini_info,
5193                                                 beiscsi_ini_get_attr_visibility,
5194                                                 beiscsi_boot_kobj_release);
5195         if (!boot_kobj)
5196                 goto put_shost;
5197
5198         if (!scsi_host_get(phba->shost))
5199                 goto free_kset;
5200
5201         boot_kobj = iscsi_boot_create_ethernet(bs->boot_kset, 0, phba,
5202                                                beiscsi_show_boot_eth_info,
5203                                                beiscsi_eth_get_attr_visibility,
5204                                                beiscsi_boot_kobj_release);
5205         if (!boot_kobj)
5206                 goto put_shost;
5207
5208         return 0;
5209
5210 put_shost:
5211         scsi_host_put(phba->shost);
5212 free_kset:
5213         iscsi_boot_destroy_kset(bs->boot_kset);
5214         bs->boot_kset = NULL;
5215         return -ENOMEM;
5216 }
5217
5218 static void beiscsi_boot_work(struct work_struct *work)
5219 {
5220         struct beiscsi_hba *phba =
5221                 container_of(work, struct beiscsi_hba, boot_work);
5222         struct boot_struct *bs = &phba->boot_struct;
5223         unsigned int tag = 0;
5224
5225         if (!beiscsi_hba_is_online(phba))
5226                 return;
5227
5228         beiscsi_log(phba, KERN_INFO,
5229                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
5230                     "BM_%d : %s action %d\n",
5231                     __func__, phba->boot_struct.action);
5232
5233         switch (phba->boot_struct.action) {
5234         case BEISCSI_BOOT_REOPEN_SESS:
5235                 tag = beiscsi_boot_reopen_sess(phba);
5236                 break;
5237         case BEISCSI_BOOT_GET_SHANDLE:
5238                 tag = __beiscsi_boot_get_shandle(phba, 1);
5239                 break;
5240         case BEISCSI_BOOT_GET_SINFO:
5241                 tag = beiscsi_boot_get_sinfo(phba);
5242                 break;
5243         case BEISCSI_BOOT_LOGOUT_SESS:
5244                 tag = beiscsi_boot_logout_sess(phba);
5245                 break;
5246         case BEISCSI_BOOT_CREATE_KSET:
5247                 beiscsi_boot_create_kset(phba);
5248                 /**
5249                  * updated boot_kset is made visible to all before
5250                  * ending the boot work.
5251                  */
5252                 mb();
5253                 clear_bit(BEISCSI_HBA_BOOT_WORK, &phba->state);
5254                 return;
5255         }
5256         if (!tag) {
5257                 if (bs->retry--)
5258                         schedule_work(&phba->boot_work);
5259                 else
5260                         clear_bit(BEISCSI_HBA_BOOT_WORK, &phba->state);
5261         }
5262 }
5263
5264 static void beiscsi_eqd_update_work(struct work_struct *work)
5265 {
5266         struct hwi_context_memory *phwi_context;
5267         struct be_set_eqd set_eqd[MAX_CPUS];
5268         struct hwi_controller *phwi_ctrlr;
5269         struct be_eq_obj *pbe_eq;
5270         struct beiscsi_hba *phba;
5271         unsigned int pps, delta;
5272         struct be_aic_obj *aic;
5273         int eqd, i, num = 0;
5274         unsigned long now;
5275
5276         phba = container_of(work, struct beiscsi_hba, eqd_update.work);
5277         if (!beiscsi_hba_is_online(phba))
5278                 return;
5279
5280         phwi_ctrlr = phba->phwi_ctrlr;
5281         phwi_context = phwi_ctrlr->phwi_ctxt;
5282
5283         for (i = 0; i <= phba->num_cpus; i++) {
5284                 aic = &phba->aic_obj[i];
5285                 pbe_eq = &phwi_context->be_eq[i];
5286                 now = jiffies;
5287                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
5288                     pbe_eq->cq_count < aic->eq_prev) {
5289                         aic->jiffies = now;
5290                         aic->eq_prev = pbe_eq->cq_count;
5291                         continue;
5292                 }
5293                 delta = jiffies_to_msecs(now - aic->jiffies);
5294                 pps = (((u32)(pbe_eq->cq_count - aic->eq_prev) * 1000) / delta);
5295                 eqd = (pps / 1500) << 2;
5296
5297                 if (eqd < 8)
5298                         eqd = 0;
5299                 eqd = min_t(u32, eqd, phwi_context->max_eqd);
5300                 eqd = max_t(u32, eqd, phwi_context->min_eqd);
5301
5302                 aic->jiffies = now;
5303                 aic->eq_prev = pbe_eq->cq_count;
5304
5305                 if (eqd != aic->prev_eqd) {
5306                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
5307                         set_eqd[num].eq_id = pbe_eq->q.id;
5308                         aic->prev_eqd = eqd;
5309                         num++;
5310                 }
5311         }
5312         if (num)
5313                 /* completion of this is ignored */
5314                 beiscsi_modify_eq_delay(phba, set_eqd, num);
5315
5316         schedule_delayed_work(&phba->eqd_update,
5317                               msecs_to_jiffies(BEISCSI_EQD_UPDATE_INTERVAL));
5318 }
5319
5320 static void beiscsi_msix_enable(struct beiscsi_hba *phba)
5321 {
5322         int i, status;
5323
5324         for (i = 0; i <= phba->num_cpus; i++)
5325                 phba->msix_entries[i].entry = i;
5326
5327         status = pci_enable_msix_range(phba->pcidev, phba->msix_entries,
5328                                        phba->num_cpus + 1, phba->num_cpus + 1);
5329         if (status > 0)
5330                 phba->msix_enabled = true;
5331 }
5332
5333 static void beiscsi_hw_tpe_check(unsigned long ptr)
5334 {
5335         struct beiscsi_hba *phba;
5336         u32 wait;
5337
5338         phba = (struct beiscsi_hba *)ptr;
5339         /* if not TPE, do nothing */
5340         if (!beiscsi_detect_tpe(phba))
5341                 return;
5342
5343         /* wait default 4000ms before recovering */
5344         wait = 4000;
5345         if (phba->ue2rp > BEISCSI_UE_DETECT_INTERVAL)
5346                 wait = phba->ue2rp - BEISCSI_UE_DETECT_INTERVAL;
5347         queue_delayed_work(phba->wq, &phba->recover_port,
5348                            msecs_to_jiffies(wait));
5349 }
5350
5351 static void beiscsi_hw_health_check(unsigned long ptr)
5352 {
5353         struct beiscsi_hba *phba;
5354
5355         phba = (struct beiscsi_hba *)ptr;
5356         beiscsi_detect_ue(phba);
5357         if (beiscsi_detect_ue(phba)) {
5358                 __beiscsi_log(phba, KERN_ERR,
5359                               "BM_%d : port in error: %lx\n", phba->state);
5360                 /* sessions are no longer valid, so first fail the sessions */
5361                 queue_work(phba->wq, &phba->sess_work);
5362
5363                 /* detect UER supported */
5364                 if (!test_bit(BEISCSI_HBA_UER_SUPP, &phba->state))
5365                         return;
5366                 /* modify this timer to check TPE */
5367                 phba->hw_check.function = beiscsi_hw_tpe_check;
5368         }
5369
5370         mod_timer(&phba->hw_check,
5371                   jiffies + msecs_to_jiffies(BEISCSI_UE_DETECT_INTERVAL));
5372 }
5373
5374 /*
5375  * beiscsi_enable_port()- Enables the disabled port.
5376  * Only port resources freed in disable function are reallocated.
5377  * This is called in HBA error handling path.
5378  *
5379  * @phba: Instance of driver private structure
5380  *
5381  **/
5382 static int beiscsi_enable_port(struct beiscsi_hba *phba)
5383 {
5384         struct hwi_context_memory *phwi_context;
5385         struct hwi_controller *phwi_ctrlr;
5386         struct be_eq_obj *pbe_eq;
5387         int ret, i;
5388
5389         if (test_bit(BEISCSI_HBA_ONLINE, &phba->state)) {
5390                 __beiscsi_log(phba, KERN_ERR,
5391                               "BM_%d : %s : port is online %lx\n",
5392                               __func__, phba->state);
5393                 return 0;
5394         }
5395
5396         ret = beiscsi_init_sliport(phba);
5397         if (ret)
5398                 return ret;
5399
5400         if (enable_msix)
5401                 find_num_cpus(phba);
5402         else
5403                 phba->num_cpus = 1;
5404         if (enable_msix) {
5405                 beiscsi_msix_enable(phba);
5406                 if (!phba->msix_enabled)
5407                         phba->num_cpus = 1;
5408         }
5409
5410         beiscsi_get_params(phba);
5411         /* Re-enable UER. If different TPE occurs then it is recoverable. */
5412         beiscsi_set_uer_feature(phba);
5413
5414         phba->shost->max_id = phba->params.cxns_per_ctrl;
5415         phba->shost->can_queue = phba->params.ios_per_ctrl;
5416         ret = hwi_init_controller(phba);
5417         if (ret) {
5418                 __beiscsi_log(phba, KERN_ERR,
5419                               "BM_%d : init controller failed %d\n", ret);
5420                 goto disable_msix;
5421         }
5422
5423         for (i = 0; i < MAX_MCC_CMD; i++) {
5424                 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
5425                 phba->ctrl.mcc_tag[i] = i + 1;
5426                 phba->ctrl.mcc_tag_status[i + 1] = 0;
5427                 phba->ctrl.mcc_tag_available++;
5428         }
5429
5430         phwi_ctrlr = phba->phwi_ctrlr;
5431         phwi_context = phwi_ctrlr->phwi_ctxt;
5432         for (i = 0; i < phba->num_cpus; i++) {
5433                 pbe_eq = &phwi_context->be_eq[i];
5434                 irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget, be_iopoll);
5435         }
5436
5437         i = (phba->msix_enabled) ? i : 0;
5438         /* Work item for MCC handling */
5439         pbe_eq = &phwi_context->be_eq[i];
5440         INIT_WORK(&pbe_eq->mcc_work, beiscsi_mcc_work);
5441
5442         ret = beiscsi_init_irqs(phba);
5443         if (ret < 0) {
5444                 __beiscsi_log(phba, KERN_ERR,
5445                               "BM_%d : setup IRQs failed %d\n", ret);
5446                 goto cleanup_port;
5447         }
5448         hwi_enable_intr(phba);
5449         /* port operational: clear all error bits */
5450         set_bit(BEISCSI_HBA_ONLINE, &phba->state);
5451         __beiscsi_log(phba, KERN_INFO,
5452                       "BM_%d : port online: 0x%lx\n", phba->state);
5453
5454         /* start hw_check timer and eqd_update work */
5455         schedule_delayed_work(&phba->eqd_update,
5456                               msecs_to_jiffies(BEISCSI_EQD_UPDATE_INTERVAL));
5457
5458         /**
5459          * Timer function gets modified for TPE detection.
5460          * Always reinit to do health check first.
5461          */
5462         phba->hw_check.function = beiscsi_hw_health_check;
5463         mod_timer(&phba->hw_check,
5464                   jiffies + msecs_to_jiffies(BEISCSI_UE_DETECT_INTERVAL));
5465         return 0;
5466
5467 cleanup_port:
5468         for (i = 0; i < phba->num_cpus; i++) {
5469                 pbe_eq = &phwi_context->be_eq[i];
5470                 irq_poll_disable(&pbe_eq->iopoll);
5471         }
5472         hwi_cleanup_port(phba);
5473
5474 disable_msix:
5475         if (phba->msix_enabled)
5476                 pci_disable_msix(phba->pcidev);
5477
5478         return ret;
5479 }
5480
5481 /*
5482  * beiscsi_disable_port()- Disable port and cleanup driver resources.
5483  * This is called in HBA error handling and driver removal.
5484  * @phba: Instance Priv structure
5485  * @unload: indicate driver is unloading
5486  *
5487  * Free the OS and HW resources held by the driver
5488  **/
5489 static void beiscsi_disable_port(struct beiscsi_hba *phba, int unload)
5490 {
5491         struct hwi_context_memory *phwi_context;
5492         struct hwi_controller *phwi_ctrlr;
5493         struct be_eq_obj *pbe_eq;
5494         unsigned int i, msix_vec;
5495
5496         if (!test_and_clear_bit(BEISCSI_HBA_ONLINE, &phba->state))
5497                 return;
5498
5499         phwi_ctrlr = phba->phwi_ctrlr;
5500         phwi_context = phwi_ctrlr->phwi_ctxt;
5501         hwi_disable_intr(phba);
5502         if (phba->msix_enabled) {
5503                 for (i = 0; i <= phba->num_cpus; i++) {
5504                         msix_vec = phba->msix_entries[i].vector;
5505                         free_irq(msix_vec, &phwi_context->be_eq[i]);
5506                         kfree(phba->msi_name[i]);
5507                 }
5508         } else
5509                 if (phba->pcidev->irq)
5510                         free_irq(phba->pcidev->irq, phba);
5511         pci_disable_msix(phba->pcidev);
5512
5513         for (i = 0; i < phba->num_cpus; i++) {
5514                 pbe_eq = &phwi_context->be_eq[i];
5515                 irq_poll_disable(&pbe_eq->iopoll);
5516         }
5517         cancel_delayed_work_sync(&phba->eqd_update);
5518         cancel_work_sync(&phba->boot_work);
5519         /* WQ might be running cancel queued mcc_work if we are not exiting */
5520         if (!unload && beiscsi_hba_in_error(phba)) {
5521                 pbe_eq = &phwi_context->be_eq[i];
5522                 cancel_work_sync(&pbe_eq->mcc_work);
5523         }
5524         hwi_cleanup_port(phba);
5525 }
5526
5527 static void beiscsi_sess_work(struct work_struct *work)
5528 {
5529         struct beiscsi_hba *phba;
5530
5531         phba = container_of(work, struct beiscsi_hba, sess_work);
5532         /*
5533          * This work gets scheduled only in case of HBA error.
5534          * Old sessions are gone so need to be re-established.
5535          * iscsi_session_failure needs process context hence this work.
5536          */
5537         iscsi_host_for_each_session(phba->shost, beiscsi_session_fail);
5538 }
5539
5540 static void beiscsi_recover_port(struct work_struct *work)
5541 {
5542         struct beiscsi_hba *phba;
5543
5544         phba = container_of(work, struct beiscsi_hba, recover_port.work);
5545         beiscsi_disable_port(phba, 0);
5546         beiscsi_enable_port(phba);
5547 }
5548
5549 static pci_ers_result_t beiscsi_eeh_err_detected(struct pci_dev *pdev,
5550                 pci_channel_state_t state)
5551 {
5552         struct beiscsi_hba *phba = NULL;
5553
5554         phba = (struct beiscsi_hba *)pci_get_drvdata(pdev);
5555         set_bit(BEISCSI_HBA_PCI_ERR, &phba->state);
5556
5557         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5558                     "BM_%d : EEH error detected\n");
5559
5560         /* first stop UE detection when PCI error detected */
5561         del_timer_sync(&phba->hw_check);
5562         cancel_delayed_work_sync(&phba->recover_port);
5563
5564         /* sessions are no longer valid, so first fail the sessions */
5565         iscsi_host_for_each_session(phba->shost, beiscsi_session_fail);
5566         beiscsi_disable_port(phba, 0);
5567
5568         if (state == pci_channel_io_perm_failure) {
5569                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5570                             "BM_%d : EEH : State PERM Failure");
5571                 return PCI_ERS_RESULT_DISCONNECT;
5572         }
5573
5574         pci_disable_device(pdev);
5575
5576         /* The error could cause the FW to trigger a flash debug dump.
5577          * Resetting the card while flash dump is in progress
5578          * can cause it not to recover; wait for it to finish.
5579          * Wait only for first function as it is needed only once per
5580          * adapter.
5581          **/
5582         if (pdev->devfn == 0)
5583                 ssleep(30);
5584
5585         return PCI_ERS_RESULT_NEED_RESET;
5586 }
5587
5588 static pci_ers_result_t beiscsi_eeh_reset(struct pci_dev *pdev)
5589 {
5590         struct beiscsi_hba *phba = NULL;
5591         int status = 0;
5592
5593         phba = (struct beiscsi_hba *)pci_get_drvdata(pdev);
5594
5595         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5596                     "BM_%d : EEH Reset\n");
5597
5598         status = pci_enable_device(pdev);
5599         if (status)
5600                 return PCI_ERS_RESULT_DISCONNECT;
5601
5602         pci_set_master(pdev);
5603         pci_set_power_state(pdev, PCI_D0);
5604         pci_restore_state(pdev);
5605
5606         status = beiscsi_check_fw_rdy(phba);
5607         if (status) {
5608                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
5609                             "BM_%d : EEH Reset Completed\n");
5610         } else {
5611                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
5612                             "BM_%d : EEH Reset Completion Failure\n");
5613                 return PCI_ERS_RESULT_DISCONNECT;
5614         }
5615
5616         pci_cleanup_aer_uncorrect_error_status(pdev);
5617         return PCI_ERS_RESULT_RECOVERED;
5618 }
5619
5620 static void beiscsi_eeh_resume(struct pci_dev *pdev)
5621 {
5622         struct beiscsi_hba *phba;
5623         int ret;
5624
5625         phba = (struct beiscsi_hba *)pci_get_drvdata(pdev);
5626         pci_save_state(pdev);
5627
5628         ret = beiscsi_enable_port(phba);
5629         if (ret)
5630                 __beiscsi_log(phba, KERN_ERR,
5631                               "BM_%d : AER EEH resume failed\n");
5632 }
5633
5634 static int beiscsi_dev_probe(struct pci_dev *pcidev,
5635                              const struct pci_device_id *id)
5636 {
5637         struct beiscsi_hba *phba = NULL;
5638         struct hwi_controller *phwi_ctrlr;
5639         struct hwi_context_memory *phwi_context;
5640         struct be_eq_obj *pbe_eq;
5641         unsigned int s_handle;
5642         int ret, i;
5643
5644         ret = beiscsi_enable_pci(pcidev);
5645         if (ret < 0) {
5646                 dev_err(&pcidev->dev,
5647                         "beiscsi_dev_probe - Failed to enable pci device\n");
5648                 return ret;
5649         }
5650
5651         phba = beiscsi_hba_alloc(pcidev);
5652         if (!phba) {
5653                 dev_err(&pcidev->dev,
5654                         "beiscsi_dev_probe - Failed in beiscsi_hba_alloc\n");
5655                 ret = -ENOMEM;
5656                 goto disable_pci;
5657         }
5658
5659         /* Enable EEH reporting */
5660         ret = pci_enable_pcie_error_reporting(pcidev);
5661         if (ret)
5662                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
5663                             "BM_%d : PCIe Error Reporting "
5664                             "Enabling Failed\n");
5665
5666         pci_save_state(pcidev);
5667
5668         /* Initialize Driver configuration Paramters */
5669         beiscsi_hba_attrs_init(phba);
5670
5671         phba->mac_addr_set = false;
5672
5673         switch (pcidev->device) {
5674         case BE_DEVICE_ID1:
5675         case OC_DEVICE_ID1:
5676         case OC_DEVICE_ID2:
5677                 phba->generation = BE_GEN2;
5678                 phba->iotask_fn = beiscsi_iotask;
5679                 break;
5680         case BE_DEVICE_ID2:
5681         case OC_DEVICE_ID3:
5682                 phba->generation = BE_GEN3;
5683                 phba->iotask_fn = beiscsi_iotask;
5684                 break;
5685         case OC_SKH_ID1:
5686                 phba->generation = BE_GEN4;
5687                 phba->iotask_fn = beiscsi_iotask_v2;
5688                 break;
5689         default:
5690                 phba->generation = 0;
5691         }
5692
5693         ret = be_ctrl_init(phba, pcidev);
5694         if (ret) {
5695                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5696                             "BM_%d : be_ctrl_init failed\n");
5697                 goto hba_free;
5698         }
5699
5700         ret = beiscsi_init_sliport(phba);
5701         if (ret)
5702                 goto hba_free;
5703
5704         spin_lock_init(&phba->io_sgl_lock);
5705         spin_lock_init(&phba->mgmt_sgl_lock);
5706         spin_lock_init(&phba->async_pdu_lock);
5707         ret = beiscsi_get_fw_config(&phba->ctrl, phba);
5708         if (ret != 0) {
5709                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5710                             "BM_%d : Error getting fw config\n");
5711                 goto free_port;
5712         }
5713         beiscsi_get_port_name(&phba->ctrl, phba);
5714         beiscsi_get_params(phba);
5715         beiscsi_set_uer_feature(phba);
5716
5717         if (enable_msix)
5718                 find_num_cpus(phba);
5719         else
5720                 phba->num_cpus = 1;
5721
5722         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
5723                     "BM_%d : num_cpus = %d\n",
5724                     phba->num_cpus);
5725
5726         if (enable_msix) {
5727                 beiscsi_msix_enable(phba);
5728                 if (!phba->msix_enabled)
5729                         phba->num_cpus = 1;
5730         }
5731
5732         phba->shost->max_id = phba->params.cxns_per_ctrl;
5733         phba->shost->can_queue = phba->params.ios_per_ctrl;
5734         ret = beiscsi_init_port(phba);
5735         if (ret < 0) {
5736                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5737                             "BM_%d : beiscsi_dev_probe-"
5738                             "Failed in beiscsi_init_port\n");
5739                 goto free_port;
5740         }
5741
5742         for (i = 0; i < MAX_MCC_CMD; i++) {
5743                 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
5744                 phba->ctrl.mcc_tag[i] = i + 1;
5745                 phba->ctrl.mcc_tag_status[i + 1] = 0;
5746                 phba->ctrl.mcc_tag_available++;
5747                 memset(&phba->ctrl.ptag_state[i].tag_mem_state, 0,
5748                        sizeof(struct be_dma_mem));
5749         }
5750
5751         phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
5752
5753         snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_%02x_wq",
5754                  phba->shost->host_no);
5755         phba->wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 1, phba->wq_name);
5756         if (!phba->wq) {
5757                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5758                             "BM_%d : beiscsi_dev_probe-"
5759                             "Failed to allocate work queue\n");
5760                 ret = -ENOMEM;
5761                 goto free_twq;
5762         }
5763
5764         INIT_DELAYED_WORK(&phba->eqd_update, beiscsi_eqd_update_work);
5765
5766         phwi_ctrlr = phba->phwi_ctrlr;
5767         phwi_context = phwi_ctrlr->phwi_ctxt;
5768
5769         for (i = 0; i < phba->num_cpus; i++) {
5770                 pbe_eq = &phwi_context->be_eq[i];
5771                 irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget, be_iopoll);
5772         }
5773
5774         i = (phba->msix_enabled) ? i : 0;
5775         /* Work item for MCC handling */
5776         pbe_eq = &phwi_context->be_eq[i];
5777         INIT_WORK(&pbe_eq->mcc_work, beiscsi_mcc_work);
5778
5779         ret = beiscsi_init_irqs(phba);
5780         if (ret < 0) {
5781                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5782                             "BM_%d : beiscsi_dev_probe-"
5783                             "Failed to beiscsi_init_irqs\n");
5784                 goto free_blkenbld;
5785         }
5786         hwi_enable_intr(phba);
5787
5788         ret = iscsi_host_add(phba->shost, &phba->pcidev->dev);
5789         if (ret)
5790                 goto free_blkenbld;
5791
5792         /* set online bit after port is operational */
5793         set_bit(BEISCSI_HBA_ONLINE, &phba->state);
5794         __beiscsi_log(phba, KERN_INFO,
5795                       "BM_%d : port online: 0x%lx\n", phba->state);
5796
5797         INIT_WORK(&phba->boot_work, beiscsi_boot_work);
5798         ret = beiscsi_boot_get_shandle(phba, &s_handle);
5799         if (ret > 0) {
5800                 beiscsi_start_boot_work(phba, s_handle);
5801                 /**
5802                  * Set this bit after starting the work to let
5803                  * probe handle it first.
5804                  * ASYNC event can too schedule this work.
5805                  */
5806                 set_bit(BEISCSI_HBA_BOOT_FOUND, &phba->state);
5807         }
5808
5809         beiscsi_iface_create_default(phba);
5810         schedule_delayed_work(&phba->eqd_update,
5811                               msecs_to_jiffies(BEISCSI_EQD_UPDATE_INTERVAL));
5812
5813         INIT_WORK(&phba->sess_work, beiscsi_sess_work);
5814         INIT_DELAYED_WORK(&phba->recover_port, beiscsi_recover_port);
5815         /**
5816          * Start UE detection here. UE before this will cause stall in probe
5817          * and eventually fail the probe.
5818          */
5819         init_timer(&phba->hw_check);
5820         phba->hw_check.function = beiscsi_hw_health_check;
5821         phba->hw_check.data = (unsigned long)phba;
5822         mod_timer(&phba->hw_check,
5823                   jiffies + msecs_to_jiffies(BEISCSI_UE_DETECT_INTERVAL));
5824         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
5825                     "\n\n\n BM_%d : SUCCESS - DRIVER LOADED\n\n\n");
5826         return 0;
5827
5828 free_blkenbld:
5829         destroy_workqueue(phba->wq);
5830         for (i = 0; i < phba->num_cpus; i++) {
5831                 pbe_eq = &phwi_context->be_eq[i];
5832                 irq_poll_disable(&pbe_eq->iopoll);
5833         }
5834 free_twq:
5835         hwi_cleanup_port(phba);
5836         beiscsi_cleanup_port(phba);
5837         beiscsi_free_mem(phba);
5838 free_port:
5839         pci_free_consistent(phba->pcidev,
5840                             phba->ctrl.mbox_mem_alloced.size,
5841                             phba->ctrl.mbox_mem_alloced.va,
5842                            phba->ctrl.mbox_mem_alloced.dma);
5843         beiscsi_unmap_pci_function(phba);
5844 hba_free:
5845         if (phba->msix_enabled)
5846                 pci_disable_msix(phba->pcidev);
5847         pci_dev_put(phba->pcidev);
5848         iscsi_host_free(phba->shost);
5849         pci_set_drvdata(pcidev, NULL);
5850 disable_pci:
5851         pci_release_regions(pcidev);
5852         pci_disable_device(pcidev);
5853         return ret;
5854 }
5855
5856 static void beiscsi_remove(struct pci_dev *pcidev)
5857 {
5858         struct beiscsi_hba *phba = NULL;
5859
5860         phba = pci_get_drvdata(pcidev);
5861         if (!phba) {
5862                 dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");
5863                 return;
5864         }
5865
5866         /* first stop UE detection before unloading */
5867         del_timer_sync(&phba->hw_check);
5868         cancel_delayed_work_sync(&phba->recover_port);
5869         cancel_work_sync(&phba->sess_work);
5870
5871         beiscsi_iface_destroy_default(phba);
5872         iscsi_host_remove(phba->shost);
5873         beiscsi_disable_port(phba, 1);
5874
5875         /* after cancelling boot_work */
5876         iscsi_boot_destroy_kset(phba->boot_struct.boot_kset);
5877
5878         /* free all resources */
5879         destroy_workqueue(phba->wq);
5880         beiscsi_cleanup_port(phba);
5881         beiscsi_free_mem(phba);
5882
5883         /* ctrl uninit */
5884         beiscsi_unmap_pci_function(phba);
5885         pci_free_consistent(phba->pcidev,
5886                             phba->ctrl.mbox_mem_alloced.size,
5887                             phba->ctrl.mbox_mem_alloced.va,
5888                             phba->ctrl.mbox_mem_alloced.dma);
5889
5890         pci_dev_put(phba->pcidev);
5891         iscsi_host_free(phba->shost);
5892         pci_disable_pcie_error_reporting(pcidev);
5893         pci_set_drvdata(pcidev, NULL);
5894         pci_release_regions(pcidev);
5895         pci_disable_device(pcidev);
5896 }
5897
5898
5899 static struct pci_error_handlers beiscsi_eeh_handlers = {
5900         .error_detected = beiscsi_eeh_err_detected,
5901         .slot_reset = beiscsi_eeh_reset,
5902         .resume = beiscsi_eeh_resume,
5903 };
5904
5905 struct iscsi_transport beiscsi_iscsi_transport = {
5906         .owner = THIS_MODULE,
5907         .name = DRV_NAME,
5908         .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
5909                 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
5910         .create_session = beiscsi_session_create,
5911         .destroy_session = beiscsi_session_destroy,
5912         .create_conn = beiscsi_conn_create,
5913         .bind_conn = beiscsi_conn_bind,
5914         .destroy_conn = iscsi_conn_teardown,
5915         .attr_is_visible = beiscsi_attr_is_visible,
5916         .set_iface_param = beiscsi_iface_set_param,
5917         .get_iface_param = beiscsi_iface_get_param,
5918         .set_param = beiscsi_set_param,
5919         .get_conn_param = iscsi_conn_get_param,
5920         .get_session_param = iscsi_session_get_param,
5921         .get_host_param = beiscsi_get_host_param,
5922         .start_conn = beiscsi_conn_start,
5923         .stop_conn = iscsi_conn_stop,
5924         .send_pdu = iscsi_conn_send_pdu,
5925         .xmit_task = beiscsi_task_xmit,
5926         .cleanup_task = beiscsi_cleanup_task,
5927         .alloc_pdu = beiscsi_alloc_pdu,
5928         .parse_pdu_itt = beiscsi_parse_pdu,
5929         .get_stats = beiscsi_conn_get_stats,
5930         .get_ep_param = beiscsi_ep_get_param,
5931         .ep_connect = beiscsi_ep_connect,
5932         .ep_poll = beiscsi_ep_poll,
5933         .ep_disconnect = beiscsi_ep_disconnect,
5934         .session_recovery_timedout = iscsi_session_recovery_timedout,
5935         .bsg_request = beiscsi_bsg_request,
5936 };
5937
5938 static struct pci_driver beiscsi_pci_driver = {
5939         .name = DRV_NAME,
5940         .probe = beiscsi_dev_probe,
5941         .remove = beiscsi_remove,
5942         .id_table = beiscsi_pci_id_table,
5943         .err_handler = &beiscsi_eeh_handlers
5944 };
5945
5946 static int __init beiscsi_module_init(void)
5947 {
5948         int ret;
5949
5950         beiscsi_scsi_transport =
5951                         iscsi_register_transport(&beiscsi_iscsi_transport);
5952         if (!beiscsi_scsi_transport) {
5953                 printk(KERN_ERR
5954                        "beiscsi_module_init - Unable to  register beiscsi transport.\n");
5955                 return -ENOMEM;
5956         }
5957         printk(KERN_INFO "In beiscsi_module_init, tt=%p\n",
5958                &beiscsi_iscsi_transport);
5959
5960         ret = pci_register_driver(&beiscsi_pci_driver);
5961         if (ret) {
5962                 printk(KERN_ERR
5963                        "beiscsi_module_init - Unable to  register beiscsi pci driver.\n");
5964                 goto unregister_iscsi_transport;
5965         }
5966         return 0;
5967
5968 unregister_iscsi_transport:
5969         iscsi_unregister_transport(&beiscsi_iscsi_transport);
5970         return ret;
5971 }
5972
5973 static void __exit beiscsi_module_exit(void)
5974 {
5975         pci_unregister_driver(&beiscsi_pci_driver);
5976         iscsi_unregister_transport(&beiscsi_iscsi_transport);
5977 }
5978
5979 module_init(beiscsi_module_init);
5980 module_exit(beiscsi_module_exit);