drivers/scsi/be2iscsi/be_main.c

   1 /**
   2  * Copyright (C) 2005 - 2010 ServerEngines
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Written by: Jayamohan Kallickal (jayamohank@serverengines.com)
  11  *
  12  * Contact Information:
  13  * linux-drivers@serverengines.com
  14  *
  15  *  ServerEngines
  16  * 209 N. Fair Oaks Ave
  17  * Sunnyvale, CA 94085
  18  *
  19  */
  20 #include <linux/reboot.h>
  21 #include <linux/delay.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/blkdev.h>
  24 #include <linux/pci.h>
  25 #include <linux/string.h>
  26 #include <linux/kernel.h>
  27 #include <linux/semaphore.h>
  28
  29 #include <scsi/libiscsi.h>
  30 #include <scsi/scsi_transport_iscsi.h>
  31 #include <scsi/scsi_transport.h>
  32 #include <scsi/scsi_cmnd.h>
  33 #include <scsi/scsi_device.h>
  34 #include <scsi/scsi_host.h>
  35 #include <scsi/scsi.h>
  36 #include "be_main.h"
  37 #include "be_iscsi.h"
  38 #include "be_mgmt.h"
  39
  40 static unsigned int be_iopoll_budget = 10;
  41 static unsigned int be_max_phys_size = 64;
  42 static unsigned int enable_msix = 1;
  43
  44 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
  45 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
  46 MODULE_AUTHOR("ServerEngines Corporation");
  47 MODULE_LICENSE("GPL");
  48 module_param(be_iopoll_budget, int, 0);
  49 module_param(enable_msix, int, 0);
  50 module_param(be_max_phys_size, uint, S_IRUGO);
  51 MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
  52                                    "contiguous memory that can be allocated."
  53                                    "Range is 16 - 128");
  54
  55 static int beiscsi_slave_configure(struct scsi_device *sdev)
  56 {
  57         blk_queue_max_segment_size(sdev->request_queue, 65536);
  58         return 0;
  59 }
  60
  61 static int beiscsi_eh_abort(struct scsi_cmnd *sc)
  62 {
  63         struct iscsi_cls_session *cls_session;
  64         struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
  65         struct beiscsi_io_task *aborted_io_task;
  66         struct iscsi_conn *conn;
  67         struct beiscsi_conn *beiscsi_conn;
  68         struct beiscsi_hba *phba;
  69         struct iscsi_session *session;
  70         struct invalidate_command_table *inv_tbl;
  71         unsigned int cid, tag, num_invalidate;
  72
  73         cls_session = starget_to_session(scsi_target(sc->device));
  74         session = cls_session->dd_data;
  75
  76         spin_lock_bh(&session->lock);
  77         if (!aborted_task || !aborted_task->sc) {
  78                 /* we raced */
  79                 spin_unlock_bh(&session->lock);
  80                 return SUCCESS;
  81         }
  82
  83         aborted_io_task = aborted_task->dd_data;
  84         if (!aborted_io_task->scsi_cmnd) {
  85                 /* raced or invalid command */
  86                 spin_unlock_bh(&session->lock);
  87                 return SUCCESS;
  88         }
  89         spin_unlock_bh(&session->lock);
  90         conn = aborted_task->conn;
  91         beiscsi_conn = conn->dd_data;
  92         phba = beiscsi_conn->phba;
  93
  94         /* invalidate iocb */
  95         cid = beiscsi_conn->beiscsi_conn_cid;
  96         inv_tbl = phba->inv_tbl;
  97         memset(inv_tbl, 0x0, sizeof(*inv_tbl));
  98         inv_tbl->cid = cid;
  99         inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
 100         num_invalidate = 1;
 101         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate, cid);
 102         if (!tag) {
 103                 shost_printk(KERN_WARNING, phba->shost,
 104                              "mgmt_invalidate_icds could not be"
 105                              " submitted\n");
 106                 return FAILED;
 107         } else {
 108                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
 109                                          phba->ctrl.mcc_numtag[tag]);
 110                 free_mcc_tag(&phba->ctrl, tag);
 111         }
 112
 113         return iscsi_eh_abort(sc);
 114 }
 115
 116 static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
 117 {
 118         struct iscsi_task *abrt_task;
 119         struct beiscsi_io_task *abrt_io_task;
 120         struct iscsi_conn *conn;
 121         struct beiscsi_conn *beiscsi_conn;
 122         struct beiscsi_hba *phba;
 123         struct iscsi_session *session;
 124         struct iscsi_cls_session *cls_session;
 125         struct invalidate_command_table *inv_tbl;
 126         unsigned int cid, tag, i, num_invalidate;
 127         int rc = FAILED;
 128
 129         /* invalidate iocbs */
 130         cls_session = starget_to_session(scsi_target(sc->device));
 131         session = cls_session->dd_data;
 132         spin_lock_bh(&session->lock);
 133         if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN)
 134                 goto unlock;
 135
 136         conn = session->leadconn;
 137         beiscsi_conn = conn->dd_data;
 138         phba = beiscsi_conn->phba;
 139         cid = beiscsi_conn->beiscsi_conn_cid;
 140         inv_tbl = phba->inv_tbl;
 141         memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
 142         num_invalidate = 0;
 143         for (i = 0; i < conn->session->cmds_max; i++) {
 144                 abrt_task = conn->session->cmds[i];
 145                 abrt_io_task = abrt_task->dd_data;
 146                 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
 147                         continue;
 148
 149                 if (abrt_task->sc->device->lun != abrt_task->sc->device->lun)
 150                         continue;
 151
 152                 inv_tbl->cid = cid;
 153                 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
 154                 num_invalidate++;
 155                 inv_tbl++;
 156         }
 157         spin_unlock_bh(&session->lock);
 158         inv_tbl = phba->inv_tbl;
 159
 160         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate, cid);
 161         if (!tag) {
 162                 shost_printk(KERN_WARNING, phba->shost,
 163                              "mgmt_invalidate_icds could not be"
 164                              " submitted\n");
 165                 return FAILED;
 166         } else {
 167                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
 168                                          phba->ctrl.mcc_numtag[tag]);
 169                 free_mcc_tag(&phba->ctrl, tag);
 170         }
 171
 172         return iscsi_eh_device_reset(sc);
 173 unlock:
 174         spin_unlock_bh(&session->lock);
 175         return rc;
 176 }
 177
 178 /*------------------- PCI Driver operations and data ----------------- */
 179 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
 180         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
 181         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
 182         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
 183         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
 184         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
 185         { 0 }
 186 };
 187 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
 188
 189 static struct scsi_host_template beiscsi_sht = {
 190         .module = THIS_MODULE,
 191         .name = "ServerEngines 10Gbe open-iscsi Initiator Driver",
 192         .proc_name = DRV_NAME,
 193         .queuecommand = iscsi_queuecommand,
 194         .change_queue_depth = iscsi_change_queue_depth,
 195         .slave_configure = beiscsi_slave_configure,
 196         .target_alloc = iscsi_target_alloc,
 197         .eh_abort_handler = beiscsi_eh_abort,
 198         .eh_device_reset_handler = beiscsi_eh_device_reset,
 199         .eh_target_reset_handler = iscsi_eh_session_reset,
 200         .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
 201         .can_queue = BE2_IO_DEPTH,
 202         .this_id = -1,
 203         .max_sectors = BEISCSI_MAX_SECTORS,
 204         .cmd_per_lun = BEISCSI_CMD_PER_LUN,
 205         .use_clustering = ENABLE_CLUSTERING,
 206 };
 207
 208 static struct scsi_transport_template *beiscsi_scsi_transport;
 209
 210 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
 211 {
 212         struct beiscsi_hba *phba;
 213         struct Scsi_Host *shost;
 214
 215         shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
 216         if (!shost) {
 217                 dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
 218                         "iscsi_host_alloc failed \n");
 219                 return NULL;
 220         }
 221         shost->dma_boundary = pcidev->dma_mask;
 222         shost->max_id = BE2_MAX_SESSIONS;
 223         shost->max_channel = 0;
 224         shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
 225         shost->max_lun = BEISCSI_NUM_MAX_LUN;
 226         shost->transportt = beiscsi_scsi_transport;
 227         phba = iscsi_host_priv(shost);
 228         memset(phba, 0, sizeof(*phba));
 229         phba->shost = shost;
 230         phba->pcidev = pci_dev_get(pcidev);
 231         pci_set_drvdata(pcidev, phba);
 232
 233         if (iscsi_host_add(shost, &phba->pcidev->dev))
 234                 goto free_devices;
 235         return phba;
 236
 237 free_devices:
 238         pci_dev_put(phba->pcidev);
 239         iscsi_host_free(phba->shost);
 240         return NULL;
 241 }
 242
 243 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
 244 {
 245         if (phba->csr_va) {
 246                 iounmap(phba->csr_va);
 247                 phba->csr_va = NULL;
 248         }
 249         if (phba->db_va) {
 250                 iounmap(phba->db_va);
 251                 phba->db_va = NULL;
 252         }
 253         if (phba->pci_va) {
 254                 iounmap(phba->pci_va);
 255                 phba->pci_va = NULL;
 256         }
 257 }
 258
 259 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
 260                                 struct pci_dev *pcidev)
 261 {
 262         u8 __iomem *addr;
 263         int pcicfg_reg;
 264
 265         addr = ioremap_nocache(pci_resource_start(pcidev, 2),
 266                                pci_resource_len(pcidev, 2));
 267         if (addr == NULL)
 268                 return -ENOMEM;
 269         phba->ctrl.csr = addr;
 270         phba->csr_va = addr;
 271         phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
 272
 273         addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
 274         if (addr == NULL)
 275                 goto pci_map_err;
 276         phba->ctrl.db = addr;
 277         phba->db_va = addr;
 278         phba->db_pa.u.a64.address =  pci_resource_start(pcidev, 4);
 279
 280         if (phba->generation == BE_GEN2)
 281                 pcicfg_reg = 1;
 282         else
 283                 pcicfg_reg = 0;
 284
 285         addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
 286                                pci_resource_len(pcidev, pcicfg_reg));
 287
 288         if (addr == NULL)
 289                 goto pci_map_err;
 290         phba->ctrl.pcicfg = addr;
 291         phba->pci_va = addr;
 292         phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
 293         return 0;
 294
 295 pci_map_err:
 296         beiscsi_unmap_pci_function(phba);
 297         return -ENOMEM;
 298 }
 299
 300 static int beiscsi_enable_pci(struct pci_dev *pcidev)
 301 {
 302         int ret;
 303
 304         ret = pci_enable_device(pcidev);
 305         if (ret) {
 306                 dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
 307                         "failed. Returning -ENODEV\n");
 308                 return ret;
 309         }
 310
 311         pci_set_master(pcidev);
 312         if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
 313                 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
 314                 if (ret) {
 315                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
 316                         pci_disable_device(pcidev);
 317                         return ret;
 318                 }
 319         }
 320         return 0;
 321 }
 322
 323 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
 324 {
 325         struct be_ctrl_info *ctrl = &phba->ctrl;
 326         struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
 327         struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
 328         int status = 0;
 329
 330         ctrl->pdev = pdev;
 331         status = beiscsi_map_pci_bars(phba, pdev);
 332         if (status)
 333                 return status;
 334         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
 335         mbox_mem_alloc->va = pci_alloc_consistent(pdev,
 336                                                   mbox_mem_alloc->size,
 337                                                   &mbox_mem_alloc->dma);
 338         if (!mbox_mem_alloc->va) {
 339                 beiscsi_unmap_pci_function(phba);
 340                 status = -ENOMEM;
 341                 return status;
 342         }
 343
 344         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
 345         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
 346         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
 347         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
 348         spin_lock_init(&ctrl->mbox_lock);
 349         spin_lock_init(&phba->ctrl.mcc_lock);
 350         spin_lock_init(&phba->ctrl.mcc_cq_lock);
 351
 352         return status;
 353 }
 354
 355 static void beiscsi_get_params(struct beiscsi_hba *phba)
 356 {
 357         phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
 358                                     - (phba->fw_config.iscsi_cid_count
 359                                     + BE2_TMFS
 360                                     + BE2_NOPOUT_REQ));
 361         phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
 362         phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2;
 363         phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;;
 364         phba->params.num_sge_per_io = BE2_SGE;
 365         phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
 366         phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
 367         phba->params.eq_timer = 64;
 368         phba->params.num_eq_entries =
 369             (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
 370                                     + BE2_TMFS) / 512) + 1) * 512;
 371         phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
 372                                 ? 1024 : phba->params.num_eq_entries;
 373         SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d \n",
 374                              phba->params.num_eq_entries);
 375         phba->params.num_cq_entries =
 376             (((BE2_CMDS_PER_CXN * 2 +  phba->fw_config.iscsi_cid_count * 2
 377                                     + BE2_TMFS) / 512) + 1) * 512;
 378         phba->params.wrbs_per_cxn = 256;
 379 }
 380
 381 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
 382                            unsigned int id, unsigned int clr_interrupt,
 383                            unsigned int num_processed,
 384                            unsigned char rearm, unsigned char event)
 385 {
 386         u32 val = 0;
 387         val |= id & DB_EQ_RING_ID_MASK;
 388         if (rearm)
 389                 val |= 1 << DB_EQ_REARM_SHIFT;
 390         if (clr_interrupt)
 391                 val |= 1 << DB_EQ_CLR_SHIFT;
 392         if (event)
 393                 val |= 1 << DB_EQ_EVNT_SHIFT;
 394         val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
 395         iowrite32(val, phba->db_va + DB_EQ_OFFSET);
 396 }
 397
 398 /**
 399  * be_isr_mcc - The isr routine of the driver.
 400  * @irq: Not used
 401  * @dev_id: Pointer to host adapter structure
 402  */
 403 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
 404 {
 405         struct beiscsi_hba *phba;
 406         struct be_eq_entry *eqe = NULL;
 407         struct be_queue_info *eq;
 408         struct be_queue_info *mcc;
 409         unsigned int num_eq_processed;
 410         struct be_eq_obj *pbe_eq;
 411         unsigned long flags;
 412
 413         pbe_eq = dev_id;
 414         eq = &pbe_eq->q;
 415         phba =  pbe_eq->phba;
 416         mcc = &phba->ctrl.mcc_obj.cq;
 417         eqe = queue_tail_node(eq);
 418         if (!eqe)
 419                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 420
 421         num_eq_processed = 0;
 422
 423         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 424                                 & EQE_VALID_MASK) {
 425                 if (((eqe->dw[offsetof(struct amap_eq_entry,
 426                      resource_id) / 32] &
 427                      EQE_RESID_MASK) >> 16) == mcc->id) {
 428                         spin_lock_irqsave(&phba->isr_lock, flags);
 429                         phba->todo_mcc_cq = 1;
 430                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 431                 }
 432                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 433                 queue_tail_inc(eq);
 434                 eqe = queue_tail_node(eq);
 435                 num_eq_processed++;
 436         }
 437         if (phba->todo_mcc_cq)
 438                 queue_work(phba->wq, &phba->work_cqs);
 439         if (num_eq_processed)
 440                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
 441
 442         return IRQ_HANDLED;
 443 }
 444
 445 /**
 446  * be_isr_msix - The isr routine of the driver.
 447  * @irq: Not used
 448  * @dev_id: Pointer to host adapter structure
 449  */
 450 static irqreturn_t be_isr_msix(int irq, void *dev_id)
 451 {
 452         struct beiscsi_hba *phba;
 453         struct be_eq_entry *eqe = NULL;
 454         struct be_queue_info *eq;
 455         struct be_queue_info *cq;
 456         unsigned int num_eq_processed;
 457         struct be_eq_obj *pbe_eq;
 458         unsigned long flags;
 459
 460         pbe_eq = dev_id;
 461         eq = &pbe_eq->q;
 462         cq = pbe_eq->cq;
 463         eqe = queue_tail_node(eq);
 464         if (!eqe)
 465                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 466
 467         phba = pbe_eq->phba;
 468         num_eq_processed = 0;
 469         if (blk_iopoll_enabled) {
 470                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 471                                         & EQE_VALID_MASK) {
 472                         if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 473                                 blk_iopoll_sched(&pbe_eq->iopoll);
 474
 475                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 476                         queue_tail_inc(eq);
 477                         eqe = queue_tail_node(eq);
 478                         num_eq_processed++;
 479                 }
 480                 if (num_eq_processed)
 481                         hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
 482
 483                 return IRQ_HANDLED;
 484         } else {
 485                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 486                                                 & EQE_VALID_MASK) {
 487                         spin_lock_irqsave(&phba->isr_lock, flags);
 488                         phba->todo_cq = 1;
 489                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 490                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 491                         queue_tail_inc(eq);
 492                         eqe = queue_tail_node(eq);
 493                         num_eq_processed++;
 494                 }
 495                 if (phba->todo_cq)
 496                         queue_work(phba->wq, &phba->work_cqs);
 497
 498                 if (num_eq_processed)
 499                         hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
 500
 501                 return IRQ_HANDLED;
 502         }
 503 }
 504
 505 /**
 506  * be_isr - The isr routine of the driver.
 507  * @irq: Not used
 508  * @dev_id: Pointer to host adapter structure
 509  */
 510 static irqreturn_t be_isr(int irq, void *dev_id)
 511 {
 512         struct beiscsi_hba *phba;
 513         struct hwi_controller *phwi_ctrlr;
 514         struct hwi_context_memory *phwi_context;
 515         struct be_eq_entry *eqe = NULL;
 516         struct be_queue_info *eq;
 517         struct be_queue_info *cq;
 518         struct be_queue_info *mcc;
 519         unsigned long flags, index;
 520         unsigned int num_mcceq_processed, num_ioeq_processed;
 521         struct be_ctrl_info *ctrl;
 522         struct be_eq_obj *pbe_eq;
 523         int isr;
 524
 525         phba = dev_id;
 526         ctrl = &phba->ctrl;;
 527         isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
 528                        (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
 529         if (!isr)
 530                 return IRQ_NONE;
 531
 532         phwi_ctrlr = phba->phwi_ctrlr;
 533         phwi_context = phwi_ctrlr->phwi_ctxt;
 534         pbe_eq = &phwi_context->be_eq[0];
 535
 536         eq = &phwi_context->be_eq[0].q;
 537         mcc = &phba->ctrl.mcc_obj.cq;
 538         index = 0;
 539         eqe = queue_tail_node(eq);
 540         if (!eqe)
 541                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 542
 543         num_ioeq_processed = 0;
 544         num_mcceq_processed = 0;
 545         if (blk_iopoll_enabled) {
 546                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 547                                         & EQE_VALID_MASK) {
 548                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 549                              resource_id) / 32] &
 550                              EQE_RESID_MASK) >> 16) == mcc->id) {
 551                                 spin_lock_irqsave(&phba->isr_lock, flags);
 552                                 phba->todo_mcc_cq = 1;
 553                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 554                                 num_mcceq_processed++;
 555                         } else {
 556                                 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 557                                         blk_iopoll_sched(&pbe_eq->iopoll);
 558                                 num_ioeq_processed++;
 559                         }
 560                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 561                         queue_tail_inc(eq);
 562                         eqe = queue_tail_node(eq);
 563                 }
 564                 if (num_ioeq_processed || num_mcceq_processed) {
 565                         if (phba->todo_mcc_cq)
 566                                 queue_work(phba->wq, &phba->work_cqs);
 567
 568                         if ((num_mcceq_processed) && (!num_ioeq_processed))
 569                                 hwi_ring_eq_db(phba, eq->id, 0,
 570                                               (num_ioeq_processed +
 571                                                num_mcceq_processed) , 1, 1);
 572                         else
 573                                 hwi_ring_eq_db(phba, eq->id, 0,
 574                                                (num_ioeq_processed +
 575                                                 num_mcceq_processed), 0, 1);
 576
 577                         return IRQ_HANDLED;
 578                 } else
 579                         return IRQ_NONE;
 580         } else {
 581                 cq = &phwi_context->be_cq[0];
 582                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 583                                                 & EQE_VALID_MASK) {
 584
 585                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 586                              resource_id) / 32] &
 587                              EQE_RESID_MASK) >> 16) != cq->id) {
 588                                 spin_lock_irqsave(&phba->isr_lock, flags);
 589                                 phba->todo_mcc_cq = 1;
 590                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 591                         } else {
 592                                 spin_lock_irqsave(&phba->isr_lock, flags);
 593                                 phba->todo_cq = 1;
 594                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 595                         }
 596                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 597                         queue_tail_inc(eq);
 598                         eqe = queue_tail_node(eq);
 599                         num_ioeq_processed++;
 600                 }
 601                 if (phba->todo_cq || phba->todo_mcc_cq)
 602                         queue_work(phba->wq, &phba->work_cqs);
 603
 604                 if (num_ioeq_processed) {
 605                         hwi_ring_eq_db(phba, eq->id, 0,
 606                                        num_ioeq_processed, 1, 1);
 607                         return IRQ_HANDLED;
 608                 } else
 609                         return IRQ_NONE;
 610         }
 611 }
 612
 613 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
 614 {
 615         struct pci_dev *pcidev = phba->pcidev;
 616         struct hwi_controller *phwi_ctrlr;
 617         struct hwi_context_memory *phwi_context;
 618         int ret, msix_vec, i = 0;
 619         char desc[32];
 620
 621         phwi_ctrlr = phba->phwi_ctrlr;
 622         phwi_context = phwi_ctrlr->phwi_ctxt;
 623
 624         if (phba->msix_enabled) {
 625                 for (i = 0; i < phba->num_cpus; i++) {
 626                         sprintf(desc, "beiscsi_msix_%04x", i);
 627                         msix_vec = phba->msix_entries[i].vector;
 628                         ret = request_irq(msix_vec, be_isr_msix, 0, desc,
 629                                           &phwi_context->be_eq[i]);
 630                 }
 631                 msix_vec = phba->msix_entries[i].vector;
 632                 ret = request_irq(msix_vec, be_isr_mcc, 0, "beiscsi_msix_mcc",
 633                                   &phwi_context->be_eq[i]);
 634         } else {
 635                 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
 636                                   "beiscsi", phba);
 637                 if (ret) {
 638                         shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
 639                                      "Failed to register irq\\n");
 640                         return ret;
 641                 }
 642         }
 643         return 0;
 644 }
 645
 646 static void hwi_ring_cq_db(struct beiscsi_hba *phba,
 647                            unsigned int id, unsigned int num_processed,
 648                            unsigned char rearm, unsigned char event)
 649 {
 650         u32 val = 0;
 651         val |= id & DB_CQ_RING_ID_MASK;
 652         if (rearm)
 653                 val |= 1 << DB_CQ_REARM_SHIFT;
 654         val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
 655         iowrite32(val, phba->db_va + DB_CQ_OFFSET);
 656 }
 657
 658 static unsigned int
 659 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
 660                           struct beiscsi_hba *phba,
 661                           unsigned short cid,
 662                           struct pdu_base *ppdu,
 663                           unsigned long pdu_len,
 664                           void *pbuffer, unsigned long buf_len)
 665 {
 666         struct iscsi_conn *conn = beiscsi_conn->conn;
 667         struct iscsi_session *session = conn->session;
 668         struct iscsi_task *task;
 669         struct beiscsi_io_task *io_task;
 670         struct iscsi_hdr *login_hdr;
 671
 672         switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
 673                                                 PDUBASE_OPCODE_MASK) {
 674         case ISCSI_OP_NOOP_IN:
 675                 pbuffer = NULL;
 676                 buf_len = 0;
 677                 break;
 678         case ISCSI_OP_ASYNC_EVENT:
 679                 break;
 680         case ISCSI_OP_REJECT:
 681                 WARN_ON(!pbuffer);
 682                 WARN_ON(!(buf_len == 48));
 683                 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
 684                 break;
 685         case ISCSI_OP_LOGIN_RSP:
 686         case ISCSI_OP_TEXT_RSP:
 687                 task = conn->login_task;
 688                 io_task = task->dd_data;
 689                 login_hdr = (struct iscsi_hdr *)ppdu;
 690                 login_hdr->itt = io_task->libiscsi_itt;
 691                 break;
 692         default:
 693                 shost_printk(KERN_WARNING, phba->shost,
 694                              "Unrecognized opcode 0x%x in async msg \n",
 695                              (ppdu->
 696                              dw[offsetof(struct amap_pdu_base, opcode) / 32]
 697                                                 & PDUBASE_OPCODE_MASK));
 698                 return 1;
 699         }
 700
 701         spin_lock_bh(&session->lock);
 702         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
 703         spin_unlock_bh(&session->lock);
 704         return 0;
 705 }
 706
 707 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
 708 {
 709         struct sgl_handle *psgl_handle;
 710
 711         if (phba->io_sgl_hndl_avbl) {
 712                 SE_DEBUG(DBG_LVL_8,
 713                          "In alloc_io_sgl_handle,io_sgl_alloc_index=%d \n",
 714                          phba->io_sgl_alloc_index);
 715                 psgl_handle = phba->io_sgl_hndl_base[phba->
 716                                                 io_sgl_alloc_index];
 717                 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
 718                 phba->io_sgl_hndl_avbl--;
 719                 if (phba->io_sgl_alloc_index == (phba->params.
 720                                                  ios_per_ctrl - 1))
 721                         phba->io_sgl_alloc_index = 0;
 722                 else
 723                         phba->io_sgl_alloc_index++;
 724         } else
 725                 psgl_handle = NULL;
 726         return psgl_handle;
 727 }
 728
 729 static void
 730 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
 731 {
 732         SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d \n",
 733                  phba->io_sgl_free_index);
 734         if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
 735                 /*
 736                  * this can happen if clean_task is called on a task that
 737                  * failed in xmit_task or alloc_pdu.
 738                  */
 739                  SE_DEBUG(DBG_LVL_8,
 740                          "Double Free in IO SGL io_sgl_free_index=%d,"
 741                          "value there=%p \n", phba->io_sgl_free_index,
 742                          phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
 743                 return;
 744         }
 745         phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
 746         phba->io_sgl_hndl_avbl++;
 747         if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
 748                 phba->io_sgl_free_index = 0;
 749         else
 750                 phba->io_sgl_free_index++;
 751 }
 752
 753 /**
 754  * alloc_wrb_handle - To allocate a wrb handle
 755  * @phba: The hba pointer
 756  * @cid: The cid to use for allocation
 757  *
 758  * This happens under session_lock until submission to chip
 759  */
 760 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
 761 {
 762         struct hwi_wrb_context *pwrb_context;
 763         struct hwi_controller *phwi_ctrlr;
 764         struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
 765
 766         phwi_ctrlr = phba->phwi_ctrlr;
 767         pwrb_context = &phwi_ctrlr->wrb_context[cid];
 768         if (pwrb_context->wrb_handles_available >= 2) {
 769                 pwrb_handle = pwrb_context->pwrb_handle_base[
 770                                             pwrb_context->alloc_index];
 771                 pwrb_context->wrb_handles_available--;
 772                 if (pwrb_context->alloc_index ==
 773                                                 (phba->params.wrbs_per_cxn - 1))
 774                         pwrb_context->alloc_index = 0;
 775                 else
 776                         pwrb_context->alloc_index++;
 777                 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
 778                                                 pwrb_context->alloc_index];
 779                 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
 780         } else
 781                 pwrb_handle = NULL;
 782         return pwrb_handle;
 783 }
 784
 785 /**
 786  * free_wrb_handle - To free the wrb handle back to pool
 787  * @phba: The hba pointer
 788  * @pwrb_context: The context to free from
 789  * @pwrb_handle: The wrb_handle to free
 790  *
 791  * This happens under session_lock until submission to chip
 792  */
 793 static void
 794 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
 795                 struct wrb_handle *pwrb_handle)
 796 {
 797         pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
 798         pwrb_context->wrb_handles_available++;
 799         if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
 800                 pwrb_context->free_index = 0;
 801         else
 802                 pwrb_context->free_index++;
 803
 804         SE_DEBUG(DBG_LVL_8,
 805                  "FREE WRB: pwrb_handle=%p free_index=0x%x"
 806                  "wrb_handles_available=%d \n",
 807                  pwrb_handle, pwrb_context->free_index,
 808                  pwrb_context->wrb_handles_available);
 809 }
 810
 811 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
 812 {
 813         struct sgl_handle *psgl_handle;
 814
 815         if (phba->eh_sgl_hndl_avbl) {
 816                 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
 817                 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
 818                 SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x \n",
 819                          phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
 820                 phba->eh_sgl_hndl_avbl--;
 821                 if (phba->eh_sgl_alloc_index ==
 822                     (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
 823                      1))
 824                         phba->eh_sgl_alloc_index = 0;
 825                 else
 826                         phba->eh_sgl_alloc_index++;
 827         } else
 828                 psgl_handle = NULL;
 829         return psgl_handle;
 830 }
 831
 832 void
 833 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
 834 {
 835
 836         SE_DEBUG(DBG_LVL_8, "In  free_mgmt_sgl_handle,eh_sgl_free_index=%d \n",
 837                              phba->eh_sgl_free_index);
 838         if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
 839                 /*
 840                  * this can happen if clean_task is called on a task that
 841                  * failed in xmit_task or alloc_pdu.
 842                  */
 843                 SE_DEBUG(DBG_LVL_8,
 844                          "Double Free in eh SGL ,eh_sgl_free_index=%d \n",
 845                          phba->eh_sgl_free_index);
 846                 return;
 847         }
 848         phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
 849         phba->eh_sgl_hndl_avbl++;
 850         if (phba->eh_sgl_free_index ==
 851             (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
 852                 phba->eh_sgl_free_index = 0;
 853         else
 854                 phba->eh_sgl_free_index++;
 855 }
 856
 857 static void
 858 be_complete_io(struct beiscsi_conn *beiscsi_conn,
 859                struct iscsi_task *task, struct sol_cqe *psol)
 860 {
 861         struct beiscsi_io_task *io_task = task->dd_data;
 862         struct be_status_bhs *sts_bhs =
 863                                 (struct be_status_bhs *)io_task->cmd_bhs;
 864         struct iscsi_conn *conn = beiscsi_conn->conn;
 865         unsigned int sense_len;
 866         unsigned char *sense;
 867         u32 resid = 0, exp_cmdsn, max_cmdsn;
 868         u8 rsp, status, flags;
 869
 870         exp_cmdsn = (psol->
 871                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
 872                         & SOL_EXP_CMD_SN_MASK);
 873         max_cmdsn = ((psol->
 874                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
 875                         & SOL_EXP_CMD_SN_MASK) +
 876                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
 877                                 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
 878         rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
 879                                                 & SOL_RESP_MASK) >> 16);
 880         status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
 881                                                 & SOL_STS_MASK) >> 8);
 882         flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
 883                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
 884
 885         task->sc->result = (DID_OK << 16) | status;
 886         if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
 887                 task->sc->result = DID_ERROR << 16;
 888                 goto unmap;
 889         }
 890
 891         /* bidi not initially supported */
 892         if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
 893                 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
 894                                 32] & SOL_RES_CNT_MASK);
 895
 896                 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
 897                         task->sc->result = DID_ERROR << 16;
 898
 899                 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
 900                         scsi_set_resid(task->sc, resid);
 901                         if (!status && (scsi_bufflen(task->sc) - resid <
 902                             task->sc->underflow))
 903                                 task->sc->result = DID_ERROR << 16;
 904                 }
 905         }
 906
 907         if (status == SAM_STAT_CHECK_CONDITION) {
 908                 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
 909                 sense = sts_bhs->sense_info + sizeof(unsigned short);
 910                 sense_len =  cpu_to_be16(*slen);
 911                 memcpy(task->sc->sense_buffer, sense,
 912                        min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
 913         }
 914
 915         if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
 916                 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
 917                                                         & SOL_RES_CNT_MASK)
 918                          conn->rxdata_octets += (psol->
 919                              dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
 920                              & SOL_RES_CNT_MASK);
 921         }
 922 unmap:
 923         scsi_dma_unmap(io_task->scsi_cmnd);
 924         iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
 925 }
 926
 927 static void
 928 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
 929                    struct iscsi_task *task, struct sol_cqe *psol)
 930 {
 931         struct iscsi_logout_rsp *hdr;
 932         struct beiscsi_io_task *io_task = task->dd_data;
 933         struct iscsi_conn *conn = beiscsi_conn->conn;
 934
 935         hdr = (struct iscsi_logout_rsp *)task->hdr;
 936         hdr->opcode = ISCSI_OP_LOGOUT_RSP;
 937         hdr->t2wait = 5;
 938         hdr->t2retain = 0;
 939         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
 940                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
 941         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
 942                                         32] & SOL_RESP_MASK);
 943         hdr->exp_cmdsn = cpu_to_be32(psol->
 944                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
 945                                         & SOL_EXP_CMD_SN_MASK);
 946         hdr->max_cmdsn = be32_to_cpu((psol->
 947                          dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
 948                                         & SOL_EXP_CMD_SN_MASK) +
 949                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
 950                                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
 951         hdr->dlength[0] = 0;
 952         hdr->dlength[1] = 0;
 953         hdr->dlength[2] = 0;
 954         hdr->hlength = 0;
 955         hdr->itt = io_task->libiscsi_itt;
 956         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
 957 }
 958
 959 static void
 960 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
 961                 struct iscsi_task *task, struct sol_cqe *psol)
 962 {
 963         struct iscsi_tm_rsp *hdr;
 964         struct iscsi_conn *conn = beiscsi_conn->conn;
 965         struct beiscsi_io_task *io_task = task->dd_data;
 966
 967         hdr = (struct iscsi_tm_rsp *)task->hdr;
 968         hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
 969         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
 970                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
 971         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
 972                                         32] & SOL_RESP_MASK);
 973         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
 974                                     i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
 975         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
 976                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
 977                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
 978                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
 979         hdr->itt = io_task->libiscsi_itt;
 980         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
 981 }
 982
 983 static void
 984 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
 985                        struct beiscsi_hba *phba, struct sol_cqe *psol)
 986 {
 987         struct hwi_wrb_context *pwrb_context;
 988         struct wrb_handle *pwrb_handle = NULL;
 989         struct hwi_controller *phwi_ctrlr;
 990         struct iscsi_task *task;
 991         struct beiscsi_io_task *io_task;
 992         struct iscsi_conn *conn = beiscsi_conn->conn;
 993         struct iscsi_session *session = conn->session;
 994
 995         phwi_ctrlr = phba->phwi_ctrlr;
 996         pwrb_context = &phwi_ctrlr->wrb_context[((psol->
 997                                 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
 998                                 SOL_CID_MASK) >> 6) -
 999                                 phba->fw_config.iscsi_cid_start];
1000         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
1001                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1002                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
1003         task = pwrb_handle->pio_handle;
1004
1005         io_task = task->dd_data;
1006         spin_lock(&phba->mgmt_sgl_lock);
1007         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
1008         spin_unlock(&phba->mgmt_sgl_lock);
1009         spin_lock_bh(&session->lock);
1010         free_wrb_handle(phba, pwrb_context, pwrb_handle);
1011         spin_unlock_bh(&session->lock);
1012 }
1013
1014 static void
1015 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
1016                        struct iscsi_task *task, struct sol_cqe *psol)
1017 {
1018         struct iscsi_nopin *hdr;
1019         struct iscsi_conn *conn = beiscsi_conn->conn;
1020         struct beiscsi_io_task *io_task = task->dd_data;
1021
1022         hdr = (struct iscsi_nopin *)task->hdr;
1023         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1024                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1025         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
1026                                      i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
1027         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1028                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1029                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1030                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1031         hdr->opcode = ISCSI_OP_NOOP_IN;
1032         hdr->itt = io_task->libiscsi_itt;
1033         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1034 }
1035
1036 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1037                              struct beiscsi_hba *phba, struct sol_cqe *psol)
1038 {
1039         struct hwi_wrb_context *pwrb_context;
1040         struct wrb_handle *pwrb_handle;
1041         struct iscsi_wrb *pwrb = NULL;
1042         struct hwi_controller *phwi_ctrlr;
1043         struct iscsi_task *task;
1044         unsigned int type;
1045         struct iscsi_conn *conn = beiscsi_conn->conn;
1046         struct iscsi_session *session = conn->session;
1047
1048         phwi_ctrlr = phba->phwi_ctrlr;
1049         pwrb_context = &phwi_ctrlr->wrb_context[((psol->dw[offsetof
1050                                 (struct amap_sol_cqe, cid) / 32]
1051                                 & SOL_CID_MASK) >> 6) -
1052                                 phba->fw_config.iscsi_cid_start];
1053         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
1054                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1055                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
1056         task = pwrb_handle->pio_handle;
1057         pwrb = pwrb_handle->pwrb;
1058         type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
1059                                  WRB_TYPE_MASK) >> 28;
1060
1061         spin_lock_bh(&session->lock);
1062         switch (type) {
1063         case HWH_TYPE_IO:
1064         case HWH_TYPE_IO_RD:
1065                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
1066                      ISCSI_OP_NOOP_OUT)
1067                         be_complete_nopin_resp(beiscsi_conn, task, psol);
1068                 else
1069                         be_complete_io(beiscsi_conn, task, psol);
1070                 break;
1071
1072         case HWH_TYPE_LOGOUT:
1073                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
1074                         be_complete_logout(beiscsi_conn, task, psol);
1075                 else
1076                         be_complete_tmf(beiscsi_conn, task, psol);
1077
1078                 break;
1079
1080         case HWH_TYPE_LOGIN:
1081                 SE_DEBUG(DBG_LVL_1,
1082                          "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
1083                          "- Solicited path \n");
1084                 break;
1085
1086         case HWH_TYPE_NOP:
1087                 be_complete_nopin_resp(beiscsi_conn, task, psol);
1088                 break;
1089
1090         default:
1091                 shost_printk(KERN_WARNING, phba->shost,
1092                                 "In hwi_complete_cmd, unknown type = %d"
1093                                 "wrb_index 0x%x CID 0x%x\n", type,
1094                                 ((psol->dw[offsetof(struct amap_iscsi_wrb,
1095                                 type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
1096                                 ((psol->dw[offsetof(struct amap_sol_cqe,
1097                                 cid) / 32] & SOL_CID_MASK) >> 6));
1098                 break;
1099         }
1100
1101         spin_unlock_bh(&session->lock);
1102 }
1103
1104 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1105                                           *pasync_ctx, unsigned int is_header,
1106                                           unsigned int host_write_ptr)
1107 {
1108         if (is_header)
1109                 return &pasync_ctx->async_entry[host_write_ptr].
1110                     header_busy_list;
1111         else
1112                 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1113 }
1114
1115 static struct async_pdu_handle *
1116 hwi_get_async_handle(struct beiscsi_hba *phba,
1117                      struct beiscsi_conn *beiscsi_conn,
1118                      struct hwi_async_pdu_context *pasync_ctx,
1119                      struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1120 {
1121         struct be_bus_address phys_addr;
1122         struct list_head *pbusy_list;
1123         struct async_pdu_handle *pasync_handle = NULL;
1124         int buffer_len = 0;
1125         unsigned char buffer_index = -1;
1126         unsigned char is_header = 0;
1127
1128         phys_addr.u.a32.address_lo =
1129             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
1130             ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1131                                                 & PDUCQE_DPL_MASK) >> 16);
1132         phys_addr.u.a32.address_hi =
1133             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
1134
1135         phys_addr.u.a64.address =
1136                         *((unsigned long long *)(&phys_addr.u.a64.address));
1137
1138         switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1139                         & PDUCQE_CODE_MASK) {
1140         case UNSOL_HDR_NOTIFY:
1141                 is_header = 1;
1142
1143                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
1144                         (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1145                         index) / 32] & PDUCQE_INDEX_MASK));
1146
1147                 buffer_len = (unsigned int)(phys_addr.u.a64.address -
1148                                 pasync_ctx->async_header.pa_base.u.a64.address);
1149
1150                 buffer_index = buffer_len /
1151                                 pasync_ctx->async_header.buffer_size;
1152
1153                 break;
1154         case UNSOL_DATA_NOTIFY:
1155                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
1156                                         dw[offsetof(struct amap_i_t_dpdu_cqe,
1157                                         index) / 32] & PDUCQE_INDEX_MASK));
1158                 buffer_len = (unsigned long)(phys_addr.u.a64.address -
1159                                         pasync_ctx->async_data.pa_base.u.
1160                                         a64.address);
1161                 buffer_index = buffer_len / pasync_ctx->async_data.buffer_size;
1162                 break;
1163         default:
1164                 pbusy_list = NULL;
1165                 shost_printk(KERN_WARNING, phba->shost,
1166                         "Unexpected code=%d \n",
1167                          pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1168                                         code) / 32] & PDUCQE_CODE_MASK);
1169                 return NULL;
1170         }
1171
1172         WARN_ON(!(buffer_index <= pasync_ctx->async_data.num_entries));
1173         WARN_ON(list_empty(pbusy_list));
1174         list_for_each_entry(pasync_handle, pbusy_list, link) {
1175                 WARN_ON(pasync_handle->consumed);
1176                 if (pasync_handle->index == buffer_index)
1177                         break;
1178         }
1179
1180         WARN_ON(!pasync_handle);
1181
1182         pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
1183                                              phba->fw_config.iscsi_cid_start;
1184         pasync_handle->is_header = is_header;
1185         pasync_handle->buffer_len = ((pdpdu_cqe->
1186                         dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1187                         & PDUCQE_DPL_MASK) >> 16);
1188
1189         *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1190                         index) / 32] & PDUCQE_INDEX_MASK);
1191         return pasync_handle;
1192 }
1193
1194 static unsigned int
1195 hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
1196                            unsigned int is_header, unsigned int cq_index)
1197 {
1198         struct list_head *pbusy_list;
1199         struct async_pdu_handle *pasync_handle;
1200         unsigned int num_entries, writables = 0;
1201         unsigned int *pep_read_ptr, *pwritables;
1202
1203
1204         if (is_header) {
1205                 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1206                 pwritables = &pasync_ctx->async_header.writables;
1207                 num_entries = pasync_ctx->async_header.num_entries;
1208         } else {
1209                 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1210                 pwritables = &pasync_ctx->async_data.writables;
1211                 num_entries = pasync_ctx->async_data.num_entries;
1212         }
1213
1214         while ((*pep_read_ptr) != cq_index) {
1215                 (*pep_read_ptr)++;
1216                 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1217
1218                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1219                                                      *pep_read_ptr);
1220                 if (writables == 0)
1221                         WARN_ON(list_empty(pbusy_list));
1222
1223                 if (!list_empty(pbusy_list)) {
1224                         pasync_handle = list_entry(pbusy_list->next,
1225                                                    struct async_pdu_handle,
1226                                                    link);
1227                         WARN_ON(!pasync_handle);
1228                         pasync_handle->consumed = 1;
1229                 }
1230
1231                 writables++;
1232         }
1233
1234         if (!writables) {
1235                 SE_DEBUG(DBG_LVL_1,
1236                          "Duplicate notification received - index 0x%x!!\n",
1237                          cq_index);
1238                 WARN_ON(1);
1239         }
1240
1241         *pwritables = *pwritables + writables;
1242         return 0;
1243 }
1244
1245 static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba,
1246                                        unsigned int cri)
1247 {
1248         struct hwi_controller *phwi_ctrlr;
1249         struct hwi_async_pdu_context *pasync_ctx;
1250         struct async_pdu_handle *pasync_handle, *tmp_handle;
1251         struct list_head *plist;
1252         unsigned int i = 0;
1253
1254         phwi_ctrlr = phba->phwi_ctrlr;
1255         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1256
1257         plist  = &pasync_ctx->async_entry[cri].wait_queue.list;
1258
1259         list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1260                 list_del(&pasync_handle->link);
1261
1262                 if (i == 0) {
1263                         list_add_tail(&pasync_handle->link,
1264                                       &pasync_ctx->async_header.free_list);
1265                         pasync_ctx->async_header.free_entries++;
1266                         i++;
1267                 } else {
1268                         list_add_tail(&pasync_handle->link,
1269                                       &pasync_ctx->async_data.free_list);
1270                         pasync_ctx->async_data.free_entries++;
1271                         i++;
1272                 }
1273         }
1274
1275         INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1276         pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1277         pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1278         return 0;
1279 }
1280
1281 static struct phys_addr *
1282 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1283                      unsigned int is_header, unsigned int host_write_ptr)
1284 {
1285         struct phys_addr *pasync_sge = NULL;
1286
1287         if (is_header)
1288                 pasync_sge = pasync_ctx->async_header.ring_base;
1289         else
1290                 pasync_sge = pasync_ctx->async_data.ring_base;
1291
1292         return pasync_sge + host_write_ptr;
1293 }
1294
1295 static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1296                                    unsigned int is_header)
1297 {
1298         struct hwi_controller *phwi_ctrlr;
1299         struct hwi_async_pdu_context *pasync_ctx;
1300         struct async_pdu_handle *pasync_handle;
1301         struct list_head *pfree_link, *pbusy_list;
1302         struct phys_addr *pasync_sge;
1303         unsigned int ring_id, num_entries;
1304         unsigned int host_write_num;
1305         unsigned int writables;
1306         unsigned int i = 0;
1307         u32 doorbell = 0;
1308
1309         phwi_ctrlr = phba->phwi_ctrlr;
1310         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1311
1312         if (is_header) {
1313                 num_entries = pasync_ctx->async_header.num_entries;
1314                 writables = min(pasync_ctx->async_header.writables,
1315                                 pasync_ctx->async_header.free_entries);
1316                 pfree_link = pasync_ctx->async_header.free_list.next;
1317                 host_write_num = pasync_ctx->async_header.host_write_ptr;
1318                 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1319         } else {
1320                 num_entries = pasync_ctx->async_data.num_entries;
1321                 writables = min(pasync_ctx->async_data.writables,
1322                                 pasync_ctx->async_data.free_entries);
1323                 pfree_link = pasync_ctx->async_data.free_list.next;
1324                 host_write_num = pasync_ctx->async_data.host_write_ptr;
1325                 ring_id = phwi_ctrlr->default_pdu_data.id;
1326         }
1327
1328         writables = (writables / 8) * 8;
1329         if (writables) {
1330                 for (i = 0; i < writables; i++) {
1331                         pbusy_list =
1332                             hwi_get_async_busy_list(pasync_ctx, is_header,
1333                                                     host_write_num);
1334                         pasync_handle =
1335                             list_entry(pfree_link, struct async_pdu_handle,
1336                                                                 link);
1337                         WARN_ON(!pasync_handle);
1338                         pasync_handle->consumed = 0;
1339
1340                         pfree_link = pfree_link->next;
1341
1342                         pasync_sge = hwi_get_ring_address(pasync_ctx,
1343                                                 is_header, host_write_num);
1344
1345                         pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1346                         pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1347
1348                         list_move(&pasync_handle->link, pbusy_list);
1349
1350                         host_write_num++;
1351                         host_write_num = host_write_num % num_entries;
1352                 }
1353
1354                 if (is_header) {
1355                         pasync_ctx->async_header.host_write_ptr =
1356                                                         host_write_num;
1357                         pasync_ctx->async_header.free_entries -= writables;
1358                         pasync_ctx->async_header.writables -= writables;
1359                         pasync_ctx->async_header.busy_entries += writables;
1360                 } else {
1361                         pasync_ctx->async_data.host_write_ptr = host_write_num;
1362                         pasync_ctx->async_data.free_entries -= writables;
1363                         pasync_ctx->async_data.writables -= writables;
1364                         pasync_ctx->async_data.busy_entries += writables;
1365                 }
1366
1367                 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1368                 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1369                 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1370                 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1371                                         << DB_DEF_PDU_CQPROC_SHIFT;
1372
1373                 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1374         }
1375 }
1376
1377 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1378                                          struct beiscsi_conn *beiscsi_conn,
1379                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1380 {
1381         struct hwi_controller *phwi_ctrlr;
1382         struct hwi_async_pdu_context *pasync_ctx;
1383         struct async_pdu_handle *pasync_handle = NULL;
1384         unsigned int cq_index = -1;
1385
1386         phwi_ctrlr = phba->phwi_ctrlr;
1387         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1388
1389         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1390                                              pdpdu_cqe, &cq_index);
1391         BUG_ON(pasync_handle->is_header != 0);
1392         if (pasync_handle->consumed == 0)
1393                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1394                                            cq_index);
1395
1396         hwi_free_async_msg(phba, pasync_handle->cri);
1397         hwi_post_async_buffers(phba, pasync_handle->is_header);
1398 }
1399
1400 static unsigned int
1401 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1402                   struct beiscsi_hba *phba,
1403                   struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1404 {
1405         struct list_head *plist;
1406         struct async_pdu_handle *pasync_handle;
1407         void *phdr = NULL;
1408         unsigned int hdr_len = 0, buf_len = 0;
1409         unsigned int status, index = 0, offset = 0;
1410         void *pfirst_buffer = NULL;
1411         unsigned int num_buf = 0;
1412
1413         plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1414
1415         list_for_each_entry(pasync_handle, plist, link) {
1416                 if (index == 0) {
1417                         phdr = pasync_handle->pbuffer;
1418                         hdr_len = pasync_handle->buffer_len;
1419                 } else {
1420                         buf_len = pasync_handle->buffer_len;
1421                         if (!num_buf) {
1422                                 pfirst_buffer = pasync_handle->pbuffer;
1423                                 num_buf++;
1424                         }
1425                         memcpy(pfirst_buffer + offset,
1426                                pasync_handle->pbuffer, buf_len);
1427                         offset = buf_len;
1428                 }
1429                 index++;
1430         }
1431
1432         status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1433                                            (beiscsi_conn->beiscsi_conn_cid -
1434                                             phba->fw_config.iscsi_cid_start),
1435                                             phdr, hdr_len, pfirst_buffer,
1436                                             buf_len);
1437
1438         if (status == 0)
1439                 hwi_free_async_msg(phba, cri);
1440         return 0;
1441 }
1442
1443 static unsigned int
1444 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1445                      struct beiscsi_hba *phba,
1446                      struct async_pdu_handle *pasync_handle)
1447 {
1448         struct hwi_async_pdu_context *pasync_ctx;
1449         struct hwi_controller *phwi_ctrlr;
1450         unsigned int bytes_needed = 0, status = 0;
1451         unsigned short cri = pasync_handle->cri;
1452         struct pdu_base *ppdu;
1453
1454         phwi_ctrlr = phba->phwi_ctrlr;
1455         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1456
1457         list_del(&pasync_handle->link);
1458         if (pasync_handle->is_header) {
1459                 pasync_ctx->async_header.busy_entries--;
1460                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1461                         hwi_free_async_msg(phba, cri);
1462                         BUG();
1463                 }
1464
1465                 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1466                 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1467                 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1468                                 (unsigned short)pasync_handle->buffer_len;
1469                 list_add_tail(&pasync_handle->link,
1470                               &pasync_ctx->async_entry[cri].wait_queue.list);
1471
1472                 ppdu = pasync_handle->pbuffer;
1473                 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1474                         data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1475                         0xFFFF0000) | ((be16_to_cpu((ppdu->
1476                         dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1477                         & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1478
1479                 if (status == 0) {
1480                         pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1481                             bytes_needed;
1482
1483                         if (bytes_needed == 0)
1484                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1485                                                            pasync_ctx, cri);
1486                 }
1487         } else {
1488                 pasync_ctx->async_data.busy_entries--;
1489                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1490                         list_add_tail(&pasync_handle->link,
1491                                       &pasync_ctx->async_entry[cri].wait_queue.
1492                                       list);
1493                         pasync_ctx->async_entry[cri].wait_queue.
1494                                 bytes_received +=
1495                                 (unsigned short)pasync_handle->buffer_len;
1496
1497                         if (pasync_ctx->async_entry[cri].wait_queue.
1498                             bytes_received >=
1499                             pasync_ctx->async_entry[cri].wait_queue.
1500                             bytes_needed)
1501                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1502                                                            pasync_ctx, cri);
1503                 }
1504         }
1505         return status;
1506 }
1507
1508 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1509                                          struct beiscsi_hba *phba,
1510                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1511 {
1512         struct hwi_controller *phwi_ctrlr;
1513         struct hwi_async_pdu_context *pasync_ctx;
1514         struct async_pdu_handle *pasync_handle = NULL;
1515         unsigned int cq_index = -1;
1516
1517         phwi_ctrlr = phba->phwi_ctrlr;
1518         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1519         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1520                                              pdpdu_cqe, &cq_index);
1521
1522         if (pasync_handle->consumed == 0)
1523                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1524                                            cq_index);
1525         hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1526         hwi_post_async_buffers(phba, pasync_handle->is_header);
1527 }
1528
1529 static void  beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
1530 {
1531         struct be_queue_info *mcc_cq;
1532         struct  be_mcc_compl *mcc_compl;
1533         unsigned int num_processed = 0;
1534
1535         mcc_cq = &phba->ctrl.mcc_obj.cq;
1536         mcc_compl = queue_tail_node(mcc_cq);
1537         mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1538         while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
1539
1540                 if (num_processed >= 32) {
1541                         hwi_ring_cq_db(phba, mcc_cq->id,
1542                                         num_processed, 0, 0);
1543                         num_processed = 0;
1544                 }
1545                 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
1546                         /* Interpret flags as an async trailer */
1547                         if (is_link_state_evt(mcc_compl->flags))
1548                                 /* Interpret compl as a async link evt */
1549                                 beiscsi_async_link_state_process(phba,
1550                                 (struct be_async_event_link_state *) mcc_compl);
1551                         else
1552                                 SE_DEBUG(DBG_LVL_1,
1553                                         " Unsupported Async Event, flags"
1554                                         " = 0x%08x \n", mcc_compl->flags);
1555                 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
1556                         be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
1557                         atomic_dec(&phba->ctrl.mcc_obj.q.used);
1558                 }
1559
1560                 mcc_compl->flags = 0;
1561                 queue_tail_inc(mcc_cq);
1562                 mcc_compl = queue_tail_node(mcc_cq);
1563                 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1564                 num_processed++;
1565         }
1566
1567         if (num_processed > 0)
1568                 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0);
1569
1570 }
1571
1572 static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
1573 {
1574         struct be_queue_info *cq;
1575         struct sol_cqe *sol;
1576         struct dmsg_cqe *dmsg;
1577         unsigned int num_processed = 0;
1578         unsigned int tot_nump = 0;
1579         struct beiscsi_conn *beiscsi_conn;
1580         struct beiscsi_endpoint *beiscsi_ep;
1581         struct iscsi_endpoint *ep;
1582         struct beiscsi_hba *phba;
1583
1584         cq = pbe_eq->cq;
1585         sol = queue_tail_node(cq);
1586         phba = pbe_eq->phba;
1587
1588         while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1589                CQE_VALID_MASK) {
1590                 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1591
1592                 ep = phba->ep_array[(u32) ((sol->
1593                                    dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1594                                    SOL_CID_MASK) >> 6) -
1595                                    phba->fw_config.iscsi_cid_start];
1596
1597                 beiscsi_ep = ep->dd_data;
1598                 beiscsi_conn = beiscsi_ep->conn;
1599
1600                 if (num_processed >= 32) {
1601                         hwi_ring_cq_db(phba, cq->id,
1602                                         num_processed, 0, 0);
1603                         tot_nump += num_processed;
1604                         num_processed = 0;
1605                 }
1606
1607                 switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) /
1608                         32] & CQE_CODE_MASK) {
1609                 case SOL_CMD_COMPLETE:
1610                         hwi_complete_cmd(beiscsi_conn, phba, sol);
1611                         break;
1612                 case DRIVERMSG_NOTIFY:
1613                         SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY \n");
1614                         dmsg = (struct dmsg_cqe *)sol;
1615                         hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1616                         break;
1617                 case UNSOL_HDR_NOTIFY:
1618                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR_ NOTIFY\n");
1619                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1620                                              (struct i_t_dpdu_cqe *)sol);
1621                         break;
1622                 case UNSOL_DATA_NOTIFY:
1623                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_DATA_NOTIFY\n");
1624                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1625                                              (struct i_t_dpdu_cqe *)sol);
1626                         break;
1627                 case CXN_INVALIDATE_INDEX_NOTIFY:
1628                 case CMD_INVALIDATED_NOTIFY:
1629                 case CXN_INVALIDATE_NOTIFY:
1630                         SE_DEBUG(DBG_LVL_1,
1631                                  "Ignoring CQ Error notification for cmd/cxn"
1632                                  "invalidate\n");
1633                         break;
1634                 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1635                 case CMD_KILLED_INVALID_STATSN_RCVD:
1636                 case CMD_KILLED_INVALID_R2T_RCVD:
1637                 case CMD_CXN_KILLED_LUN_INVALID:
1638                 case CMD_CXN_KILLED_ICD_INVALID:
1639                 case CMD_CXN_KILLED_ITT_INVALID:
1640                 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1641                 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
1642                         SE_DEBUG(DBG_LVL_1,
1643                                  "CQ Error notification for cmd.. "
1644                                  "code %d cid 0x%x\n",
1645                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1646                                  32] & CQE_CODE_MASK,
1647                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1648                                  32] & SOL_CID_MASK));
1649                         break;
1650                 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1651                         SE_DEBUG(DBG_LVL_1,
1652                                  "Digest error on def pdu ring, dropping..\n");
1653                         hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
1654                                              (struct i_t_dpdu_cqe *) sol);
1655                         break;
1656                 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
1657                 case CXN_KILLED_BURST_LEN_MISMATCH:
1658                 case CXN_KILLED_AHS_RCVD:
1659                 case CXN_KILLED_HDR_DIGEST_ERR:
1660                 case CXN_KILLED_UNKNOWN_HDR:
1661                 case CXN_KILLED_STALE_ITT_TTT_RCVD:
1662                 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
1663                 case CXN_KILLED_TIMED_OUT:
1664                 case CXN_KILLED_FIN_RCVD:
1665                 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
1666                 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
1667                 case CXN_KILLED_OVER_RUN_RESIDUAL:
1668                 case CXN_KILLED_UNDER_RUN_RESIDUAL:
1669                 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
1670                         SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
1671                                  "0x%x...\n",
1672                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1673                                  32] & CQE_CODE_MASK,
1674                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1675                                  32] & CQE_CID_MASK));
1676                         iscsi_conn_failure(beiscsi_conn->conn,
1677                                            ISCSI_ERR_CONN_FAILED);
1678                         break;
1679                 case CXN_KILLED_RST_SENT:
1680                 case CXN_KILLED_RST_RCVD:
1681                         SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
1682                                 "received/sent on CID 0x%x...\n",
1683                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1684                                  32] & CQE_CODE_MASK,
1685                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1686                                  32] & CQE_CID_MASK));
1687                         iscsi_conn_failure(beiscsi_conn->conn,
1688                                            ISCSI_ERR_CONN_FAILED);
1689                         break;
1690                 default:
1691                         SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d "
1692                                  "received on CID 0x%x...\n",
1693                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1694                                  32] & CQE_CODE_MASK,
1695                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1696                                  32] & CQE_CID_MASK));
1697                         break;
1698                 }
1699
1700                 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
1701                 queue_tail_inc(cq);
1702                 sol = queue_tail_node(cq);
1703                 num_processed++;
1704         }
1705
1706         if (num_processed > 0) {
1707                 tot_nump += num_processed;
1708                 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
1709         }
1710         return tot_nump;
1711 }
1712
1713 void beiscsi_process_all_cqs(struct work_struct *work)
1714 {
1715         unsigned long flags;
1716         struct hwi_controller *phwi_ctrlr;
1717         struct hwi_context_memory *phwi_context;
1718         struct be_eq_obj *pbe_eq;
1719         struct beiscsi_hba *phba =
1720             container_of(work, struct beiscsi_hba, work_cqs);
1721
1722         phwi_ctrlr = phba->phwi_ctrlr;
1723         phwi_context = phwi_ctrlr->phwi_ctxt;
1724         if (phba->msix_enabled)
1725                 pbe_eq = &phwi_context->be_eq[phba->num_cpus];
1726         else
1727                 pbe_eq = &phwi_context->be_eq[0];
1728
1729         if (phba->todo_mcc_cq) {
1730                 spin_lock_irqsave(&phba->isr_lock, flags);
1731                 phba->todo_mcc_cq = 0;
1732                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1733                 beiscsi_process_mcc_isr(phba);
1734         }
1735
1736         if (phba->todo_cq) {
1737                 spin_lock_irqsave(&phba->isr_lock, flags);
1738                 phba->todo_cq = 0;
1739                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1740                 beiscsi_process_cq(pbe_eq);
1741         }
1742 }
1743
1744 static int be_iopoll(struct blk_iopoll *iop, int budget)
1745 {
1746         static unsigned int ret;
1747         struct beiscsi_hba *phba;
1748         struct be_eq_obj *pbe_eq;
1749
1750         pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
1751         ret = beiscsi_process_cq(pbe_eq);
1752         if (ret < budget) {
1753                 phba = pbe_eq->phba;
1754                 blk_iopoll_complete(iop);
1755                 SE_DEBUG(DBG_LVL_8, "rearm pbe_eq->q.id =%d\n", pbe_eq->q.id);
1756                 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
1757         }
1758         return ret;
1759 }
1760
1761 static void
1762 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
1763               unsigned int num_sg, struct beiscsi_io_task *io_task)
1764 {
1765         struct iscsi_sge *psgl;
1766         unsigned short sg_len, index;
1767         unsigned int sge_len = 0;
1768         unsigned long long addr;
1769         struct scatterlist *l_sg;
1770         unsigned int offset;
1771
1772         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1773                                       io_task->bhs_pa.u.a32.address_lo);
1774         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1775                                       io_task->bhs_pa.u.a32.address_hi);
1776
1777         l_sg = sg;
1778         for (index = 0; (index < num_sg) && (index < 2); index++,
1779                                                          sg = sg_next(sg)) {
1780                 if (index == 0) {
1781                         sg_len = sg_dma_len(sg);
1782                         addr = (u64) sg_dma_address(sg);
1783                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1784                                                         (addr & 0xFFFFFFFF));
1785                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1786                                                         (addr >> 32));
1787                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1788                                                         sg_len);
1789                         sge_len = sg_len;
1790                 } else {
1791                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
1792                                                         pwrb, sge_len);
1793                         sg_len = sg_dma_len(sg);
1794                         addr = (u64) sg_dma_address(sg);
1795                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
1796                                                         (addr & 0xFFFFFFFF));
1797                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
1798                                                         (addr >> 32));
1799                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
1800                                                         sg_len);
1801                 }
1802         }
1803         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1804         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
1805
1806         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
1807
1808         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1809                         io_task->bhs_pa.u.a32.address_hi);
1810         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1811                         io_task->bhs_pa.u.a32.address_lo);
1812
1813         if (num_sg == 1) {
1814                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1815                                                                 1);
1816                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
1817                                                                 0);
1818         } else if (num_sg == 2) {
1819                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1820                                                                 0);
1821                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
1822                                                                 1);
1823         } else {
1824                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1825                                                                 0);
1826                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
1827                                                                 0);
1828         }
1829         sg = l_sg;
1830         psgl++;
1831         psgl++;
1832         offset = 0;
1833         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
1834                 sg_len = sg_dma_len(sg);
1835                 addr = (u64) sg_dma_address(sg);
1836                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1837                                                 (addr & 0xFFFFFFFF));
1838                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1839                                                 (addr >> 32));
1840                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
1841                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
1842                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1843                 offset += sg_len;
1844         }
1845         psgl--;
1846         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1847 }
1848
1849 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
1850 {
1851         struct iscsi_sge *psgl;
1852         unsigned long long addr;
1853         struct beiscsi_io_task *io_task = task->dd_data;
1854         struct beiscsi_conn *beiscsi_conn = io_task->conn;
1855         struct beiscsi_hba *phba = beiscsi_conn->phba;
1856
1857         io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
1858         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1859                                 io_task->bhs_pa.u.a32.address_lo);
1860         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1861                                 io_task->bhs_pa.u.a32.address_hi);
1862
1863         if (task->data) {
1864                 if (task->data_count) {
1865                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
1866                         addr = (u64) pci_map_single(phba->pcidev,
1867                                                     task->data,
1868                                                     task->data_count, 1);
1869                 } else {
1870                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1871                         addr = 0;
1872                 }
1873                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1874                                                 (addr & 0xFFFFFFFF));
1875                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1876                                                 (addr >> 32));
1877                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1878                                                 task->data_count);
1879
1880                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
1881         } else {
1882                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1883                 addr = 0;
1884         }
1885
1886         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1887
1888         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
1889
1890         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1891                       io_task->bhs_pa.u.a32.address_hi);
1892         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1893                       io_task->bhs_pa.u.a32.address_lo);
1894         if (task->data) {
1895                 psgl++;
1896                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
1897                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
1898                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
1899                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
1900                 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
1901                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1902
1903                 psgl++;
1904                 if (task->data) {
1905                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1906                                                 (addr & 0xFFFFFFFF));
1907                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1908                                                 (addr >> 32));
1909                 }
1910                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
1911         }
1912         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1913 }
1914
1915 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
1916 {
1917         unsigned int num_cq_pages, num_async_pdu_buf_pages;
1918         unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
1919         unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
1920
1921         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
1922                                       sizeof(struct sol_cqe));
1923         num_async_pdu_buf_pages =
1924                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1925                                        phba->params.defpdu_hdr_sz);
1926         num_async_pdu_buf_sgl_pages =
1927                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1928                                        sizeof(struct phys_addr));
1929         num_async_pdu_data_pages =
1930                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1931                                        phba->params.defpdu_data_sz);
1932         num_async_pdu_data_sgl_pages =
1933                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1934                                        sizeof(struct phys_addr));
1935
1936         phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
1937
1938         phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
1939                                                  BE_ISCSI_PDU_HEADER_SIZE;
1940         phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
1941                                             sizeof(struct hwi_context_memory);
1942
1943
1944         phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
1945             * (phba->params.wrbs_per_cxn)
1946             * phba->params.cxns_per_ctrl;
1947         wrb_sz_per_cxn =  sizeof(struct wrb_handle) *
1948                                  (phba->params.wrbs_per_cxn);
1949         phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
1950                                 phba->params.cxns_per_ctrl);
1951
1952         phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
1953                 phba->params.icds_per_ctrl;
1954         phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
1955                 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
1956
1957         phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
1958                 num_async_pdu_buf_pages * PAGE_SIZE;
1959         phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
1960                 num_async_pdu_data_pages * PAGE_SIZE;
1961         phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
1962                 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
1963         phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
1964                 num_async_pdu_data_sgl_pages * PAGE_SIZE;
1965         phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
1966                 phba->params.asyncpdus_per_ctrl *
1967                 sizeof(struct async_pdu_handle);
1968         phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
1969                 phba->params.asyncpdus_per_ctrl *
1970                 sizeof(struct async_pdu_handle);
1971         phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
1972                 sizeof(struct hwi_async_pdu_context) +
1973                 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
1974 }
1975
1976 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
1977 {
1978         struct be_mem_descriptor *mem_descr;
1979         dma_addr_t bus_add;
1980         struct mem_array *mem_arr, *mem_arr_orig;
1981         unsigned int i, j, alloc_size, curr_alloc_size;
1982
1983         phba->phwi_ctrlr = kmalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
1984         if (!phba->phwi_ctrlr)
1985                 return -ENOMEM;
1986
1987         phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
1988                                  GFP_KERNEL);
1989         if (!phba->init_mem) {
1990                 kfree(phba->phwi_ctrlr);
1991                 return -ENOMEM;
1992         }
1993
1994         mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
1995                                GFP_KERNEL);
1996         if (!mem_arr_orig) {
1997                 kfree(phba->init_mem);
1998                 kfree(phba->phwi_ctrlr);
1999                 return -ENOMEM;
2000         }
2001
2002         mem_descr = phba->init_mem;
2003         for (i = 0; i < SE_MEM_MAX; i++) {
2004                 j = 0;
2005                 mem_arr = mem_arr_orig;
2006                 alloc_size = phba->mem_req[i];
2007                 memset(mem_arr, 0, sizeof(struct mem_array) *
2008                        BEISCSI_MAX_FRAGS_INIT);
2009                 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2010                 do {
2011                         mem_arr->virtual_address = pci_alloc_consistent(
2012                                                         phba->pcidev,
2013                                                         curr_alloc_size,
2014                                                         &bus_add);
2015                         if (!mem_arr->virtual_address) {
2016                                 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2017                                         goto free_mem;
2018                                 if (curr_alloc_size -
2019                                         rounddown_pow_of_two(curr_alloc_size))
2020                                         curr_alloc_size = rounddown_pow_of_two
2021                                                              (curr_alloc_size);
2022                                 else
2023                                         curr_alloc_size = curr_alloc_size / 2;
2024                         } else {
2025                                 mem_arr->bus_address.u.
2026                                     a64.address = (__u64) bus_add;
2027                                 mem_arr->size = curr_alloc_size;
2028                                 alloc_size -= curr_alloc_size;
2029                                 curr_alloc_size = min(be_max_phys_size *
2030                                                       1024, alloc_size);
2031                                 j++;
2032                                 mem_arr++;
2033                         }
2034                 } while (alloc_size);
2035                 mem_descr->num_elements = j;
2036                 mem_descr->size_in_bytes = phba->mem_req[i];
2037                 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
2038                                                GFP_KERNEL);
2039                 if (!mem_descr->mem_array)
2040                         goto free_mem;
2041
2042                 memcpy(mem_descr->mem_array, mem_arr_orig,
2043                        sizeof(struct mem_array) * j);
2044                 mem_descr++;
2045         }
2046         kfree(mem_arr_orig);
2047         return 0;
2048 free_mem:
2049         mem_descr->num_elements = j;
2050         while ((i) || (j)) {
2051                 for (j = mem_descr->num_elements; j > 0; j--) {
2052                         pci_free_consistent(phba->pcidev,
2053                                             mem_descr->mem_array[j - 1].size,
2054                                             mem_descr->mem_array[j - 1].
2055                                             virtual_address,
2056                                             mem_descr->mem_array[j - 1].
2057                                             bus_address.u.a64.address);
2058                 }
2059                 if (i) {
2060                         i--;
2061                         kfree(mem_descr->mem_array);
2062                         mem_descr--;
2063                 }
2064         }
2065         kfree(mem_arr_orig);
2066         kfree(phba->init_mem);
2067         kfree(phba->phwi_ctrlr);
2068         return -ENOMEM;
2069 }
2070
2071 static int beiscsi_get_memory(struct beiscsi_hba *phba)
2072 {
2073         beiscsi_find_mem_req(phba);
2074         return beiscsi_alloc_mem(phba);
2075 }
2076
2077 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2078 {
2079         struct pdu_data_out *pdata_out;
2080         struct pdu_nop_out *pnop_out;
2081         struct be_mem_descriptor *mem_descr;
2082
2083         mem_descr = phba->init_mem;
2084         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2085         pdata_out =
2086             (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2087         memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2088
2089         AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2090                       IIOC_SCSI_DATA);
2091
2092         pnop_out =
2093             (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2094                                    virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2095
2096         memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2097         AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2098         AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2099         AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2100 }
2101
2102 static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
2103 {
2104         struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
2105         struct wrb_handle *pwrb_handle;
2106         struct hwi_controller *phwi_ctrlr;
2107         struct hwi_wrb_context *pwrb_context;
2108         struct iscsi_wrb *pwrb;
2109         unsigned int num_cxn_wrbh;
2110         unsigned int num_cxn_wrb, j, idx, index;
2111
2112         mem_descr_wrbh = phba->init_mem;
2113         mem_descr_wrbh += HWI_MEM_WRBH;
2114
2115         mem_descr_wrb = phba->init_mem;
2116         mem_descr_wrb += HWI_MEM_WRB;
2117
2118         idx = 0;
2119         pwrb_handle = mem_descr_wrbh->mem_array[idx].virtual_address;
2120         num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2121                         ((sizeof(struct wrb_handle)) *
2122                          phba->params.wrbs_per_cxn));
2123         phwi_ctrlr = phba->phwi_ctrlr;
2124
2125         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2126                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2127                 pwrb_context->pwrb_handle_base =
2128                                 kzalloc(sizeof(struct wrb_handle *) *
2129                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2130                 pwrb_context->pwrb_handle_basestd =
2131                                 kzalloc(sizeof(struct wrb_handle *) *
2132                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2133                 if (num_cxn_wrbh) {
2134                         pwrb_context->alloc_index = 0;
2135                         pwrb_context->wrb_handles_available = 0;
2136                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2137                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2138                                 pwrb_context->pwrb_handle_basestd[j] =
2139                                                                 pwrb_handle;
2140                                 pwrb_context->wrb_handles_available++;
2141                                 pwrb_handle->wrb_index = j;
2142                                 pwrb_handle++;
2143                         }
2144                         pwrb_context->free_index = 0;
2145                         num_cxn_wrbh--;
2146                 } else {
2147                         idx++;
2148                         pwrb_handle =
2149                             mem_descr_wrbh->mem_array[idx].virtual_address;
2150                         num_cxn_wrbh =
2151                             ((mem_descr_wrbh->mem_array[idx].size) /
2152                              ((sizeof(struct wrb_handle)) *
2153                               phba->params.wrbs_per_cxn));
2154                         pwrb_context->alloc_index = 0;
2155                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2156                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2157                                 pwrb_context->pwrb_handle_basestd[j] =
2158                                     pwrb_handle;
2159                                 pwrb_context->wrb_handles_available++;
2160                                 pwrb_handle->wrb_index = j;
2161                                 pwrb_handle++;
2162                         }
2163                         pwrb_context->free_index = 0;
2164                         num_cxn_wrbh--;
2165                 }
2166         }
2167         idx = 0;
2168         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2169         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2170                       ((sizeof(struct iscsi_wrb) *
2171                         phba->params.wrbs_per_cxn));
2172         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2173                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2174                 if (num_cxn_wrb) {
2175                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2176                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2177                                 pwrb_handle->pwrb = pwrb;
2178                                 pwrb++;
2179                         }
2180                         num_cxn_wrb--;
2181                 } else {
2182                         idx++;
2183                         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2184                         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2185                                       ((sizeof(struct iscsi_wrb) *
2186                                         phba->params.wrbs_per_cxn));
2187                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2188                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2189                                 pwrb_handle->pwrb = pwrb;
2190                                 pwrb++;
2191                         }
2192                         num_cxn_wrb--;
2193                 }
2194         }
2195 }
2196
2197 static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2198 {
2199         struct hwi_controller *phwi_ctrlr;
2200         struct hba_parameters *p = &phba->params;
2201         struct hwi_async_pdu_context *pasync_ctx;
2202         struct async_pdu_handle *pasync_header_h, *pasync_data_h;
2203         unsigned int index;
2204         struct be_mem_descriptor *mem_descr;
2205
2206         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2207         mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
2208
2209         phwi_ctrlr = phba->phwi_ctrlr;
2210         phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
2211                                 mem_descr->mem_array[0].virtual_address;
2212         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
2213         memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2214
2215         pasync_ctx->async_header.num_entries = p->asyncpdus_per_ctrl;
2216         pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz;
2217         pasync_ctx->async_data.buffer_size = p->defpdu_data_sz;
2218         pasync_ctx->async_data.num_entries = p->asyncpdus_per_ctrl;
2219
2220         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2221         mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
2222         if (mem_descr->mem_array[0].virtual_address) {
2223                 SE_DEBUG(DBG_LVL_8,
2224                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
2225                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2226         } else
2227                 shost_printk(KERN_WARNING, phba->shost,
2228                              "No Virtual address \n");
2229
2230         pasync_ctx->async_header.va_base =
2231                         mem_descr->mem_array[0].virtual_address;
2232
2233         pasync_ctx->async_header.pa_base.u.a64.address =
2234                         mem_descr->mem_array[0].bus_address.u.a64.address;
2235
2236         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2237         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2238         if (mem_descr->mem_array[0].virtual_address) {
2239                 SE_DEBUG(DBG_LVL_8,
2240                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
2241                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2242         } else
2243                 shost_printk(KERN_WARNING, phba->shost,
2244                             "No Virtual address \n");
2245         pasync_ctx->async_header.ring_base =
2246                         mem_descr->mem_array[0].virtual_address;
2247
2248         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2249         mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
2250         if (mem_descr->mem_array[0].virtual_address) {
2251                 SE_DEBUG(DBG_LVL_8,
2252                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
2253                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2254         } else
2255                 shost_printk(KERN_WARNING, phba->shost,
2256                             "No Virtual address \n");
2257
2258         pasync_ctx->async_header.handle_base =
2259                         mem_descr->mem_array[0].virtual_address;
2260         pasync_ctx->async_header.writables = 0;
2261         INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2262
2263         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2264         mem_descr += HWI_MEM_ASYNC_DATA_BUF;
2265         if (mem_descr->mem_array[0].virtual_address) {
2266                 SE_DEBUG(DBG_LVL_8,
2267                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
2268                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2269         } else
2270                 shost_printk(KERN_WARNING, phba->shost,
2271                             "No Virtual address \n");
2272         pasync_ctx->async_data.va_base =
2273                         mem_descr->mem_array[0].virtual_address;
2274         pasync_ctx->async_data.pa_base.u.a64.address =
2275                         mem_descr->mem_array[0].bus_address.u.a64.address;
2276
2277         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2278         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2279         if (mem_descr->mem_array[0].virtual_address) {
2280                 SE_DEBUG(DBG_LVL_8,
2281                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
2282                          "va=%p \n", mem_descr->mem_array[0].virtual_address);
2283         } else
2284                 shost_printk(KERN_WARNING, phba->shost,
2285                              "No Virtual address \n");
2286
2287         pasync_ctx->async_data.ring_base =
2288                         mem_descr->mem_array[0].virtual_address;
2289
2290         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2291         mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
2292         if (!mem_descr->mem_array[0].virtual_address)
2293                 shost_printk(KERN_WARNING, phba->shost,
2294                             "No Virtual address \n");
2295
2296         pasync_ctx->async_data.handle_base =
2297                         mem_descr->mem_array[0].virtual_address;
2298         pasync_ctx->async_data.writables = 0;
2299         INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2300
2301         pasync_header_h =
2302                 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
2303         pasync_data_h =
2304                 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
2305
2306         for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
2307                 pasync_header_h->cri = -1;
2308                 pasync_header_h->index = (char)index;
2309                 INIT_LIST_HEAD(&pasync_header_h->link);
2310                 pasync_header_h->pbuffer =
2311                         (void *)((unsigned long)
2312                         (pasync_ctx->async_header.va_base) +
2313                         (p->defpdu_hdr_sz * index));
2314
2315                 pasync_header_h->pa.u.a64.address =
2316                         pasync_ctx->async_header.pa_base.u.a64.address +
2317                         (p->defpdu_hdr_sz * index);
2318
2319                 list_add_tail(&pasync_header_h->link,
2320                                 &pasync_ctx->async_header.free_list);
2321                 pasync_header_h++;
2322                 pasync_ctx->async_header.free_entries++;
2323                 pasync_ctx->async_header.writables++;
2324
2325                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
2326                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2327                                header_busy_list);
2328                 pasync_data_h->cri = -1;
2329                 pasync_data_h->index = (char)index;
2330                 INIT_LIST_HEAD(&pasync_data_h->link);
2331                 pasync_data_h->pbuffer =
2332                         (void *)((unsigned long)
2333                         (pasync_ctx->async_data.va_base) +
2334                         (p->defpdu_data_sz * index));
2335
2336                 pasync_data_h->pa.u.a64.address =
2337                     pasync_ctx->async_data.pa_base.u.a64.address +
2338                     (p->defpdu_data_sz * index);
2339
2340                 list_add_tail(&pasync_data_h->link,
2341                               &pasync_ctx->async_data.free_list);
2342                 pasync_data_h++;
2343                 pasync_ctx->async_data.free_entries++;
2344                 pasync_ctx->async_data.writables++;
2345
2346                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
2347         }
2348
2349         pasync_ctx->async_header.host_write_ptr = 0;
2350         pasync_ctx->async_header.ep_read_ptr = -1;
2351         pasync_ctx->async_data.host_write_ptr = 0;
2352         pasync_ctx->async_data.ep_read_ptr = -1;
2353 }
2354
2355 static int
2356 be_sgl_create_contiguous(void *virtual_address,
2357                          u64 physical_address, u32 length,
2358                          struct be_dma_mem *sgl)
2359 {
2360         WARN_ON(!virtual_address);
2361         WARN_ON(!physical_address);
2362         WARN_ON(!length > 0);
2363         WARN_ON(!sgl);
2364
2365         sgl->va = virtual_address;
2366         sgl->dma = physical_address;
2367         sgl->size = length;
2368
2369         return 0;
2370 }
2371
2372 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2373 {
2374         memset(sgl, 0, sizeof(*sgl));
2375 }
2376
2377 static void
2378 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2379                      struct mem_array *pmem, struct be_dma_mem *sgl)
2380 {
2381         if (sgl->va)
2382                 be_sgl_destroy_contiguous(sgl);
2383
2384         be_sgl_create_contiguous(pmem->virtual_address,
2385                                  pmem->bus_address.u.a64.address,
2386                                  pmem->size, sgl);
2387 }
2388
2389 static void
2390 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2391                            struct mem_array *pmem, struct be_dma_mem *sgl)
2392 {
2393         if (sgl->va)
2394                 be_sgl_destroy_contiguous(sgl);
2395
2396         be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2397                                  pmem->bus_address.u.a64.address,
2398                                  pmem->size, sgl);
2399 }
2400
2401 static int be_fill_queue(struct be_queue_info *q,
2402                 u16 len, u16 entry_size, void *vaddress)
2403 {
2404         struct be_dma_mem *mem = &q->dma_mem;
2405
2406         memset(q, 0, sizeof(*q));
2407         q->len = len;
2408         q->entry_size = entry_size;
2409         mem->size = len * entry_size;
2410         mem->va = vaddress;
2411         if (!mem->va)
2412                 return -ENOMEM;
2413         memset(mem->va, 0, mem->size);
2414         return 0;
2415 }
2416
2417 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
2418                              struct hwi_context_memory *phwi_context)
2419 {
2420         unsigned int i, num_eq_pages;
2421         int ret, eq_for_mcc;
2422         struct be_queue_info *eq;
2423         struct be_dma_mem *mem;
2424         void *eq_vaddress;
2425         dma_addr_t paddr;
2426
2427         num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
2428                                       sizeof(struct be_eq_entry));
2429
2430         if (phba->msix_enabled)
2431                 eq_for_mcc = 1;
2432         else
2433                 eq_for_mcc = 0;
2434         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
2435                 eq = &phwi_context->be_eq[i].q;
2436                 mem = &eq->dma_mem;
2437                 phwi_context->be_eq[i].phba = phba;
2438                 eq_vaddress = pci_alloc_consistent(phba->pcidev,
2439                                                      num_eq_pages * PAGE_SIZE,
2440                                                      &paddr);
2441                 if (!eq_vaddress)
2442                         goto create_eq_error;
2443
2444                 mem->va = eq_vaddress;
2445                 ret = be_fill_queue(eq, phba->params.num_eq_entries,
2446                                     sizeof(struct be_eq_entry), eq_vaddress);
2447                 if (ret) {
2448                         shost_printk(KERN_ERR, phba->shost,
2449                                      "be_fill_queue Failed for EQ \n");
2450                         goto create_eq_error;
2451                 }
2452
2453                 mem->dma = paddr;
2454                 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2455                                             phwi_context->cur_eqd);
2456                 if (ret) {
2457                         shost_printk(KERN_ERR, phba->shost,
2458                                      "beiscsi_cmd_eq_create"
2459                                      "Failedfor EQ \n");
2460                         goto create_eq_error;
2461                 }
2462                 SE_DEBUG(DBG_LVL_8, "eqid = %d\n", phwi_context->be_eq[i].q.id);
2463         }
2464         return 0;
2465 create_eq_error:
2466         for (i = 0; i < (phba->num_cpus + 1); i++) {
2467                 eq = &phwi_context->be_eq[i].q;
2468                 mem = &eq->dma_mem;
2469                 if (mem->va)
2470                         pci_free_consistent(phba->pcidev, num_eq_pages
2471                                             * PAGE_SIZE,
2472                                             mem->va, mem->dma);
2473         }
2474         return ret;
2475 }
2476
2477 static int beiscsi_create_cqs(struct beiscsi_hba *phba,
2478                              struct hwi_context_memory *phwi_context)
2479 {
2480         unsigned int i, num_cq_pages;
2481         int ret;
2482         struct be_queue_info *cq, *eq;
2483         struct be_dma_mem *mem;
2484         struct be_eq_obj *pbe_eq;
2485         void *cq_vaddress;
2486         dma_addr_t paddr;
2487
2488         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2489                                       sizeof(struct sol_cqe));
2490
2491         for (i = 0; i < phba->num_cpus; i++) {
2492                 cq = &phwi_context->be_cq[i];
2493                 eq = &phwi_context->be_eq[i].q;
2494                 pbe_eq = &phwi_context->be_eq[i];
2495                 pbe_eq->cq = cq;
2496                 pbe_eq->phba = phba;
2497                 mem = &cq->dma_mem;
2498                 cq_vaddress = pci_alloc_consistent(phba->pcidev,
2499                                                      num_cq_pages * PAGE_SIZE,
2500                                                      &paddr);
2501                 if (!cq_vaddress)
2502                         goto create_cq_error;
2503                 ret = be_fill_queue(cq, phba->params.num_cq_entries,
2504                                     sizeof(struct sol_cqe), cq_vaddress);
2505                 if (ret) {
2506                         shost_printk(KERN_ERR, phba->shost,
2507                                      "be_fill_queue Failed for ISCSI CQ \n");
2508                         goto create_cq_error;
2509                 }
2510
2511                 mem->dma = paddr;
2512                 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
2513                                             false, 0);
2514                 if (ret) {
2515                         shost_printk(KERN_ERR, phba->shost,
2516                                      "beiscsi_cmd_eq_create"
2517                                      "Failed for ISCSI CQ \n");
2518                         goto create_cq_error;
2519                 }
2520                 SE_DEBUG(DBG_LVL_8, "iscsi cq_id is %d for eq_id %d\n",
2521                                                  cq->id, eq->id);
2522                 SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
2523         }
2524         return 0;
2525
2526 create_cq_error:
2527         for (i = 0; i < phba->num_cpus; i++) {
2528                 cq = &phwi_context->be_cq[i];
2529                 mem = &cq->dma_mem;
2530                 if (mem->va)
2531                         pci_free_consistent(phba->pcidev, num_cq_pages
2532                                             * PAGE_SIZE,
2533                                             mem->va, mem->dma);
2534         }
2535         return ret;
2536
2537 }
2538
2539 static int
2540 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2541                        struct hwi_context_memory *phwi_context,
2542                        struct hwi_controller *phwi_ctrlr,
2543                        unsigned int def_pdu_ring_sz)
2544 {
2545         unsigned int idx;
2546         int ret;
2547         struct be_queue_info *dq, *cq;
2548         struct be_dma_mem *mem;
2549         struct be_mem_descriptor *mem_descr;
2550         void *dq_vaddress;
2551
2552         idx = 0;
2553         dq = &phwi_context->be_def_hdrq;
2554         cq = &phwi_context->be_cq[0];
2555         mem = &dq->dma_mem;
2556         mem_descr = phba->init_mem;
2557         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2558         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2559         ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
2560                             sizeof(struct phys_addr),
2561                             sizeof(struct phys_addr), dq_vaddress);
2562         if (ret) {
2563                 shost_printk(KERN_ERR, phba->shost,
2564                              "be_fill_queue Failed for DEF PDU HDR\n");
2565                 return ret;
2566         }
2567         mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2568         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
2569                                               def_pdu_ring_sz,
2570                                               phba->params.defpdu_hdr_sz);
2571         if (ret) {
2572                 shost_printk(KERN_ERR, phba->shost,
2573                              "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2574                 return ret;
2575         }
2576         phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
2577         SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n",
2578                  phwi_context->be_def_hdrq.id);
2579         hwi_post_async_buffers(phba, 1);
2580         return 0;
2581 }
2582
2583 static int
2584 beiscsi_create_def_data(struct beiscsi_hba *phba,
2585                         struct hwi_context_memory *phwi_context,
2586                         struct hwi_controller *phwi_ctrlr,
2587                         unsigned int def_pdu_ring_sz)
2588 {
2589         unsigned int idx;
2590         int ret;
2591         struct be_queue_info *dataq, *cq;
2592         struct be_dma_mem *mem;
2593         struct be_mem_descriptor *mem_descr;
2594         void *dq_vaddress;
2595
2596         idx = 0;
2597         dataq = &phwi_context->be_def_dataq;
2598         cq = &phwi_context->be_cq[0];
2599         mem = &dataq->dma_mem;
2600         mem_descr = phba->init_mem;
2601         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2602         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2603         ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
2604                             sizeof(struct phys_addr),
2605                             sizeof(struct phys_addr), dq_vaddress);
2606         if (ret) {
2607                 shost_printk(KERN_ERR, phba->shost,
2608                              "be_fill_queue Failed for DEF PDU DATA\n");
2609                 return ret;
2610         }
2611         mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2612         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
2613                                               def_pdu_ring_sz,
2614                                               phba->params.defpdu_data_sz);
2615         if (ret) {
2616                 shost_printk(KERN_ERR, phba->shost,
2617                              "be_cmd_create_default_pdu_queue Failed"
2618                              " for DEF PDU DATA\n");
2619                 return ret;
2620         }
2621         phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
2622         SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n",
2623                  phwi_context->be_def_dataq.id);
2624         hwi_post_async_buffers(phba, 0);
2625         SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED \n");
2626         return 0;
2627 }
2628
2629 static int
2630 beiscsi_post_pages(struct beiscsi_hba *phba)
2631 {
2632         struct be_mem_descriptor *mem_descr;
2633         struct mem_array *pm_arr;
2634         unsigned int page_offset, i;
2635         struct be_dma_mem sgl;
2636         int status;
2637
2638         mem_descr = phba->init_mem;
2639         mem_descr += HWI_MEM_SGE;
2640         pm_arr = mem_descr->mem_array;
2641
2642         page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
2643                         phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
2644         for (i = 0; i < mem_descr->num_elements; i++) {
2645                 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
2646                 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
2647                                                 page_offset,
2648                                                 (pm_arr->size / PAGE_SIZE));
2649                 page_offset += pm_arr->size / PAGE_SIZE;
2650                 if (status != 0) {
2651                         shost_printk(KERN_ERR, phba->shost,
2652                                      "post sgl failed.\n");
2653                         return status;
2654                 }
2655                 pm_arr++;
2656         }
2657         SE_DEBUG(DBG_LVL_8, "POSTED PAGES \n");
2658         return 0;
2659 }
2660
2661 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
2662 {
2663         struct be_dma_mem *mem = &q->dma_mem;
2664         if (mem->va)
2665                 pci_free_consistent(phba->pcidev, mem->size,
2666                         mem->va, mem->dma);
2667 }
2668
2669 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
2670                 u16 len, u16 entry_size)
2671 {
2672         struct be_dma_mem *mem = &q->dma_mem;
2673
2674         memset(q, 0, sizeof(*q));
2675         q->len = len;
2676         q->entry_size = entry_size;
2677         mem->size = len * entry_size;
2678         mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
2679         if (!mem->va)
2680                 return -1;
2681         memset(mem->va, 0, mem->size);
2682         return 0;
2683 }
2684
2685 static int
2686 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
2687                          struct hwi_context_memory *phwi_context,
2688                          struct hwi_controller *phwi_ctrlr)
2689 {
2690         unsigned int wrb_mem_index, offset, size, num_wrb_rings;
2691         u64 pa_addr_lo;
2692         unsigned int idx, num, i;
2693         struct mem_array *pwrb_arr;
2694         void *wrb_vaddr;
2695         struct be_dma_mem sgl;
2696         struct be_mem_descriptor *mem_descr;
2697         int status;
2698
2699         idx = 0;
2700         mem_descr = phba->init_mem;
2701         mem_descr += HWI_MEM_WRB;
2702         pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
2703                            GFP_KERNEL);
2704         if (!pwrb_arr) {
2705                 shost_printk(KERN_ERR, phba->shost,
2706                              "Memory alloc failed in create wrb ring.\n");
2707                 return -ENOMEM;
2708         }
2709         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2710         pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
2711         num_wrb_rings = mem_descr->mem_array[idx].size /
2712                 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
2713
2714         for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
2715                 if (num_wrb_rings) {
2716                         pwrb_arr[num].virtual_address = wrb_vaddr;
2717                         pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
2718                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2719                                             sizeof(struct iscsi_wrb);
2720                         wrb_vaddr += pwrb_arr[num].size;
2721                         pa_addr_lo += pwrb_arr[num].size;
2722                         num_wrb_rings--;
2723                 } else {
2724                         idx++;
2725                         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2726                         pa_addr_lo = mem_descr->mem_array[idx].\
2727                                         bus_address.u.a64.address;
2728                         num_wrb_rings = mem_descr->mem_array[idx].size /
2729                                         (phba->params.wrbs_per_cxn *
2730                                         sizeof(struct iscsi_wrb));
2731                         pwrb_arr[num].virtual_address = wrb_vaddr;
2732                         pwrb_arr[num].bus_address.u.a64.address\
2733                                                 = pa_addr_lo;
2734                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2735                                                  sizeof(struct iscsi_wrb);
2736                         wrb_vaddr += pwrb_arr[num].size;
2737                         pa_addr_lo   += pwrb_arr[num].size;
2738                         num_wrb_rings--;
2739                 }
2740         }
2741         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2742                 wrb_mem_index = 0;
2743                 offset = 0;
2744                 size = 0;
2745
2746                 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
2747                 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
2748                                             &phwi_context->be_wrbq[i]);
2749                 if (status != 0) {
2750                         shost_printk(KERN_ERR, phba->shost,
2751                                      "wrbq create failed.");
2752                         return status;
2753                 }
2754                 phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i].
2755                                                                    id;
2756         }
2757         kfree(pwrb_arr);
2758         return 0;
2759 }
2760
2761 static void free_wrb_handles(struct beiscsi_hba *phba)
2762 {
2763         unsigned int index;
2764         struct hwi_controller *phwi_ctrlr;
2765         struct hwi_wrb_context *pwrb_context;
2766
2767         phwi_ctrlr = phba->phwi_ctrlr;
2768         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2769                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2770                 kfree(pwrb_context->pwrb_handle_base);
2771                 kfree(pwrb_context->pwrb_handle_basestd);
2772         }
2773 }
2774
2775 static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
2776 {
2777         struct be_queue_info *q;
2778         struct be_ctrl_info *ctrl = &phba->ctrl;
2779
2780         q = &phba->ctrl.mcc_obj.q;
2781         if (q->created)
2782                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
2783         be_queue_free(phba, q);
2784
2785         q = &phba->ctrl.mcc_obj.cq;
2786         if (q->created)
2787                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
2788         be_queue_free(phba, q);
2789 }
2790
2791 static void hwi_cleanup(struct beiscsi_hba *phba)
2792 {
2793         struct be_queue_info *q;
2794         struct be_ctrl_info *ctrl = &phba->ctrl;
2795         struct hwi_controller *phwi_ctrlr;
2796         struct hwi_context_memory *phwi_context;
2797         int i, eq_num;
2798
2799         phwi_ctrlr = phba->phwi_ctrlr;
2800         phwi_context = phwi_ctrlr->phwi_ctxt;
2801         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2802                 q = &phwi_context->be_wrbq[i];
2803                 if (q->created)
2804                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
2805         }
2806         free_wrb_handles(phba);
2807
2808         q = &phwi_context->be_def_hdrq;
2809         if (q->created)
2810                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2811
2812         q = &phwi_context->be_def_dataq;
2813         if (q->created)
2814                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2815
2816         beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
2817
2818         for (i = 0; i < (phba->num_cpus); i++) {
2819                 q = &phwi_context->be_cq[i];
2820                 if (q->created)
2821                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
2822         }
2823         if (phba->msix_enabled)
2824                 eq_num = 1;
2825         else
2826                 eq_num = 0;
2827         for (i = 0; i < (phba->num_cpus + eq_num); i++) {
2828                 q = &phwi_context->be_eq[i].q;
2829                 if (q->created)
2830                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
2831         }
2832         be_mcc_queues_destroy(phba);
2833 }
2834
2835 static int be_mcc_queues_create(struct beiscsi_hba *phba,
2836                                 struct hwi_context_memory *phwi_context)
2837 {
2838         struct be_queue_info *q, *cq;
2839         struct be_ctrl_info *ctrl = &phba->ctrl;
2840
2841         /* Alloc MCC compl queue */
2842         cq = &phba->ctrl.mcc_obj.cq;
2843         if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
2844                         sizeof(struct be_mcc_compl)))
2845                 goto err;
2846         /* Ask BE to create MCC compl queue; */
2847         if (phba->msix_enabled) {
2848                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
2849                                          [phba->num_cpus].q, false, true, 0))
2850                 goto mcc_cq_free;
2851         } else {
2852                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
2853                                           false, true, 0))
2854                 goto mcc_cq_free;
2855         }
2856
2857         /* Alloc MCC queue */
2858         q = &phba->ctrl.mcc_obj.q;
2859         if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2860                 goto mcc_cq_destroy;
2861
2862         /* Ask BE to create MCC queue */
2863         if (beiscsi_cmd_mccq_create(phba, q, cq))
2864                 goto mcc_q_free;
2865
2866         return 0;
2867
2868 mcc_q_free:
2869         be_queue_free(phba, q);
2870 mcc_cq_destroy:
2871         beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
2872 mcc_cq_free:
2873         be_queue_free(phba, cq);
2874 err:
2875         return -1;
2876 }
2877
2878 static int find_num_cpus(void)
2879 {
2880         int  num_cpus = 0;
2881
2882         num_cpus = num_online_cpus();
2883         if (num_cpus >= MAX_CPUS)
2884                 num_cpus = MAX_CPUS - 1;
2885
2886         SE_DEBUG(DBG_LVL_8, "num_cpus = %d \n", num_cpus);
2887         return num_cpus;
2888 }
2889
2890 static int hwi_init_port(struct beiscsi_hba *phba)
2891 {
2892         struct hwi_controller *phwi_ctrlr;
2893         struct hwi_context_memory *phwi_context;
2894         unsigned int def_pdu_ring_sz;
2895         struct be_ctrl_info *ctrl = &phba->ctrl;
2896         int status;
2897
2898         def_pdu_ring_sz =
2899                 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
2900         phwi_ctrlr = phba->phwi_ctrlr;
2901         phwi_context = phwi_ctrlr->phwi_ctxt;
2902         phwi_context->max_eqd = 0;
2903         phwi_context->min_eqd = 0;
2904         phwi_context->cur_eqd = 64;
2905         be_cmd_fw_initialize(&phba->ctrl);
2906
2907         status = beiscsi_create_eqs(phba, phwi_context);
2908         if (status != 0) {
2909                 shost_printk(KERN_ERR, phba->shost, "EQ not created \n");
2910                 goto error;
2911         }
2912
2913         status = be_mcc_queues_create(phba, phwi_context);
2914         if (status != 0)
2915                 goto error;
2916
2917         status = mgmt_check_supported_fw(ctrl, phba);
2918         if (status != 0) {
2919                 shost_printk(KERN_ERR, phba->shost,
2920                              "Unsupported fw version \n");
2921                 goto error;
2922         }
2923
2924         status = beiscsi_create_cqs(phba, phwi_context);
2925         if (status != 0) {
2926                 shost_printk(KERN_ERR, phba->shost, "CQ not created\n");
2927                 goto error;
2928         }
2929
2930         status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
2931                                         def_pdu_ring_sz);
2932         if (status != 0) {
2933                 shost_printk(KERN_ERR, phba->shost,
2934                              "Default Header not created\n");
2935                 goto error;
2936         }
2937
2938         status = beiscsi_create_def_data(phba, phwi_context,
2939                                          phwi_ctrlr, def_pdu_ring_sz);
2940         if (status != 0) {
2941                 shost_printk(KERN_ERR, phba->shost,
2942                              "Default Data not created\n");
2943                 goto error;
2944         }
2945
2946         status = beiscsi_post_pages(phba);
2947         if (status != 0) {
2948                 shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n");
2949                 goto error;
2950         }
2951
2952         status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
2953         if (status != 0) {
2954                 shost_printk(KERN_ERR, phba->shost,
2955                              "WRB Rings not created\n");
2956                 goto error;
2957         }
2958
2959         SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n");
2960         return 0;
2961
2962 error:
2963         shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed");
2964         hwi_cleanup(phba);
2965         return -ENOMEM;
2966 }
2967
2968 static int hwi_init_controller(struct beiscsi_hba *phba)
2969 {
2970         struct hwi_controller *phwi_ctrlr;
2971
2972         phwi_ctrlr = phba->phwi_ctrlr;
2973         if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
2974                 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
2975                     init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
2976                 SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p \n",
2977                          phwi_ctrlr->phwi_ctxt);
2978         } else {
2979                 shost_printk(KERN_ERR, phba->shost,
2980                              "HWI_MEM_ADDN_CONTEXT is more than one element."
2981                              "Failing to load\n");
2982                 return -ENOMEM;
2983         }
2984
2985         iscsi_init_global_templates(phba);
2986         beiscsi_init_wrb_handle(phba);
2987         hwi_init_async_pdu_ctx(phba);
2988         if (hwi_init_port(phba) != 0) {
2989                 shost_printk(KERN_ERR, phba->shost,
2990                              "hwi_init_controller failed\n");
2991                 return -ENOMEM;
2992         }
2993         return 0;
2994 }
2995
2996 static void beiscsi_free_mem(struct beiscsi_hba *phba)
2997 {
2998         struct be_mem_descriptor *mem_descr;
2999         int i, j;
3000
3001         mem_descr = phba->init_mem;
3002         i = 0;
3003         j = 0;
3004         for (i = 0; i < SE_MEM_MAX; i++) {
3005                 for (j = mem_descr->num_elements; j > 0; j--) {
3006                         pci_free_consistent(phba->pcidev,
3007                           mem_descr->mem_array[j - 1].size,
3008                           mem_descr->mem_array[j - 1].virtual_address,
3009                           mem_descr->mem_array[j - 1].bus_address.
3010                                 u.a64.address);
3011                 }
3012                 kfree(mem_descr->mem_array);
3013                 mem_descr++;
3014         }
3015         kfree(phba->init_mem);
3016         kfree(phba->phwi_ctrlr);
3017 }
3018
3019 static int beiscsi_init_controller(struct beiscsi_hba *phba)
3020 {
3021         int ret = -ENOMEM;
3022
3023         ret = beiscsi_get_memory(phba);
3024         if (ret < 0) {
3025                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -"
3026                              "Failed in beiscsi_alloc_memory \n");
3027                 return ret;
3028         }
3029
3030         ret = hwi_init_controller(phba);
3031         if (ret)
3032                 goto free_init;
3033         SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller");
3034         return 0;
3035
3036 free_init:
3037         beiscsi_free_mem(phba);
3038         return -ENOMEM;
3039 }
3040
3041 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
3042 {
3043         struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
3044         struct sgl_handle *psgl_handle;
3045         struct iscsi_sge *pfrag;
3046         unsigned int arr_index, i, idx;
3047
3048         phba->io_sgl_hndl_avbl = 0;
3049         phba->eh_sgl_hndl_avbl = 0;
3050
3051         mem_descr_sglh = phba->init_mem;
3052         mem_descr_sglh += HWI_MEM_SGLH;
3053         if (1 == mem_descr_sglh->num_elements) {
3054                 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3055                                                  phba->params.ios_per_ctrl,
3056                                                  GFP_KERNEL);
3057                 if (!phba->io_sgl_hndl_base) {
3058                         shost_printk(KERN_ERR, phba->shost,
3059                                      "Mem Alloc Failed. Failing to load\n");
3060                         return -ENOMEM;
3061                 }
3062                 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3063                                                  (phba->params.icds_per_ctrl -
3064                                                  phba->params.ios_per_ctrl),
3065                                                  GFP_KERNEL);
3066                 if (!phba->eh_sgl_hndl_base) {
3067                         kfree(phba->io_sgl_hndl_base);
3068                         shost_printk(KERN_ERR, phba->shost,
3069                                      "Mem Alloc Failed. Failing to load\n");
3070                         return -ENOMEM;
3071                 }
3072         } else {
3073                 shost_printk(KERN_ERR, phba->shost,
3074                              "HWI_MEM_SGLH is more than one element."
3075                              "Failing to load\n");
3076                 return -ENOMEM;
3077         }
3078
3079         arr_index = 0;
3080         idx = 0;
3081         while (idx < mem_descr_sglh->num_elements) {
3082                 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
3083
3084                 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
3085                       sizeof(struct sgl_handle)); i++) {
3086                         if (arr_index < phba->params.ios_per_ctrl) {
3087                                 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
3088                                 phba->io_sgl_hndl_avbl++;
3089                                 arr_index++;
3090                         } else {
3091                                 phba->eh_sgl_hndl_base[arr_index -
3092                                         phba->params.ios_per_ctrl] =
3093                                                                 psgl_handle;
3094                                 arr_index++;
3095                                 phba->eh_sgl_hndl_avbl++;
3096                         }
3097                         psgl_handle++;
3098                 }
3099                 idx++;
3100         }
3101         SE_DEBUG(DBG_LVL_8,
3102                  "phba->io_sgl_hndl_avbl=%d"
3103                  "phba->eh_sgl_hndl_avbl=%d \n",
3104                  phba->io_sgl_hndl_avbl,
3105                  phba->eh_sgl_hndl_avbl);
3106         mem_descr_sg = phba->init_mem;
3107         mem_descr_sg += HWI_MEM_SGE;
3108         SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d \n",
3109                  mem_descr_sg->num_elements);
3110         arr_index = 0;
3111         idx = 0;
3112         while (idx < mem_descr_sg->num_elements) {
3113                 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
3114
3115                 for (i = 0;
3116                      i < (mem_descr_sg->mem_array[idx].size) /
3117                      (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
3118                      i++) {
3119                         if (arr_index < phba->params.ios_per_ctrl)
3120                                 psgl_handle = phba->io_sgl_hndl_base[arr_index];
3121                         else
3122                                 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
3123                                                 phba->params.ios_per_ctrl];
3124                         psgl_handle->pfrag = pfrag;
3125                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
3126                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
3127                         pfrag += phba->params.num_sge_per_io;
3128                         psgl_handle->sgl_index =
3129                                 phba->fw_config.iscsi_icd_start + arr_index++;
3130                 }
3131                 idx++;
3132         }
3133         phba->io_sgl_free_index = 0;
3134         phba->io_sgl_alloc_index = 0;
3135         phba->eh_sgl_free_index = 0;
3136         phba->eh_sgl_alloc_index = 0;
3137         return 0;
3138 }
3139
3140 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
3141 {
3142         int i, new_cid;
3143
3144         phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
3145                                   GFP_KERNEL);
3146         if (!phba->cid_array) {
3147                 shost_printk(KERN_ERR, phba->shost,
3148                              "Failed to allocate memory in "
3149                              "hba_setup_cid_tbls\n");
3150                 return -ENOMEM;
3151         }
3152         phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
3153                                  phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
3154         if (!phba->ep_array) {
3155                 shost_printk(KERN_ERR, phba->shost,
3156                              "Failed to allocate memory in "
3157                              "hba_setup_cid_tbls \n");
3158                 kfree(phba->cid_array);
3159                 return -ENOMEM;
3160         }
3161         new_cid = phba->fw_config.iscsi_cid_start;
3162         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3163                 phba->cid_array[i] = new_cid;
3164                 new_cid += 2;
3165         }
3166         phba->avlbl_cids = phba->params.cxns_per_ctrl;
3167         return 0;
3168 }
3169
3170 static unsigned char hwi_enable_intr(struct beiscsi_hba *phba)
3171 {
3172         struct be_ctrl_info *ctrl = &phba->ctrl;
3173         struct hwi_controller *phwi_ctrlr;
3174         struct hwi_context_memory *phwi_context;
3175         struct be_queue_info *eq;
3176         u8 __iomem *addr;
3177         u32 reg, i;
3178         u32 enabled;
3179
3180         phwi_ctrlr = phba->phwi_ctrlr;
3181         phwi_context = phwi_ctrlr->phwi_ctxt;
3182
3183         addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
3184                         PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
3185         reg = ioread32(addr);
3186         SE_DEBUG(DBG_LVL_8, "reg =x%08x \n", reg);
3187
3188         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3189         if (!enabled) {
3190                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3191                 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p \n", reg, addr);
3192                 iowrite32(reg, addr);
3193                 if (!phba->msix_enabled) {
3194                         eq = &phwi_context->be_eq[0].q;
3195                         SE_DEBUG(DBG_LVL_8, "eq->id=%d \n", eq->id);
3196                         hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3197                 } else {
3198                         for (i = 0; i <= phba->num_cpus; i++) {
3199                                 eq = &phwi_context->be_eq[i].q;
3200                                 SE_DEBUG(DBG_LVL_8, "eq->id=%d \n", eq->id);
3201                                 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3202                         }
3203                 }
3204         }
3205         return true;
3206 }
3207
3208 static void hwi_disable_intr(struct beiscsi_hba *phba)
3209 {
3210         struct be_ctrl_info *ctrl = &phba->ctrl;
3211
3212         u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
3213         u32 reg = ioread32(addr);
3214
3215         u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3216         if (enabled) {
3217                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3218                 iowrite32(reg, addr);
3219         } else
3220                 shost_printk(KERN_WARNING, phba->shost,
3221                              "In hwi_disable_intr, Already Disabled \n");
3222 }
3223
3224 static int beiscsi_init_port(struct beiscsi_hba *phba)
3225 {
3226         int ret;
3227
3228         ret = beiscsi_init_controller(phba);
3229         if (ret < 0) {
3230                 shost_printk(KERN_ERR, phba->shost,
3231                              "beiscsi_dev_probe - Failed in"
3232                              "beiscsi_init_controller \n");
3233                 return ret;
3234         }
3235         ret = beiscsi_init_sgl_handle(phba);
3236         if (ret < 0) {
3237                 shost_printk(KERN_ERR, phba->shost,
3238                              "beiscsi_dev_probe - Failed in"
3239                              "beiscsi_init_sgl_handle \n");
3240                 goto do_cleanup_ctrlr;
3241         }
3242
3243         if (hba_setup_cid_tbls(phba)) {
3244                 shost_printk(KERN_ERR, phba->shost,
3245                              "Failed in hba_setup_cid_tbls\n");
3246                 kfree(phba->io_sgl_hndl_base);
3247                 kfree(phba->eh_sgl_hndl_base);
3248                 goto do_cleanup_ctrlr;
3249         }
3250
3251         return ret;
3252
3253 do_cleanup_ctrlr:
3254         hwi_cleanup(phba);
3255         return ret;
3256 }
3257
3258 static void hwi_purge_eq(struct beiscsi_hba *phba)
3259 {
3260         struct hwi_controller *phwi_ctrlr;
3261         struct hwi_context_memory *phwi_context;
3262         struct be_queue_info *eq;
3263         struct be_eq_entry *eqe = NULL;
3264         int i, eq_msix;
3265         unsigned int num_processed;
3266
3267         phwi_ctrlr = phba->phwi_ctrlr;
3268         phwi_context = phwi_ctrlr->phwi_ctxt;
3269         if (phba->msix_enabled)
3270                 eq_msix = 1;
3271         else
3272                 eq_msix = 0;
3273
3274         for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
3275                 eq = &phwi_context->be_eq[i].q;
3276                 eqe = queue_tail_node(eq);
3277                 num_processed = 0;
3278                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
3279                                         & EQE_VALID_MASK) {
3280                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
3281                         queue_tail_inc(eq);
3282                         eqe = queue_tail_node(eq);
3283                         num_processed++;
3284                 }
3285
3286                 if (num_processed)
3287                         hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
3288         }
3289 }
3290
3291 static void beiscsi_clean_port(struct beiscsi_hba *phba)
3292 {
3293         unsigned char mgmt_status;
3294
3295         mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
3296         if (mgmt_status)
3297                 shost_printk(KERN_WARNING, phba->shost,
3298                              "mgmt_epfw_cleanup FAILED \n");
3299
3300         hwi_purge_eq(phba);
3301         hwi_cleanup(phba);
3302         kfree(phba->io_sgl_hndl_base);
3303         kfree(phba->eh_sgl_hndl_base);
3304         kfree(phba->cid_array);
3305         kfree(phba->ep_array);
3306 }
3307
3308 void
3309 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
3310                            struct beiscsi_offload_params *params)
3311 {
3312         struct wrb_handle *pwrb_handle;
3313         struct iscsi_target_context_update_wrb *pwrb = NULL;
3314         struct be_mem_descriptor *mem_descr;
3315         struct beiscsi_hba *phba = beiscsi_conn->phba;
3316         u32 doorbell = 0;
3317
3318         /*
3319          * We can always use 0 here because it is reserved by libiscsi for
3320          * login/startup related tasks.
3321          */
3322         pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid -
3323                                        phba->fw_config.iscsi_cid_start));
3324         pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb;
3325         memset(pwrb, 0, sizeof(*pwrb));
3326         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3327                       max_burst_length, pwrb, params->dw[offsetof
3328                       (struct amap_beiscsi_offload_params,
3329                       max_burst_length) / 32]);
3330         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3331                       max_send_data_segment_length, pwrb,
3332                       params->dw[offsetof(struct amap_beiscsi_offload_params,
3333                       max_send_data_segment_length) / 32]);
3334         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3335                       first_burst_length,
3336                       pwrb,
3337                       params->dw[offsetof(struct amap_beiscsi_offload_params,
3338                       first_burst_length) / 32]);
3339
3340         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb,
3341                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3342                       erl) / 32] & OFFLD_PARAMS_ERL));
3343         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb,
3344                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3345                       dde) / 32] & OFFLD_PARAMS_DDE) >> 2);
3346         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb,
3347                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3348                       hde) / 32] & OFFLD_PARAMS_HDE) >> 3);
3349         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb,
3350                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3351                       ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4);
3352         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb,
3353                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3354                        imd) / 32] & OFFLD_PARAMS_IMD) >> 5);
3355         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn,
3356                       pwrb,
3357                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3358                       exp_statsn) / 32] + 1));
3359         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb,
3360                       0x7);
3361         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx,
3362                       pwrb, pwrb_handle->wrb_index);
3363         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb,
3364                       pwrb, pwrb_handle->nxt_wrb_index);
3365         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3366                         session_state, pwrb, 0);
3367         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack,
3368                       pwrb, 1);
3369         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq,
3370                       pwrb, 0);
3371         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb,
3372                       0);
3373
3374         mem_descr = phba->init_mem;
3375         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
3376
3377         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3378                         pad_buffer_addr_hi, pwrb,
3379                       mem_descr->mem_array[0].bus_address.u.a32.address_hi);
3380         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3381                         pad_buffer_addr_lo, pwrb,
3382                       mem_descr->mem_array[0].bus_address.u.a32.address_lo);
3383
3384         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));
3385
3386         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3387         doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
3388                              << DB_DEF_PDU_WRB_INDEX_SHIFT;
3389         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3390
3391         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3392 }
3393
3394 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
3395                               int *index, int *age)
3396 {
3397         *index = (int)itt;
3398         if (age)
3399                 *age = conn->session->age;
3400 }
3401
3402 /**
3403  * beiscsi_alloc_pdu - allocates pdu and related resources
3404  * @task: libiscsi task
3405  * @opcode: opcode of pdu for task
3406  *
3407  * This is called with the session lock held. It will allocate
3408  * the wrb and sgl if needed for the command. And it will prep
3409  * the pdu's itt. beiscsi_parse_pdu will later translate
3410  * the pdu itt to the libiscsi task itt.
3411  */
3412 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
3413 {
3414         struct beiscsi_io_task *io_task = task->dd_data;
3415         struct iscsi_conn *conn = task->conn;
3416         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3417         struct beiscsi_hba *phba = beiscsi_conn->phba;
3418         struct hwi_wrb_context *pwrb_context;
3419         struct hwi_controller *phwi_ctrlr;
3420         itt_t itt;
3421         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3422         dma_addr_t paddr;
3423
3424         io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
3425                                           GFP_KERNEL, &paddr);
3426         if (!io_task->cmd_bhs)
3427                 return -ENOMEM;
3428         io_task->bhs_pa.u.a64.address = paddr;
3429         io_task->libiscsi_itt = (itt_t)task->itt;
3430         io_task->pwrb_handle = alloc_wrb_handle(phba,
3431                                                 beiscsi_conn->beiscsi_conn_cid -
3432                                                 phba->fw_config.iscsi_cid_start
3433                                                 );
3434         io_task->conn = beiscsi_conn;
3435
3436         task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
3437         task->hdr_max = sizeof(struct be_cmd_bhs);
3438
3439         if (task->sc) {
3440                 spin_lock(&phba->io_sgl_lock);
3441                 io_task->psgl_handle = alloc_io_sgl_handle(phba);
3442                 spin_unlock(&phba->io_sgl_lock);
3443                 if (!io_task->psgl_handle)
3444                         goto free_hndls;
3445         } else {
3446                 io_task->scsi_cmnd = NULL;
3447                 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
3448                         if (!beiscsi_conn->login_in_progress) {
3449                                 spin_lock(&phba->mgmt_sgl_lock);
3450                                 io_task->psgl_handle = (struct sgl_handle *)
3451                                                 alloc_mgmt_sgl_handle(phba);
3452                                 spin_unlock(&phba->mgmt_sgl_lock);
3453                                 if (!io_task->psgl_handle)
3454                                         goto free_hndls;
3455
3456                                 beiscsi_conn->login_in_progress = 1;
3457                                 beiscsi_conn->plogin_sgl_handle =
3458                                                         io_task->psgl_handle;
3459                         } else {
3460                                 io_task->psgl_handle =
3461                                                 beiscsi_conn->plogin_sgl_handle;
3462                         }
3463                 } else {
3464                         spin_lock(&phba->mgmt_sgl_lock);
3465                         io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
3466                         spin_unlock(&phba->mgmt_sgl_lock);
3467                         if (!io_task->psgl_handle)
3468                                 goto free_hndls;
3469                 }
3470         }
3471         itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
3472                                  wrb_index << 16) | (unsigned int)
3473                                 (io_task->psgl_handle->sgl_index));
3474         io_task->pwrb_handle->pio_handle = task;
3475
3476         io_task->cmd_bhs->iscsi_hdr.itt = itt;
3477         return 0;
3478
3479 free_hndls:
3480         phwi_ctrlr = phba->phwi_ctrlr;
3481         pwrb_context = &phwi_ctrlr->wrb_context[
3482                         beiscsi_conn->beiscsi_conn_cid -
3483                         phba->fw_config.iscsi_cid_start];
3484         free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3485         io_task->pwrb_handle = NULL;
3486         pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3487                       io_task->bhs_pa.u.a64.address);
3488         SE_DEBUG(DBG_LVL_1, "Alloc of SGL_ICD Failed \n");
3489         return -ENOMEM;
3490 }
3491
3492 static void beiscsi_cleanup_task(struct iscsi_task *task)
3493 {
3494         struct beiscsi_io_task *io_task = task->dd_data;
3495         struct iscsi_conn *conn = task->conn;
3496         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3497         struct beiscsi_hba *phba = beiscsi_conn->phba;
3498         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3499         struct hwi_wrb_context *pwrb_context;
3500         struct hwi_controller *phwi_ctrlr;
3501
3502         phwi_ctrlr = phba->phwi_ctrlr;
3503         pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid
3504                         - phba->fw_config.iscsi_cid_start];
3505         if (io_task->pwrb_handle) {
3506                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3507                 io_task->pwrb_handle = NULL;
3508         }
3509
3510         if (io_task->cmd_bhs) {
3511                 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3512                               io_task->bhs_pa.u.a64.address);
3513         }
3514
3515         if (task->sc) {
3516                 if (io_task->psgl_handle) {
3517                         spin_lock(&phba->io_sgl_lock);
3518                         free_io_sgl_handle(phba, io_task->psgl_handle);
3519                         spin_unlock(&phba->io_sgl_lock);
3520                         io_task->psgl_handle = NULL;
3521                 }
3522         } else {
3523                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN)
3524                         return;
3525                 if (io_task->psgl_handle) {
3526                         spin_lock(&phba->mgmt_sgl_lock);
3527                         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
3528                         spin_unlock(&phba->mgmt_sgl_lock);
3529                         io_task->psgl_handle = NULL;
3530                 }
3531         }
3532 }
3533
3534 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
3535                           unsigned int num_sg, unsigned int xferlen,
3536                           unsigned int writedir)
3537 {
3538
3539         struct beiscsi_io_task *io_task = task->dd_data;
3540         struct iscsi_conn *conn = task->conn;
3541         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3542         struct beiscsi_hba *phba = beiscsi_conn->phba;
3543         struct iscsi_wrb *pwrb = NULL;
3544         unsigned int doorbell = 0;
3545
3546         pwrb = io_task->pwrb_handle->pwrb;
3547         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
3548         io_task->bhs_len = sizeof(struct be_cmd_bhs);
3549
3550         if (writedir) {
3551                 memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48);
3552                 AMAP_SET_BITS(struct amap_pdu_data_out, itt,
3553                               &io_task->cmd_bhs->iscsi_data_pdu,
3554                               (unsigned int)io_task->cmd_bhs->iscsi_hdr.itt);
3555                 AMAP_SET_BITS(struct amap_pdu_data_out, opcode,
3556                               &io_task->cmd_bhs->iscsi_data_pdu,
3557                               ISCSI_OPCODE_SCSI_DATA_OUT);
3558                 AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,
3559                               &io_task->cmd_bhs->iscsi_data_pdu, 1);
3560                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3561                               INI_WR_CMD);
3562                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3563         } else {
3564                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3565                               INI_RD_CMD);
3566                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
3567         }
3568         memcpy(&io_task->cmd_bhs->iscsi_data_pdu.
3569                dw[offsetof(struct amap_pdu_data_out, lun) / 32],
3570                io_task->cmd_bhs->iscsi_hdr.lun, sizeof(struct scsi_lun));
3571
3572         AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
3573                       cpu_to_be16((unsigned short)io_task->cmd_bhs->iscsi_hdr.
3574                                   lun[0]));
3575         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
3576         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3577                       io_task->pwrb_handle->wrb_index);
3578         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3579                       be32_to_cpu(task->cmdsn));
3580         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3581                       io_task->psgl_handle->sgl_index);
3582
3583         hwi_write_sgl(pwrb, sg, num_sg, io_task);
3584
3585         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3586                       io_task->pwrb_handle->nxt_wrb_index);
3587         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3588
3589         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3590         doorbell |= (io_task->pwrb_handle->wrb_index &
3591                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3592         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3593
3594         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3595         return 0;
3596 }
3597
3598 static int beiscsi_mtask(struct iscsi_task *task)
3599 {
3600         struct beiscsi_io_task *io_task = task->dd_data;
3601         struct iscsi_conn *conn = task->conn;
3602         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3603         struct beiscsi_hba *phba = beiscsi_conn->phba;
3604         struct iscsi_wrb *pwrb = NULL;
3605         unsigned int doorbell = 0;
3606         unsigned int cid;
3607
3608         cid = beiscsi_conn->beiscsi_conn_cid;
3609         pwrb = io_task->pwrb_handle->pwrb;
3610         memset(pwrb, 0, sizeof(*pwrb));
3611         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3612                       be32_to_cpu(task->cmdsn));
3613         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3614                       io_task->pwrb_handle->wrb_index);
3615         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3616                       io_task->psgl_handle->sgl_index);
3617
3618         switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
3619         case ISCSI_OP_LOGIN:
3620                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3621                               TGT_DM_CMD);
3622                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3623                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
3624                 hwi_write_buffer(pwrb, task);
3625                 break;
3626         case ISCSI_OP_NOOP_OUT:
3627                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3628                               INI_RD_CMD);
3629                 if (task->hdr->ttt == ISCSI_RESERVED_TAG)
3630                         AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3631                 else
3632                         AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 1);
3633                 hwi_write_buffer(pwrb, task);
3634                 break;
3635         case ISCSI_OP_TEXT:
3636                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3637                               TGT_DM_CMD);
3638                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3639                 hwi_write_buffer(pwrb, task);
3640                 break;
3641         case ISCSI_OP_SCSI_TMFUNC:
3642                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3643                               INI_TMF_CMD);
3644                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3645                 hwi_write_buffer(pwrb, task);
3646                 break;
3647         case ISCSI_OP_LOGOUT:
3648                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3649                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3650                               HWH_TYPE_LOGOUT);
3651                 hwi_write_buffer(pwrb, task);
3652                 break;
3653
3654         default:
3655                 SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported \n",
3656                          task->hdr->opcode & ISCSI_OPCODE_MASK);
3657                 return -EINVAL;
3658         }
3659
3660         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
3661                       task->data_count);
3662         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3663                       io_task->pwrb_handle->nxt_wrb_index);
3664         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3665
3666         doorbell |= cid & DB_WRB_POST_CID_MASK;
3667         doorbell |= (io_task->pwrb_handle->wrb_index &
3668                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3669         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3670         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3671         return 0;
3672 }
3673
3674 static int beiscsi_task_xmit(struct iscsi_task *task)
3675 {
3676         struct beiscsi_io_task *io_task = task->dd_data;
3677         struct scsi_cmnd *sc = task->sc;
3678         struct scatterlist *sg;
3679         int num_sg;
3680         unsigned int  writedir = 0, xferlen = 0;
3681
3682         if (!sc)
3683                 return beiscsi_mtask(task);
3684
3685         io_task->scsi_cmnd = sc;
3686         num_sg = scsi_dma_map(sc);
3687         if (num_sg < 0) {
3688                 SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n")
3689                 return num_sg;
3690         }
3691         SE_DEBUG(DBG_LVL_4, "xferlen=0x%08x scmd=%p num_sg=%d sernum=%lu\n",
3692                   (scsi_bufflen(sc)), sc, num_sg, sc->serial_number);
3693         xferlen = scsi_bufflen(sc);
3694         sg = scsi_sglist(sc);
3695         if (sc->sc_data_direction == DMA_TO_DEVICE) {
3696                 writedir = 1;
3697                 SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x \n",
3698                          task->imm_count);
3699         } else
3700                 writedir = 0;
3701         return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
3702 }
3703
3704 static void beiscsi_remove(struct pci_dev *pcidev)
3705 {
3706         struct beiscsi_hba *phba = NULL;
3707         struct hwi_controller *phwi_ctrlr;
3708         struct hwi_context_memory *phwi_context;
3709         struct be_eq_obj *pbe_eq;
3710         unsigned int i, msix_vec;
3711
3712         phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
3713         if (!phba) {
3714                 dev_err(&pcidev->dev, "beiscsi_remove called with no phba \n");
3715                 return;
3716         }
3717
3718         phwi_ctrlr = phba->phwi_ctrlr;
3719         phwi_context = phwi_ctrlr->phwi_ctxt;
3720         hwi_disable_intr(phba);
3721         if (phba->msix_enabled) {
3722                 for (i = 0; i <= phba->num_cpus; i++) {
3723                         msix_vec = phba->msix_entries[i].vector;
3724                         free_irq(msix_vec, &phwi_context->be_eq[i]);
3725                 }
3726         } else
3727                 if (phba->pcidev->irq)
3728                         free_irq(phba->pcidev->irq, phba);
3729         pci_disable_msix(phba->pcidev);
3730         destroy_workqueue(phba->wq);
3731         if (blk_iopoll_enabled)
3732                 for (i = 0; i < phba->num_cpus; i++) {
3733                         pbe_eq = &phwi_context->be_eq[i];
3734                         blk_iopoll_disable(&pbe_eq->iopoll);
3735                 }
3736
3737         beiscsi_clean_port(phba);
3738         beiscsi_free_mem(phba);
3739         beiscsi_unmap_pci_function(phba);
3740         pci_free_consistent(phba->pcidev,
3741                             phba->ctrl.mbox_mem_alloced.size,
3742                             phba->ctrl.mbox_mem_alloced.va,
3743                             phba->ctrl.mbox_mem_alloced.dma);
3744         iscsi_host_remove(phba->shost);
3745         pci_dev_put(phba->pcidev);
3746         iscsi_host_free(phba->shost);
3747 }
3748
3749 static void beiscsi_msix_enable(struct beiscsi_hba *phba)
3750 {
3751         int i, status;
3752
3753         for (i = 0; i <= phba->num_cpus; i++)
3754                 phba->msix_entries[i].entry = i;
3755
3756         status = pci_enable_msix(phba->pcidev, phba->msix_entries,
3757                                  (phba->num_cpus + 1));
3758         if (!status)
3759                 phba->msix_enabled = true;
3760
3761         return;
3762 }
3763
3764 static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
3765                                 const struct pci_device_id *id)
3766 {
3767         struct beiscsi_hba *phba = NULL;
3768         struct hwi_controller *phwi_ctrlr;
3769         struct hwi_context_memory *phwi_context;
3770         struct be_eq_obj *pbe_eq;
3771         int ret, msix_vec, num_cpus, i;
3772
3773         ret = beiscsi_enable_pci(pcidev);
3774         if (ret < 0) {
3775                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3776                              "Failed to enable pci device \n");
3777                 return ret;
3778         }
3779
3780         phba = beiscsi_hba_alloc(pcidev);
3781         if (!phba) {
3782                 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
3783                         " Failed in beiscsi_hba_alloc \n");
3784                 goto disable_pci;
3785         }
3786
3787         switch (pcidev->device) {
3788         case BE_DEVICE_ID1:
3789         case OC_DEVICE_ID1:
3790         case OC_DEVICE_ID2:
3791                 phba->generation = BE_GEN2;
3792                 break;
3793         case BE_DEVICE_ID2:
3794         case OC_DEVICE_ID3:
3795                 phba->generation = BE_GEN3;
3796                 break;
3797         default:
3798                 phba->generation = 0;
3799         }
3800
3801         if (enable_msix)
3802                 num_cpus = find_num_cpus();
3803         else
3804                 num_cpus = 1;
3805         phba->num_cpus = num_cpus;
3806         SE_DEBUG(DBG_LVL_8, "num_cpus = %d \n", phba->num_cpus);
3807
3808         if (enable_msix)
3809                 beiscsi_msix_enable(phba);
3810         ret = be_ctrl_init(phba, pcidev);
3811         if (ret) {
3812                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3813                                 "Failed in be_ctrl_init\n");
3814                 goto hba_free;
3815         }
3816
3817         spin_lock_init(&phba->io_sgl_lock);
3818         spin_lock_init(&phba->mgmt_sgl_lock);
3819         spin_lock_init(&phba->isr_lock);
3820         ret = mgmt_get_fw_config(&phba->ctrl, phba);
3821         if (ret != 0) {
3822                 shost_printk(KERN_ERR, phba->shost,
3823                              "Error getting fw config\n");
3824                 goto free_port;
3825         }
3826         phba->shost->max_id = phba->fw_config.iscsi_cid_count;
3827         beiscsi_get_params(phba);
3828         phba->shost->can_queue = phba->params.ios_per_ctrl;
3829         ret = beiscsi_init_port(phba);
3830         if (ret < 0) {
3831                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3832                              "Failed in beiscsi_init_port\n");
3833                 goto free_port;
3834         }
3835
3836         for (i = 0; i < MAX_MCC_CMD ; i++) {
3837                 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
3838                 phba->ctrl.mcc_tag[i] = i + 1;
3839                 phba->ctrl.mcc_numtag[i + 1] = 0;
3840                 phba->ctrl.mcc_tag_available++;
3841         }
3842
3843         phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
3844
3845         snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u",
3846                  phba->shost->host_no);
3847         phba->wq = create_workqueue(phba->wq_name);
3848         if (!phba->wq) {
3849                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3850                                 "Failed to allocate work queue\n");
3851                 goto free_twq;
3852         }
3853
3854         INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs);
3855
3856         phwi_ctrlr = phba->phwi_ctrlr;
3857         phwi_context = phwi_ctrlr->phwi_ctxt;
3858         if (blk_iopoll_enabled) {
3859                 for (i = 0; i < phba->num_cpus; i++) {
3860                         pbe_eq = &phwi_context->be_eq[i];
3861                         blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
3862                                         be_iopoll);
3863                         blk_iopoll_enable(&pbe_eq->iopoll);
3864                 }
3865         }
3866         ret = beiscsi_init_irqs(phba);
3867         if (ret < 0) {
3868                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3869                              "Failed to beiscsi_init_irqs\n");
3870                 goto free_blkenbld;
3871         }
3872         ret = hwi_enable_intr(phba);
3873         if (ret < 0) {
3874                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3875                              "Failed to hwi_enable_intr\n");
3876                 goto free_ctrlr;
3877         }
3878         SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED \n\n\n");
3879         return 0;
3880
3881 free_ctrlr:
3882         if (phba->msix_enabled) {
3883                 for (i = 0; i <= phba->num_cpus; i++) {
3884                         msix_vec = phba->msix_entries[i].vector;
3885                         free_irq(msix_vec, &phwi_context->be_eq[i]);
3886                 }
3887         } else
3888                 if (phba->pcidev->irq)
3889                         free_irq(phba->pcidev->irq, phba);
3890         pci_disable_msix(phba->pcidev);
3891 free_blkenbld:
3892         destroy_workqueue(phba->wq);
3893         if (blk_iopoll_enabled)
3894                 for (i = 0; i < phba->num_cpus; i++) {
3895                         pbe_eq = &phwi_context->be_eq[i];
3896                         blk_iopoll_disable(&pbe_eq->iopoll);
3897                 }
3898 free_twq:
3899         beiscsi_clean_port(phba);
3900         beiscsi_free_mem(phba);
3901 free_port:
3902         pci_free_consistent(phba->pcidev,
3903                             phba->ctrl.mbox_mem_alloced.size,
3904                             phba->ctrl.mbox_mem_alloced.va,
3905                            phba->ctrl.mbox_mem_alloced.dma);
3906         beiscsi_unmap_pci_function(phba);
3907 hba_free:
3908         iscsi_host_remove(phba->shost);
3909         pci_dev_put(phba->pcidev);
3910         iscsi_host_free(phba->shost);
3911 disable_pci:
3912         pci_disable_device(pcidev);
3913         return ret;
3914 }
3915
3916 struct iscsi_transport beiscsi_iscsi_transport = {
3917         .owner = THIS_MODULE,
3918         .name = DRV_NAME,
3919         .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
3920                 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
3921         .param_mask = ISCSI_MAX_RECV_DLENGTH |
3922                 ISCSI_MAX_XMIT_DLENGTH |
3923                 ISCSI_HDRDGST_EN |
3924                 ISCSI_DATADGST_EN |
3925                 ISCSI_INITIAL_R2T_EN |
3926                 ISCSI_MAX_R2T |
3927                 ISCSI_IMM_DATA_EN |
3928                 ISCSI_FIRST_BURST |
3929                 ISCSI_MAX_BURST |
3930                 ISCSI_PDU_INORDER_EN |
3931                 ISCSI_DATASEQ_INORDER_EN |
3932                 ISCSI_ERL |
3933                 ISCSI_CONN_PORT |
3934                 ISCSI_CONN_ADDRESS |
3935                 ISCSI_EXP_STATSN |
3936                 ISCSI_PERSISTENT_PORT |
3937                 ISCSI_PERSISTENT_ADDRESS |
3938                 ISCSI_TARGET_NAME | ISCSI_TPGT |
3939                 ISCSI_USERNAME | ISCSI_PASSWORD |
3940                 ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
3941                 ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
3942                 ISCSI_LU_RESET_TMO |
3943                 ISCSI_PING_TMO | ISCSI_RECV_TMO |
3944                 ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME,
3945         .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
3946                                 ISCSI_HOST_INITIATOR_NAME,
3947         .create_session = beiscsi_session_create,
3948         .destroy_session = beiscsi_session_destroy,
3949         .create_conn = beiscsi_conn_create,
3950         .bind_conn = beiscsi_conn_bind,
3951         .destroy_conn = iscsi_conn_teardown,
3952         .set_param = beiscsi_set_param,
3953         .get_conn_param = beiscsi_conn_get_param,
3954         .get_session_param = iscsi_session_get_param,
3955         .get_host_param = beiscsi_get_host_param,
3956         .start_conn = beiscsi_conn_start,
3957         .stop_conn = beiscsi_conn_stop,
3958         .send_pdu = iscsi_conn_send_pdu,
3959         .xmit_task = beiscsi_task_xmit,
3960         .cleanup_task = beiscsi_cleanup_task,
3961         .alloc_pdu = beiscsi_alloc_pdu,
3962         .parse_pdu_itt = beiscsi_parse_pdu,
3963         .get_stats = beiscsi_conn_get_stats,
3964         .ep_connect = beiscsi_ep_connect,
3965         .ep_poll = beiscsi_ep_poll,
3966         .ep_disconnect = beiscsi_ep_disconnect,
3967         .session_recovery_timedout = iscsi_session_recovery_timedout,
3968 };
3969
3970 static struct pci_driver beiscsi_pci_driver = {
3971         .name = DRV_NAME,
3972         .probe = beiscsi_dev_probe,
3973         .remove = beiscsi_remove,
3974         .id_table = beiscsi_pci_id_table
3975 };
3976
3977
3978 static int __init beiscsi_module_init(void)
3979 {
3980         int ret;
3981
3982         beiscsi_scsi_transport =
3983                         iscsi_register_transport(&beiscsi_iscsi_transport);
3984         if (!beiscsi_scsi_transport) {
3985                 SE_DEBUG(DBG_LVL_1,
3986                          "beiscsi_module_init - Unable to  register beiscsi"
3987                          "transport.\n");
3988                 return -ENOMEM;
3989         }
3990         SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p \n",
3991                  &beiscsi_iscsi_transport);
3992
3993         ret = pci_register_driver(&beiscsi_pci_driver);
3994         if (ret) {
3995                 SE_DEBUG(DBG_LVL_1,
3996                          "beiscsi_module_init - Unable to  register"
3997                          "beiscsi pci driver.\n");
3998                 goto unregister_iscsi_transport;
3999         }
4000         return 0;
4001
4002 unregister_iscsi_transport:
4003         iscsi_unregister_transport(&beiscsi_iscsi_transport);
4004         return ret;
4005 }
4006
4007 static void __exit beiscsi_module_exit(void)
4008 {
4009         pci_unregister_driver(&beiscsi_pci_driver);
4010         iscsi_unregister_transport(&beiscsi_iscsi_transport);
4011 }
4012
4013 module_init(beiscsi_module_init);
4014 module_exit(beiscsi_module_exit);