drivers/scsi/qedi/qedi_fw.c

   1 /*
   2  * QLogic iSCSI Offload Driver
   3  * Copyright (c) 2016 Cavium Inc.
   4  *
   5  * This software is available under the terms of the GNU General Public License
   6  * (GPL) Version 2, available from the file COPYING in the main directory of
   7  * this source tree.
   8  */
   9
  10 #include <linux/blkdev.h>
  11 #include <scsi/scsi_tcq.h>
  12 #include <linux/delay.h>
  13
  14 #include "qedi.h"
  15 #include "qedi_iscsi.h"
  16 #include "qedi_gbl.h"
  17
  18 static int qedi_send_iscsi_tmf(struct qedi_conn *qedi_conn,
  19                                struct iscsi_task *mtask);
  20
  21 void qedi_iscsi_unmap_sg_list(struct qedi_cmd *cmd)
  22 {
  23         struct scsi_cmnd *sc = cmd->scsi_cmd;
  24
  25         if (cmd->io_tbl.sge_valid && sc) {
  26                 cmd->io_tbl.sge_valid = 0;
  27                 scsi_dma_unmap(sc);
  28         }
  29 }
  30
  31 static void qedi_process_logout_resp(struct qedi_ctx *qedi,
  32                                      union iscsi_cqe *cqe,
  33                                      struct iscsi_task *task,
  34                                      struct qedi_conn *qedi_conn)
  35 {
  36         struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
  37         struct iscsi_logout_rsp *resp_hdr;
  38         struct iscsi_session *session = conn->session;
  39         struct iscsi_logout_response_hdr *cqe_logout_response;
  40         struct qedi_cmd *cmd;
  41
  42         cmd = (struct qedi_cmd *)task->dd_data;
  43         cqe_logout_response = &cqe->cqe_common.iscsi_hdr.logout_response;
  44         spin_lock(&session->back_lock);
  45         resp_hdr = (struct iscsi_logout_rsp *)&qedi_conn->gen_pdu.resp_hdr;
  46         memset(resp_hdr, 0, sizeof(struct iscsi_hdr));
  47         resp_hdr->opcode = cqe_logout_response->opcode;
  48         resp_hdr->flags = cqe_logout_response->flags;
  49         resp_hdr->hlength = 0;
  50
  51         resp_hdr->itt = build_itt(cqe->cqe_solicited.itid, conn->session->age);
  52         resp_hdr->statsn = cpu_to_be32(cqe_logout_response->stat_sn);
  53         resp_hdr->exp_cmdsn = cpu_to_be32(cqe_logout_response->exp_cmd_sn);
  54         resp_hdr->max_cmdsn = cpu_to_be32(cqe_logout_response->max_cmd_sn);
  55
  56         resp_hdr->t2wait = cpu_to_be32(cqe_logout_response->time2wait);
  57         resp_hdr->t2retain = cpu_to_be32(cqe_logout_response->time2retain);
  58
  59         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,
  60                   "Freeing tid=0x%x for cid=0x%x\n",
  61                   cmd->task_id, qedi_conn->iscsi_conn_id);
  62
  63         if (likely(cmd->io_cmd_in_list)) {
  64                 cmd->io_cmd_in_list = false;
  65                 list_del_init(&cmd->io_cmd);
  66                 qedi_conn->active_cmd_count--;
  67         } else {
  68                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
  69                           "Active cmd list node already deleted, tid=0x%x, cid=0x%x, io_cmd_node=%p\n",
  70                           cmd->task_id, qedi_conn->iscsi_conn_id,
  71                           &cmd->io_cmd);
  72         }
  73
  74         cmd->state = RESPONSE_RECEIVED;
  75         qedi_clear_task_idx(qedi, cmd->task_id);
  76         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr, NULL, 0);
  77
  78         spin_unlock(&session->back_lock);
  79 }
  80
  81 static void qedi_process_text_resp(struct qedi_ctx *qedi,
  82                                    union iscsi_cqe *cqe,
  83                                    struct iscsi_task *task,
  84                                    struct qedi_conn *qedi_conn)
  85 {
  86         struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
  87         struct iscsi_session *session = conn->session;
  88         struct iscsi_task_context *task_ctx;
  89         struct iscsi_text_rsp *resp_hdr_ptr;
  90         struct iscsi_text_response_hdr *cqe_text_response;
  91         struct qedi_cmd *cmd;
  92         int pld_len;
  93         u32 *tmp;
  94
  95         cmd = (struct qedi_cmd *)task->dd_data;
  96         task_ctx = qedi_get_task_mem(&qedi->tasks, cmd->task_id);
  97
  98         cqe_text_response = &cqe->cqe_common.iscsi_hdr.text_response;
  99         spin_lock(&session->back_lock);
 100         resp_hdr_ptr =  (struct iscsi_text_rsp *)&qedi_conn->gen_pdu.resp_hdr;
 101         memset(resp_hdr_ptr, 0, sizeof(struct iscsi_hdr));
 102         resp_hdr_ptr->opcode = cqe_text_response->opcode;
 103         resp_hdr_ptr->flags = cqe_text_response->flags;
 104         resp_hdr_ptr->hlength = 0;
 105
 106         hton24(resp_hdr_ptr->dlength,
 107                (cqe_text_response->hdr_second_dword &
 108                 ISCSI_TEXT_RESPONSE_HDR_DATA_SEG_LEN_MASK));
 109         tmp = (u32 *)resp_hdr_ptr->dlength;
 110
 111         resp_hdr_ptr->itt = build_itt(cqe->cqe_solicited.itid,
 112                                       conn->session->age);
 113         resp_hdr_ptr->ttt = cqe_text_response->ttt;
 114         resp_hdr_ptr->statsn = cpu_to_be32(cqe_text_response->stat_sn);
 115         resp_hdr_ptr->exp_cmdsn = cpu_to_be32(cqe_text_response->exp_cmd_sn);
 116         resp_hdr_ptr->max_cmdsn = cpu_to_be32(cqe_text_response->max_cmd_sn);
 117
 118         pld_len = cqe_text_response->hdr_second_dword &
 119                   ISCSI_TEXT_RESPONSE_HDR_DATA_SEG_LEN_MASK;
 120         qedi_conn->gen_pdu.resp_wr_ptr = qedi_conn->gen_pdu.resp_buf + pld_len;
 121
 122         memset(task_ctx, '\0', sizeof(*task_ctx));
 123
 124         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,
 125                   "Freeing tid=0x%x for cid=0x%x\n",
 126                   cmd->task_id, qedi_conn->iscsi_conn_id);
 127
 128         if (likely(cmd->io_cmd_in_list)) {
 129                 cmd->io_cmd_in_list = false;
 130                 list_del_init(&cmd->io_cmd);
 131                 qedi_conn->active_cmd_count--;
 132         } else {
 133                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
 134                           "Active cmd list node already deleted, tid=0x%x, cid=0x%x, io_cmd_node=%p\n",
 135                           cmd->task_id, qedi_conn->iscsi_conn_id,
 136                           &cmd->io_cmd);
 137         }
 138
 139         cmd->state = RESPONSE_RECEIVED;
 140         qedi_clear_task_idx(qedi, cmd->task_id);
 141
 142         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr_ptr,
 143                              qedi_conn->gen_pdu.resp_buf,
 144                              (qedi_conn->gen_pdu.resp_wr_ptr -
 145                               qedi_conn->gen_pdu.resp_buf));
 146         spin_unlock(&session->back_lock);
 147 }
 148
 149 static void qedi_tmf_resp_work(struct work_struct *work)
 150 {
 151         struct qedi_cmd *qedi_cmd =
 152                                 container_of(work, struct qedi_cmd, tmf_work);
 153         struct qedi_conn *qedi_conn = qedi_cmd->conn;
 154         struct qedi_ctx *qedi = qedi_conn->qedi;
 155         struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
 156         struct iscsi_session *session = conn->session;
 157         struct iscsi_tm_rsp *resp_hdr_ptr;
 158         struct iscsi_cls_session *cls_sess;
 159         int rval = 0;
 160
 161         set_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags);
 162         resp_hdr_ptr =  (struct iscsi_tm_rsp *)qedi_cmd->tmf_resp_buf;
 163         cls_sess = iscsi_conn_to_session(qedi_conn->cls_conn);
 164
 165         iscsi_block_session(session->cls_session);
 166         rval = qedi_cleanup_all_io(qedi, qedi_conn, qedi_cmd->task, true);
 167         if (rval) {
 168                 qedi_clear_task_idx(qedi, qedi_cmd->task_id);
 169                 iscsi_unblock_session(session->cls_session);
 170                 goto exit_tmf_resp;
 171         }
 172
 173         iscsi_unblock_session(session->cls_session);
 174         qedi_clear_task_idx(qedi, qedi_cmd->task_id);
 175
 176         spin_lock(&session->back_lock);
 177         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr_ptr, NULL, 0);
 178         spin_unlock(&session->back_lock);
 179
 180 exit_tmf_resp:
 181         kfree(resp_hdr_ptr);
 182         clear_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags);
 183 }
 184
 185 static void qedi_process_tmf_resp(struct qedi_ctx *qedi,
 186                                   union iscsi_cqe *cqe,
 187                                   struct iscsi_task *task,
 188                                   struct qedi_conn *qedi_conn)
 189
 190 {
 191         struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
 192         struct iscsi_session *session = conn->session;
 193         struct iscsi_tmf_response_hdr *cqe_tmp_response;
 194         struct iscsi_tm_rsp *resp_hdr_ptr;
 195         struct iscsi_tm *tmf_hdr;
 196         struct qedi_cmd *qedi_cmd = NULL;
 197         u32 *tmp;
 198
 199         cqe_tmp_response = &cqe->cqe_common.iscsi_hdr.tmf_response;
 200
 201         qedi_cmd = task->dd_data;
 202         qedi_cmd->tmf_resp_buf = kzalloc(sizeof(*resp_hdr_ptr), GFP_KERNEL);
 203         if (!qedi_cmd->tmf_resp_buf) {
 204                 QEDI_ERR(&qedi->dbg_ctx,
 205                          "Failed to allocate resp buf, cid=0x%x\n",
 206                           qedi_conn->iscsi_conn_id);
 207                 return;
 208         }
 209
 210         spin_lock(&session->back_lock);
 211         resp_hdr_ptr =  (struct iscsi_tm_rsp *)qedi_cmd->tmf_resp_buf;
 212         memset(resp_hdr_ptr, 0, sizeof(struct iscsi_tm_rsp));
 213
 214         /* Fill up the header */
 215         resp_hdr_ptr->opcode = cqe_tmp_response->opcode;
 216         resp_hdr_ptr->flags = cqe_tmp_response->hdr_flags;
 217         resp_hdr_ptr->response = cqe_tmp_response->hdr_response;
 218         resp_hdr_ptr->hlength = 0;
 219
 220         hton24(resp_hdr_ptr->dlength,
 221                (cqe_tmp_response->hdr_second_dword &
 222                 ISCSI_TMF_RESPONSE_HDR_DATA_SEG_LEN_MASK));
 223         tmp = (u32 *)resp_hdr_ptr->dlength;
 224         resp_hdr_ptr->itt = build_itt(cqe->cqe_solicited.itid,
 225                                       conn->session->age);
 226         resp_hdr_ptr->statsn = cpu_to_be32(cqe_tmp_response->stat_sn);
 227         resp_hdr_ptr->exp_cmdsn  = cpu_to_be32(cqe_tmp_response->exp_cmd_sn);
 228         resp_hdr_ptr->max_cmdsn = cpu_to_be32(cqe_tmp_response->max_cmd_sn);
 229
 230         tmf_hdr = (struct iscsi_tm *)qedi_cmd->task->hdr;
 231
 232         if (likely(qedi_cmd->io_cmd_in_list)) {
 233                 qedi_cmd->io_cmd_in_list = false;
 234                 list_del_init(&qedi_cmd->io_cmd);
 235                 qedi_conn->active_cmd_count--;
 236         }
 237
 238         if (((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
 239               ISCSI_TM_FUNC_LOGICAL_UNIT_RESET) ||
 240             ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
 241               ISCSI_TM_FUNC_TARGET_WARM_RESET) ||
 242             ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
 243               ISCSI_TM_FUNC_TARGET_COLD_RESET)) {
 244                 INIT_WORK(&qedi_cmd->tmf_work, qedi_tmf_resp_work);
 245                 queue_work(qedi->tmf_thread, &qedi_cmd->tmf_work);
 246                 goto unblock_sess;
 247         }
 248
 249         qedi_clear_task_idx(qedi, qedi_cmd->task_id);
 250
 251         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr_ptr, NULL, 0);
 252         kfree(resp_hdr_ptr);
 253
 254 unblock_sess:
 255         spin_unlock(&session->back_lock);
 256 }
 257
 258 static void qedi_process_login_resp(struct qedi_ctx *qedi,
 259                                     union iscsi_cqe *cqe,
 260                                     struct iscsi_task *task,
 261                                     struct qedi_conn *qedi_conn)
 262 {
 263         struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
 264         struct iscsi_session *session = conn->session;
 265         struct iscsi_task_context *task_ctx;
 266         struct iscsi_login_rsp *resp_hdr_ptr;
 267         struct iscsi_login_response_hdr *cqe_login_response;
 268         struct qedi_cmd *cmd;
 269         int pld_len;
 270         u32 *tmp;
 271
 272         cmd = (struct qedi_cmd *)task->dd_data;
 273
 274         cqe_login_response = &cqe->cqe_common.iscsi_hdr.login_response;
 275         task_ctx = qedi_get_task_mem(&qedi->tasks, cmd->task_id);
 276
 277         spin_lock(&session->back_lock);
 278         resp_hdr_ptr =  (struct iscsi_login_rsp *)&qedi_conn->gen_pdu.resp_hdr;
 279         memset(resp_hdr_ptr, 0, sizeof(struct iscsi_login_rsp));
 280         resp_hdr_ptr->opcode = cqe_login_response->opcode;
 281         resp_hdr_ptr->flags = cqe_login_response->flags_attr;
 282         resp_hdr_ptr->hlength = 0;
 283
 284         hton24(resp_hdr_ptr->dlength,
 285                (cqe_login_response->hdr_second_dword &
 286                 ISCSI_LOGIN_RESPONSE_HDR_DATA_SEG_LEN_MASK));
 287         tmp = (u32 *)resp_hdr_ptr->dlength;
 288         resp_hdr_ptr->itt = build_itt(cqe->cqe_solicited.itid,
 289                                       conn->session->age);
 290         resp_hdr_ptr->tsih = cqe_login_response->tsih;
 291         resp_hdr_ptr->statsn = cpu_to_be32(cqe_login_response->stat_sn);
 292         resp_hdr_ptr->exp_cmdsn = cpu_to_be32(cqe_login_response->exp_cmd_sn);
 293         resp_hdr_ptr->max_cmdsn = cpu_to_be32(cqe_login_response->max_cmd_sn);
 294         resp_hdr_ptr->status_class = cqe_login_response->status_class;
 295         resp_hdr_ptr->status_detail = cqe_login_response->status_detail;
 296         pld_len = cqe_login_response->hdr_second_dword &
 297                   ISCSI_LOGIN_RESPONSE_HDR_DATA_SEG_LEN_MASK;
 298         qedi_conn->gen_pdu.resp_wr_ptr = qedi_conn->gen_pdu.resp_buf + pld_len;
 299
 300         if (likely(cmd->io_cmd_in_list)) {
 301                 cmd->io_cmd_in_list = false;
 302                 list_del_init(&cmd->io_cmd);
 303                 qedi_conn->active_cmd_count--;
 304         }
 305
 306         memset(task_ctx, '\0', sizeof(*task_ctx));
 307
 308         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr_ptr,
 309                              qedi_conn->gen_pdu.resp_buf,
 310                              (qedi_conn->gen_pdu.resp_wr_ptr -
 311                              qedi_conn->gen_pdu.resp_buf));
 312
 313         spin_unlock(&session->back_lock);
 314         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,
 315                   "Freeing tid=0x%x for cid=0x%x\n",
 316                   cmd->task_id, qedi_conn->iscsi_conn_id);
 317         cmd->state = RESPONSE_RECEIVED;
 318         qedi_clear_task_idx(qedi, cmd->task_id);
 319 }
 320
 321 static void qedi_get_rq_bdq_buf(struct qedi_ctx *qedi,
 322                                 struct iscsi_cqe_unsolicited *cqe,
 323                                 char *ptr, int len)
 324 {
 325         u16 idx = 0;
 326
 327         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
 328                   "pld_len [%d], bdq_prod_idx [%d], idx [%d]\n",
 329                   len, qedi->bdq_prod_idx,
 330                   (qedi->bdq_prod_idx % qedi->rq_num_entries));
 331
 332         /* Obtain buffer address from rqe_opaque */
 333         idx = cqe->rqe_opaque.lo;
 334         if ((idx < 0) || (idx > (QEDI_BDQ_NUM - 1))) {
 335                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
 336                           "wrong idx %d returned by FW, dropping the unsolicited pkt\n",
 337                           idx);
 338                 return;
 339         }
 340
 341         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
 342                   "rqe_opaque.lo [0x%p], rqe_opaque.hi [0x%p], idx [%d]\n",
 343                   cqe->rqe_opaque.lo, cqe->rqe_opaque.hi, idx);
 344
 345         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
 346                   "unsol_cqe_type = %d\n", cqe->unsol_cqe_type);
 347         switch (cqe->unsol_cqe_type) {
 348         case ISCSI_CQE_UNSOLICITED_SINGLE:
 349         case ISCSI_CQE_UNSOLICITED_FIRST:
 350                 if (len)
 351                         memcpy(ptr, (void *)qedi->bdq[idx].buf_addr, len);
 352                 break;
 353         case ISCSI_CQE_UNSOLICITED_MIDDLE:
 354         case ISCSI_CQE_UNSOLICITED_LAST:
 355                 break;
 356         default:
 357                 break;
 358         }
 359 }
 360
 361 static void qedi_put_rq_bdq_buf(struct qedi_ctx *qedi,
 362                                 struct iscsi_cqe_unsolicited *cqe,
 363                                 int count)
 364 {
 365         u16 tmp;
 366         u16 idx = 0;
 367         struct scsi_bd *pbl;
 368
 369         /* Obtain buffer address from rqe_opaque */
 370         idx = cqe->rqe_opaque.lo;
 371         if ((idx < 0) || (idx > (QEDI_BDQ_NUM - 1))) {
 372                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
 373                           "wrong idx %d returned by FW, dropping the unsolicited pkt\n",
 374                           idx);
 375                 return;
 376         }
 377
 378         pbl = (struct scsi_bd *)qedi->bdq_pbl;
 379         pbl += (qedi->bdq_prod_idx % qedi->rq_num_entries);
 380         pbl->address.hi = cpu_to_le32(QEDI_U64_HI(qedi->bdq[idx].buf_dma));
 381         pbl->address.lo = cpu_to_le32(QEDI_U64_LO(qedi->bdq[idx].buf_dma));
 382         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
 383                   "pbl [0x%p] pbl->address hi [0x%llx] lo [0x%llx] idx [%d]\n",
 384                   pbl, pbl->address.hi, pbl->address.lo, idx);
 385         pbl->opaque.hi = 0;
 386         pbl->opaque.lo = cpu_to_le32(QEDI_U64_LO(idx));
 387
 388         /* Increment producer to let f/w know we've handled the frame */
 389         qedi->bdq_prod_idx += count;
 390
 391         writew(qedi->bdq_prod_idx, qedi->bdq_primary_prod);
 392         tmp = readw(qedi->bdq_primary_prod);
 393
 394         writew(qedi->bdq_prod_idx, qedi->bdq_secondary_prod);
 395         tmp = readw(qedi->bdq_secondary_prod);
 396 }
 397
 398 static void qedi_unsol_pdu_adjust_bdq(struct qedi_ctx *qedi,
 399                                       struct iscsi_cqe_unsolicited *cqe,
 400                                       u32 pdu_len, u32 num_bdqs,
 401                                       char *bdq_data)
 402 {
 403         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
 404                   "num_bdqs [%d]\n", num_bdqs);
 405
 406         qedi_get_rq_bdq_buf(qedi, cqe, bdq_data, pdu_len);
 407         qedi_put_rq_bdq_buf(qedi, cqe, (num_bdqs + 1));
 408 }
 409
 410 static int qedi_process_nopin_mesg(struct qedi_ctx *qedi,
 411                                    union iscsi_cqe *cqe,
 412                                    struct iscsi_task *task,
 413                                    struct qedi_conn *qedi_conn, u16 que_idx)
 414 {
 415         struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
 416         struct iscsi_session *session = conn->session;
 417         struct iscsi_nop_in_hdr *cqe_nop_in;
 418         struct iscsi_nopin *hdr;
 419         struct qedi_cmd *cmd;
 420         int tgt_async_nop = 0;
 421         u32 lun[2];
 422         u32 pdu_len, num_bdqs;
 423         char bdq_data[QEDI_BDQ_BUF_SIZE];
 424         unsigned long flags;
 425
 426         spin_lock_bh(&session->back_lock);
 427         cqe_nop_in = &cqe->cqe_common.iscsi_hdr.nop_in;
 428
 429         pdu_len = cqe_nop_in->hdr_second_dword &
 430                   ISCSI_NOP_IN_HDR_DATA_SEG_LEN_MASK;
 431         num_bdqs = pdu_len / QEDI_BDQ_BUF_SIZE;
 432
 433         hdr = (struct iscsi_nopin *)&qedi_conn->gen_pdu.resp_hdr;
 434         memset(hdr, 0, sizeof(struct iscsi_hdr));
 435         hdr->opcode = cqe_nop_in->opcode;
 436         hdr->max_cmdsn = cpu_to_be32(cqe_nop_in->max_cmd_sn);
 437         hdr->exp_cmdsn = cpu_to_be32(cqe_nop_in->exp_cmd_sn);
 438         hdr->statsn = cpu_to_be32(cqe_nop_in->stat_sn);
 439         hdr->ttt = cpu_to_be32(cqe_nop_in->ttt);
 440
 441         if (cqe->cqe_common.cqe_type == ISCSI_CQE_TYPE_UNSOLICITED) {
 442                 spin_lock_irqsave(&qedi->hba_lock, flags);
 443                 qedi_unsol_pdu_adjust_bdq(qedi, &cqe->cqe_unsolicited,
 444                                           pdu_len, num_bdqs, bdq_data);
 445                 hdr->itt = RESERVED_ITT;
 446                 tgt_async_nop = 1;
 447                 spin_unlock_irqrestore(&qedi->hba_lock, flags);
 448                 goto done;
 449         }
 450
 451         /* Response to one of our nop-outs */
 452         if (task) {
 453                 cmd = task->dd_data;
 454                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
 455                 hdr->itt = build_itt(cqe->cqe_solicited.itid,
 456                                      conn->session->age);
 457                 lun[0] = 0xffffffff;
 458                 lun[1] = 0xffffffff;
 459                 memcpy(&hdr->lun, lun, sizeof(struct scsi_lun));
 460                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,
 461                           "Freeing tid=0x%x for cid=0x%x\n",
 462                           cmd->task_id, qedi_conn->iscsi_conn_id);
 463                 cmd->state = RESPONSE_RECEIVED;
 464                 spin_lock(&qedi_conn->list_lock);
 465                 if (likely(cmd->io_cmd_in_list)) {
 466                         cmd->io_cmd_in_list = false;
 467                         list_del_init(&cmd->io_cmd);
 468                         qedi_conn->active_cmd_count--;
 469                 }
 470
 471                 spin_unlock(&qedi_conn->list_lock);
 472                 qedi_clear_task_idx(qedi, cmd->task_id);
 473         }
 474
 475 done:
 476         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, bdq_data, pdu_len);
 477
 478         spin_unlock_bh(&session->back_lock);
 479         return tgt_async_nop;
 480 }
 481
 482 static void qedi_process_async_mesg(struct qedi_ctx *qedi,
 483                                     union iscsi_cqe *cqe,
 484                                     struct iscsi_task *task,
 485                                     struct qedi_conn *qedi_conn,
 486                                     u16 que_idx)
 487 {
 488         struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
 489         struct iscsi_session *session = conn->session;
 490         struct iscsi_async_msg_hdr *cqe_async_msg;
 491         struct iscsi_async *resp_hdr;
 492         u32 lun[2];
 493         u32 pdu_len, num_bdqs;
 494         char bdq_data[QEDI_BDQ_BUF_SIZE];
 495         unsigned long flags;
 496
 497         spin_lock_bh(&session->back_lock);
 498
 499         cqe_async_msg = &cqe->cqe_common.iscsi_hdr.async_msg;
 500         pdu_len = cqe_async_msg->hdr_second_dword &
 501                 ISCSI_ASYNC_MSG_HDR_DATA_SEG_LEN_MASK;
 502         num_bdqs = pdu_len / QEDI_BDQ_BUF_SIZE;
 503
 504         if (cqe->cqe_common.cqe_type == ISCSI_CQE_TYPE_UNSOLICITED) {
 505                 spin_lock_irqsave(&qedi->hba_lock, flags);
 506                 qedi_unsol_pdu_adjust_bdq(qedi, &cqe->cqe_unsolicited,
 507                                           pdu_len, num_bdqs, bdq_data);
 508                 spin_unlock_irqrestore(&qedi->hba_lock, flags);
 509         }
 510
 511         resp_hdr = (struct iscsi_async *)&qedi_conn->gen_pdu.resp_hdr;
 512         memset(resp_hdr, 0, sizeof(struct iscsi_hdr));
 513         resp_hdr->opcode = cqe_async_msg->opcode;
 514         resp_hdr->flags = 0x80;
 515
 516         lun[0] = cpu_to_be32(cqe_async_msg->lun.lo);
 517         lun[1] = cpu_to_be32(cqe_async_msg->lun.hi);
 518         memcpy(&resp_hdr->lun, lun, sizeof(struct scsi_lun));
 519         resp_hdr->exp_cmdsn = cpu_to_be32(cqe_async_msg->exp_cmd_sn);
 520         resp_hdr->max_cmdsn = cpu_to_be32(cqe_async_msg->max_cmd_sn);
 521         resp_hdr->statsn = cpu_to_be32(cqe_async_msg->stat_sn);
 522
 523         resp_hdr->async_event = cqe_async_msg->async_event;
 524         resp_hdr->async_vcode = cqe_async_msg->async_vcode;
 525
 526         resp_hdr->param1 = cpu_to_be16(cqe_async_msg->param1_rsrv);
 527         resp_hdr->param2 = cpu_to_be16(cqe_async_msg->param2_rsrv);
 528         resp_hdr->param3 = cpu_to_be16(cqe_async_msg->param3_rsrv);
 529
 530         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr, bdq_data,
 531                              pdu_len);
 532
 533         spin_unlock_bh(&session->back_lock);
 534 }
 535
 536 static void qedi_process_reject_mesg(struct qedi_ctx *qedi,
 537                                      union iscsi_cqe *cqe,
 538                                      struct iscsi_task *task,
 539                                      struct qedi_conn *qedi_conn,
 540                                      uint16_t que_idx)
 541 {
 542         struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
 543         struct iscsi_session *session = conn->session;
 544         struct iscsi_reject_hdr *cqe_reject;
 545         struct iscsi_reject *hdr;
 546         u32 pld_len, num_bdqs;
 547         unsigned long flags;
 548
 549         spin_lock_bh(&session->back_lock);
 550         cqe_reject = &cqe->cqe_common.iscsi_hdr.reject;
 551         pld_len = cqe_reject->hdr_second_dword &
 552                   ISCSI_REJECT_HDR_DATA_SEG_LEN_MASK;
 553         num_bdqs = pld_len / QEDI_BDQ_BUF_SIZE;
 554
 555         if (cqe->cqe_common.cqe_type == ISCSI_CQE_TYPE_UNSOLICITED) {
 556                 spin_lock_irqsave(&qedi->hba_lock, flags);
 557                 qedi_unsol_pdu_adjust_bdq(qedi, &cqe->cqe_unsolicited,
 558                                           pld_len, num_bdqs, conn->data);
 559                 spin_unlock_irqrestore(&qedi->hba_lock, flags);
 560         }
 561         hdr = (struct iscsi_reject *)&qedi_conn->gen_pdu.resp_hdr;
 562         memset(hdr, 0, sizeof(struct iscsi_hdr));
 563         hdr->opcode = cqe_reject->opcode;
 564         hdr->reason = cqe_reject->hdr_reason;
 565         hdr->flags = cqe_reject->hdr_flags;
 566         hton24(hdr->dlength, (cqe_reject->hdr_second_dword &
 567                               ISCSI_REJECT_HDR_DATA_SEG_LEN_MASK));
 568         hdr->max_cmdsn = cpu_to_be32(cqe_reject->max_cmd_sn);
 569         hdr->exp_cmdsn = cpu_to_be32(cqe_reject->exp_cmd_sn);
 570         hdr->statsn = cpu_to_be32(cqe_reject->stat_sn);
 571         hdr->ffffffff = cpu_to_be32(0xffffffff);
 572
 573         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr,
 574                              conn->data, pld_len);
 575         spin_unlock_bh(&session->back_lock);
 576 }
 577
 578 static void qedi_scsi_completion(struct qedi_ctx *qedi,
 579                                  union iscsi_cqe *cqe,
 580                                  struct iscsi_task *task,
 581                                  struct iscsi_conn *conn)
 582 {
 583         struct scsi_cmnd *sc_cmd;
 584         struct qedi_cmd *cmd = task->dd_data;
 585         struct iscsi_session *session = conn->session;
 586         struct iscsi_scsi_rsp *hdr;
 587         struct iscsi_data_in_hdr *cqe_data_in;
 588         int datalen = 0;
 589         struct qedi_conn *qedi_conn;
 590         u32 iscsi_cid;
 591         bool mark_cmd_node_deleted = false;
 592         u8 cqe_err_bits = 0;
 593
 594         iscsi_cid  = cqe->cqe_common.conn_id;
 595         qedi_conn = qedi->cid_que.conn_cid_tbl[iscsi_cid];
 596
 597         cqe_data_in = &cqe->cqe_common.iscsi_hdr.data_in;
 598         cqe_err_bits =
 599                 cqe->cqe_common.error_bitmap.error_bits.cqe_error_status_bits;
 600
 601         spin_lock_bh(&session->back_lock);
 602         /* get the scsi command */
 603         sc_cmd = cmd->scsi_cmd;
 604
 605         if (!sc_cmd) {
 606                 QEDI_WARN(&qedi->dbg_ctx, "sc_cmd is NULL!\n");
 607                 goto error;
 608         }
 609
 610         if (!sc_cmd->SCp.ptr) {
 611                 QEDI_WARN(&qedi->dbg_ctx,
 612                           "SCp.ptr is NULL, returned in another context.\n");
 613                 goto error;
 614         }
 615
 616         if (!sc_cmd->request) {
 617                 QEDI_WARN(&qedi->dbg_ctx,
 618                           "sc_cmd->request is NULL, sc_cmd=%p.\n",
 619                           sc_cmd);
 620                 goto error;
 621         }
 622
 623         if (!sc_cmd->request->special) {
 624                 QEDI_WARN(&qedi->dbg_ctx,
 625                           "request->special is NULL so request not valid, sc_cmd=%p.\n",
 626                           sc_cmd);
 627                 goto error;
 628         }
 629
 630         if (!sc_cmd->request->q) {
 631                 QEDI_WARN(&qedi->dbg_ctx,
 632                           "request->q is NULL so request is not valid, sc_cmd=%p.\n",
 633                           sc_cmd);
 634                 goto error;
 635         }
 636
 637         qedi_iscsi_unmap_sg_list(cmd);
 638
 639         hdr = (struct iscsi_scsi_rsp *)task->hdr;
 640         hdr->opcode = cqe_data_in->opcode;
 641         hdr->max_cmdsn = cpu_to_be32(cqe_data_in->max_cmd_sn);
 642         hdr->exp_cmdsn = cpu_to_be32(cqe_data_in->exp_cmd_sn);
 643         hdr->itt = build_itt(cqe->cqe_solicited.itid, conn->session->age);
 644         hdr->response = cqe_data_in->reserved1;
 645         hdr->cmd_status = cqe_data_in->status_rsvd;
 646         hdr->flags = cqe_data_in->flags;
 647         hdr->residual_count = cpu_to_be32(cqe_data_in->residual_count);
 648
 649         if (hdr->cmd_status == SAM_STAT_CHECK_CONDITION) {
 650                 datalen = cqe_data_in->reserved2 &
 651                           ISCSI_COMMON_HDR_DATA_SEG_LEN_MASK;
 652                 memcpy((char *)conn->data, (char *)cmd->sense_buffer, datalen);
 653         }
 654
 655         /* If f/w reports data underrun err then set residual to IO transfer
 656          * length, set Underrun flag and clear Overrun flag explicitly
 657          */
 658         if (unlikely(cqe_err_bits &&
 659                      GET_FIELD(cqe_err_bits, CQE_ERROR_BITMAP_UNDER_RUN_ERR))) {
 660                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
 661                           "Under flow itt=0x%x proto flags=0x%x tid=0x%x cid 0x%x fw resid 0x%x sc dlen 0x%x\n",
 662                           hdr->itt, cqe_data_in->flags, cmd->task_id,
 663                           qedi_conn->iscsi_conn_id, hdr->residual_count,
 664                           scsi_bufflen(sc_cmd));
 665                 hdr->residual_count = cpu_to_be32(scsi_bufflen(sc_cmd));
 666                 hdr->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
 667                 hdr->flags &= (~ISCSI_FLAG_CMD_OVERFLOW);
 668         }
 669
 670         spin_lock(&qedi_conn->list_lock);
 671         if (likely(cmd->io_cmd_in_list)) {
 672                 cmd->io_cmd_in_list = false;
 673                 list_del_init(&cmd->io_cmd);
 674                 qedi_conn->active_cmd_count--;
 675                 mark_cmd_node_deleted = true;
 676         }
 677         spin_unlock(&qedi_conn->list_lock);
 678
 679         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,
 680                   "Freeing tid=0x%x for cid=0x%x\n",
 681                   cmd->task_id, qedi_conn->iscsi_conn_id);
 682         cmd->state = RESPONSE_RECEIVED;
 683         if (qedi_io_tracing)
 684                 qedi_trace_io(qedi, task, cmd->task_id, QEDI_IO_TRACE_RSP);
 685
 686         qedi_clear_task_idx(qedi, cmd->task_id);
 687         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr,
 688                              conn->data, datalen);
 689 error:
 690         spin_unlock_bh(&session->back_lock);
 691 }
 692
 693 static void qedi_mtask_completion(struct qedi_ctx *qedi,
 694                                   union iscsi_cqe *cqe,
 695                                   struct iscsi_task *task,
 696                                   struct qedi_conn *conn, uint16_t que_idx)
 697 {
 698         struct iscsi_conn *iscsi_conn;
 699         u32 hdr_opcode;
 700
 701         hdr_opcode = cqe->cqe_common.iscsi_hdr.common.hdr_first_byte;
 702         iscsi_conn = conn->cls_conn->dd_data;
 703
 704         switch (hdr_opcode) {
 705         case ISCSI_OPCODE_SCSI_RESPONSE:
 706         case ISCSI_OPCODE_DATA_IN:
 707                 qedi_scsi_completion(qedi, cqe, task, iscsi_conn);
 708                 break;
 709         case ISCSI_OPCODE_LOGIN_RESPONSE:
 710                 qedi_process_login_resp(qedi, cqe, task, conn);
 711                 break;
 712         case ISCSI_OPCODE_TMF_RESPONSE:
 713                 qedi_process_tmf_resp(qedi, cqe, task, conn);
 714                 break;
 715         case ISCSI_OPCODE_TEXT_RESPONSE:
 716                 qedi_process_text_resp(qedi, cqe, task, conn);
 717                 break;
 718         case ISCSI_OPCODE_LOGOUT_RESPONSE:
 719                 qedi_process_logout_resp(qedi, cqe, task, conn);
 720                 break;
 721         case ISCSI_OPCODE_NOP_IN:
 722                 qedi_process_nopin_mesg(qedi, cqe, task, conn, que_idx);
 723                 break;
 724         default:
 725                 QEDI_ERR(&qedi->dbg_ctx, "unknown opcode\n");
 726         }
 727 }
 728
 729 static void qedi_process_nopin_local_cmpl(struct qedi_ctx *qedi,
 730                                           struct iscsi_cqe_solicited *cqe,
 731                                           struct iscsi_task *task,
 732                                           struct qedi_conn *qedi_conn)
 733 {
 734         struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
 735         struct iscsi_session *session = conn->session;
 736         struct qedi_cmd *cmd = task->dd_data;
 737
 738         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_UNSOL,
 739                   "itid=0x%x, cmd task id=0x%x\n",
 740                   cqe->itid, cmd->task_id);
 741
 742         cmd->state = RESPONSE_RECEIVED;
 743         qedi_clear_task_idx(qedi, cmd->task_id);
 744
 745         spin_lock_bh(&session->back_lock);
 746         __iscsi_put_task(task);
 747         spin_unlock_bh(&session->back_lock);
 748 }
 749
 750 static void qedi_process_cmd_cleanup_resp(struct qedi_ctx *qedi,
 751                                           struct iscsi_cqe_solicited *cqe,
 752                                           struct iscsi_task *task,
 753                                           struct iscsi_conn *conn)
 754 {
 755         struct qedi_work_map *work, *work_tmp;
 756         u32 proto_itt = cqe->itid;
 757         u32 ptmp_itt = 0;
 758         itt_t protoitt = 0;
 759         int found = 0;
 760         struct qedi_cmd *qedi_cmd = NULL;
 761         u32 rtid = 0;
 762         u32 iscsi_cid;
 763         struct qedi_conn *qedi_conn;
 764         struct qedi_cmd *cmd_new, *dbg_cmd;
 765         struct iscsi_task *mtask;
 766         struct iscsi_tm *tmf_hdr = NULL;
 767
 768         iscsi_cid = cqe->conn_id;
 769         qedi_conn = qedi->cid_que.conn_cid_tbl[iscsi_cid];
 770
 771         /* Based on this itt get the corresponding qedi_cmd */
 772         spin_lock_bh(&qedi_conn->tmf_work_lock);
 773         list_for_each_entry_safe(work, work_tmp, &qedi_conn->tmf_work_list,
 774                                  list) {
 775                 if (work->rtid == proto_itt) {
 776                         /* We found the command */
 777                         qedi_cmd = work->qedi_cmd;
 778                         if (!qedi_cmd->list_tmf_work) {
 779                                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
 780                                           "TMF work not found, cqe->tid=0x%x, cid=0x%x\n",
 781                                           proto_itt, qedi_conn->iscsi_conn_id);
 782                                 WARN_ON(1);
 783                         }
 784                         found = 1;
 785                         mtask = qedi_cmd->task;
 786                         tmf_hdr = (struct iscsi_tm *)mtask->hdr;
 787                         rtid = work->rtid;
 788
 789                         list_del_init(&work->list);
 790                         kfree(work);
 791                         qedi_cmd->list_tmf_work = NULL;
 792                 }
 793         }
 794         spin_unlock_bh(&qedi_conn->tmf_work_lock);
 795
 796         if (found) {
 797                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
 798                           "TMF work, cqe->tid=0x%x, tmf flags=0x%x, cid=0x%x\n",
 799                           proto_itt, tmf_hdr->flags, qedi_conn->iscsi_conn_id);
 800
 801                 if ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
 802                     ISCSI_TM_FUNC_ABORT_TASK) {
 803                         spin_lock_bh(&conn->session->back_lock);
 804
 805                         protoitt = build_itt(get_itt(tmf_hdr->rtt),
 806                                              conn->session->age);
 807                         task = iscsi_itt_to_task(conn, protoitt);
 808
 809                         spin_unlock_bh(&conn->session->back_lock);
 810
 811                         if (!task) {
 812                                 QEDI_NOTICE(&qedi->dbg_ctx,
 813                                             "IO task completed, tmf rtt=0x%x, cid=0x%x\n",
 814                                             get_itt(tmf_hdr->rtt),
 815                                             qedi_conn->iscsi_conn_id);
 816                                 return;
 817                         }
 818
 819                         dbg_cmd = task->dd_data;
 820
 821                         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
 822                                   "Abort tmf rtt=0x%x, i/o itt=0x%x, i/o tid=0x%x, cid=0x%x\n",
 823                                   get_itt(tmf_hdr->rtt), get_itt(task->itt),
 824                                   dbg_cmd->task_id, qedi_conn->iscsi_conn_id);
 825
 826                         if (qedi_cmd->state == CLEANUP_WAIT_FAILED)
 827                                 qedi_cmd->state = CLEANUP_RECV;
 828
 829                         qedi_clear_task_idx(qedi_conn->qedi, rtid);
 830
 831                         spin_lock(&qedi_conn->list_lock);
 832                         list_del_init(&dbg_cmd->io_cmd);
 833                         qedi_conn->active_cmd_count--;
 834                         spin_unlock(&qedi_conn->list_lock);
 835                         qedi_cmd->state = CLEANUP_RECV;
 836                         wake_up_interruptible(&qedi_conn->wait_queue);
 837                 }
 838         } else if (qedi_conn->cmd_cleanup_req > 0) {
 839                 spin_lock_bh(&conn->session->back_lock);
 840                 qedi_get_proto_itt(qedi, cqe->itid, &ptmp_itt);
 841                 protoitt = build_itt(ptmp_itt, conn->session->age);
 842                 task = iscsi_itt_to_task(conn, protoitt);
 843                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
 844                           "cleanup io itid=0x%x, protoitt=0x%x, cmd_cleanup_cmpl=%d, cid=0x%x\n",
 845                           cqe->itid, protoitt, qedi_conn->cmd_cleanup_cmpl,
 846                           qedi_conn->iscsi_conn_id);
 847
 848                 spin_unlock_bh(&conn->session->back_lock);
 849                 if (!task) {
 850                         QEDI_NOTICE(&qedi->dbg_ctx,
 851                                     "task is null, itid=0x%x, cid=0x%x\n",
 852                                     cqe->itid, qedi_conn->iscsi_conn_id);
 853                         return;
 854                 }
 855                 qedi_conn->cmd_cleanup_cmpl++;
 856                 wake_up(&qedi_conn->wait_queue);
 857                 cmd_new = task->dd_data;
 858
 859                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,
 860                           "Freeing tid=0x%x for cid=0x%x\n",
 861                           cqe->itid, qedi_conn->iscsi_conn_id);
 862                 qedi_clear_task_idx(qedi_conn->qedi, cqe->itid);
 863
 864         } else {
 865                 qedi_get_proto_itt(qedi, cqe->itid, &ptmp_itt);
 866                 protoitt = build_itt(ptmp_itt, conn->session->age);
 867                 task = iscsi_itt_to_task(conn, protoitt);
 868                 QEDI_ERR(&qedi->dbg_ctx,
 869                          "Delayed or untracked cleanup response, itt=0x%x, tid=0x%x, cid=0x%x, task=%p\n",
 870                          protoitt, cqe->itid, qedi_conn->iscsi_conn_id, task);
 871                 WARN_ON(1);
 872         }
 873 }
 874
 875 void qedi_fp_process_cqes(struct qedi_work *work)
 876 {
 877         struct qedi_ctx *qedi = work->qedi;
 878         union iscsi_cqe *cqe = &work->cqe;
 879         struct iscsi_task *task = NULL;
 880         struct iscsi_nopout *nopout_hdr;
 881         struct qedi_conn *q_conn;
 882         struct iscsi_conn *conn;
 883         struct qedi_cmd *qedi_cmd;
 884         u32 comp_type;
 885         u32 iscsi_cid;
 886         u32 hdr_opcode;
 887         u16 que_idx = work->que_idx;
 888         u8 cqe_err_bits = 0;
 889
 890         comp_type = cqe->cqe_common.cqe_type;
 891         hdr_opcode = cqe->cqe_common.iscsi_hdr.common.hdr_first_byte;
 892         cqe_err_bits =
 893                 cqe->cqe_common.error_bitmap.error_bits.cqe_error_status_bits;
 894
 895         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
 896                   "fw_cid=0x%x, cqe type=0x%x, opcode=0x%x\n",
 897                   cqe->cqe_common.conn_id, comp_type, hdr_opcode);
 898
 899         if (comp_type >= MAX_ISCSI_CQES_TYPE) {
 900                 QEDI_WARN(&qedi->dbg_ctx, "Invalid CqE type\n");
 901                 return;
 902         }
 903
 904         iscsi_cid  = cqe->cqe_common.conn_id;
 905         q_conn = qedi->cid_que.conn_cid_tbl[iscsi_cid];
 906         if (!q_conn) {
 907                 QEDI_WARN(&qedi->dbg_ctx,
 908                           "Session no longer exists for cid=0x%x!!\n",
 909                           iscsi_cid);
 910                 return;
 911         }
 912
 913         conn = q_conn->cls_conn->dd_data;
 914
 915         if (unlikely(cqe_err_bits &&
 916                      GET_FIELD(cqe_err_bits,
 917                                CQE_ERROR_BITMAP_DATA_DIGEST_ERR))) {
 918                 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
 919                 return;
 920         }
 921
 922         switch (comp_type) {
 923         case ISCSI_CQE_TYPE_SOLICITED:
 924         case ISCSI_CQE_TYPE_SOLICITED_WITH_SENSE:
 925                 qedi_cmd = container_of(work, struct qedi_cmd, cqe_work);
 926                 task = qedi_cmd->task;
 927                 if (!task) {
 928                         QEDI_WARN(&qedi->dbg_ctx, "task is NULL\n");
 929                         return;
 930                 }
 931
 932                 /* Process NOPIN local completion */
 933                 nopout_hdr = (struct iscsi_nopout *)task->hdr;
 934                 if ((nopout_hdr->itt == RESERVED_ITT) &&
 935                     (cqe->cqe_solicited.itid != (u16)RESERVED_ITT)) {
 936                         qedi_process_nopin_local_cmpl(qedi, &cqe->cqe_solicited,
 937                                                       task, q_conn);
 938                 } else {
 939                         cqe->cqe_solicited.itid =
 940                                                qedi_get_itt(cqe->cqe_solicited);
 941                         /* Process other solicited responses */
 942                         qedi_mtask_completion(qedi, cqe, task, q_conn, que_idx);
 943                 }
 944                 break;
 945         case ISCSI_CQE_TYPE_UNSOLICITED:
 946                 switch (hdr_opcode) {
 947                 case ISCSI_OPCODE_NOP_IN:
 948                         qedi_process_nopin_mesg(qedi, cqe, task, q_conn,
 949                                                 que_idx);
 950                         break;
 951                 case ISCSI_OPCODE_ASYNC_MSG:
 952                         qedi_process_async_mesg(qedi, cqe, task, q_conn,
 953                                                 que_idx);
 954                         break;
 955                 case ISCSI_OPCODE_REJECT:
 956                         qedi_process_reject_mesg(qedi, cqe, task, q_conn,
 957                                                  que_idx);
 958                         break;
 959                 }
 960                 goto exit_fp_process;
 961         case ISCSI_CQE_TYPE_DUMMY:
 962                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, "Dummy CqE\n");
 963                 goto exit_fp_process;
 964         case ISCSI_CQE_TYPE_TASK_CLEANUP:
 965                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, "CleanUp CqE\n");
 966                 qedi_process_cmd_cleanup_resp(qedi, &cqe->cqe_solicited, task,
 967                                               conn);
 968                 goto exit_fp_process;
 969         default:
 970                 QEDI_ERR(&qedi->dbg_ctx, "Error cqe.\n");
 971                 break;
 972         }
 973
 974 exit_fp_process:
 975         return;
 976 }
 977
 978 static void qedi_add_to_sq(struct qedi_conn *qedi_conn, struct iscsi_task *task,
 979                            u16 tid, uint16_t ptu_invalidate, int is_cleanup)
 980 {
 981         struct iscsi_wqe *wqe;
 982         struct iscsi_wqe_field *cont_field;
 983         struct qedi_endpoint *ep;
 984         struct scsi_cmnd *sc = task->sc;
 985         struct iscsi_login_req *login_hdr;
 986         struct qedi_cmd *cmd = task->dd_data;
 987
 988         login_hdr = (struct iscsi_login_req *)task->hdr;
 989         ep = qedi_conn->ep;
 990         wqe = &ep->sq[ep->sq_prod_idx];
 991
 992         memset(wqe, 0, sizeof(*wqe));
 993
 994         ep->sq_prod_idx++;
 995         ep->fw_sq_prod_idx++;
 996         if (ep->sq_prod_idx == QEDI_SQ_SIZE)
 997                 ep->sq_prod_idx = 0;
 998
 999         if (is_cleanup) {
1000                 SET_FIELD(wqe->flags, ISCSI_WQE_WQE_TYPE,
1001                           ISCSI_WQE_TYPE_TASK_CLEANUP);
1002                 wqe->task_id = tid;
1003                 return;
1004         }
1005
1006         if (ptu_invalidate) {
1007                 SET_FIELD(wqe->flags, ISCSI_WQE_PTU_INVALIDATE,
1008                           ISCSI_WQE_SET_PTU_INVALIDATE);
1009         }
1010
1011         cont_field = &wqe->cont_prevtid_union.cont_field;
1012
1013         switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
1014         case ISCSI_OP_LOGIN:
1015         case ISCSI_OP_TEXT:
1016                 SET_FIELD(wqe->flags, ISCSI_WQE_WQE_TYPE,
1017                           ISCSI_WQE_TYPE_MIDDLE_PATH);
1018                 SET_FIELD(wqe->flags, ISCSI_WQE_NUM_FAST_SGES,
1019                           1);
1020                 cont_field->contlen_cdbsize_field = ntoh24(login_hdr->dlength);
1021                 break;
1022         case ISCSI_OP_LOGOUT:
1023         case ISCSI_OP_NOOP_OUT:
1024         case ISCSI_OP_SCSI_TMFUNC:
1025                  SET_FIELD(wqe->flags, ISCSI_WQE_WQE_TYPE,
1026                            ISCSI_WQE_TYPE_NORMAL);
1027                 break;
1028         default:
1029                 if (!sc)
1030                         break;
1031
1032                 SET_FIELD(wqe->flags, ISCSI_WQE_WQE_TYPE,
1033                           ISCSI_WQE_TYPE_NORMAL);
1034                 cont_field->contlen_cdbsize_field =
1035                                 (sc->sc_data_direction == DMA_TO_DEVICE) ?
1036                                 scsi_bufflen(sc) : 0;
1037                 if (cmd->use_slowpath)
1038                         SET_FIELD(wqe->flags, ISCSI_WQE_NUM_FAST_SGES, 0);
1039                 else
1040                         SET_FIELD(wqe->flags, ISCSI_WQE_NUM_FAST_SGES,
1041                                   (sc->sc_data_direction ==
1042                                    DMA_TO_DEVICE) ?
1043                                   min((u16)QEDI_FAST_SGE_COUNT,
1044                                       (u16)cmd->io_tbl.sge_valid) : 0);
1045                 break;
1046         }
1047
1048         wqe->task_id = tid;
1049         /* Make sure SQ data is coherent */
1050         wmb();
1051 }
1052
1053 static void qedi_ring_doorbell(struct qedi_conn *qedi_conn)
1054 {
1055         struct iscsi_db_data dbell = { 0 };
1056
1057         dbell.agg_flags = 0;
1058
1059         dbell.params |= DB_DEST_XCM << ISCSI_DB_DATA_DEST_SHIFT;
1060         dbell.params |= DB_AGG_CMD_SET << ISCSI_DB_DATA_AGG_CMD_SHIFT;
1061         dbell.params |=
1062                    DQ_XCM_ISCSI_SQ_PROD_CMD << ISCSI_DB_DATA_AGG_VAL_SEL_SHIFT;
1063
1064         dbell.sq_prod = qedi_conn->ep->fw_sq_prod_idx;
1065         writel(*(u32 *)&dbell, qedi_conn->ep->p_doorbell);
1066
1067         /* Make sure fw write idx is coherent, and include both memory barriers
1068          * as a failsafe as for some architectures the call is the same but on
1069          * others they are two different assembly operations.
1070          */
1071         wmb();
1072         mmiowb();
1073         QEDI_INFO(&qedi_conn->qedi->dbg_ctx, QEDI_LOG_MP_REQ,
1074                   "prod_idx=0x%x, fw_prod_idx=0x%x, cid=0x%x\n",
1075                   qedi_conn->ep->sq_prod_idx, qedi_conn->ep->fw_sq_prod_idx,
1076                   qedi_conn->iscsi_conn_id);
1077 }
1078
1079 int qedi_send_iscsi_login(struct qedi_conn *qedi_conn,
1080                           struct iscsi_task *task)
1081 {
1082         struct qedi_ctx *qedi = qedi_conn->qedi;
1083         struct iscsi_task_context *fw_task_ctx;
1084         struct iscsi_login_req *login_hdr;
1085         struct iscsi_login_req_hdr *fw_login_req = NULL;
1086         struct iscsi_cached_sge_ctx *cached_sge = NULL;
1087         struct iscsi_sge *single_sge = NULL;
1088         struct iscsi_sge *req_sge = NULL;
1089         struct iscsi_sge *resp_sge = NULL;
1090         struct qedi_cmd *qedi_cmd;
1091         s16 ptu_invalidate = 0;
1092         s16 tid = 0;
1093
1094         req_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.req_bd_tbl;
1095         resp_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.resp_bd_tbl;
1096         qedi_cmd = (struct qedi_cmd *)task->dd_data;
1097         login_hdr = (struct iscsi_login_req *)task->hdr;
1098
1099         tid = qedi_get_task_idx(qedi);
1100         if (tid == -1)
1101                 return -ENOMEM;
1102
1103         fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid);
1104         memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context));
1105
1106         qedi_cmd->task_id = tid;
1107
1108         /* Ystorm context */
1109         fw_login_req = &fw_task_ctx->ystorm_st_context.pdu_hdr.login_req;
1110         fw_login_req->opcode = login_hdr->opcode;
1111         fw_login_req->version_min = login_hdr->min_version;
1112         fw_login_req->version_max = login_hdr->max_version;
1113         fw_login_req->flags_attr = login_hdr->flags;
1114         fw_login_req->isid_tabc = *((u16 *)login_hdr->isid + 2);
1115         fw_login_req->isid_d = *((u32 *)login_hdr->isid);
1116         fw_login_req->tsih = login_hdr->tsih;
1117         qedi_update_itt_map(qedi, tid, task->itt, qedi_cmd);
1118         fw_login_req->itt = qedi_set_itt(tid, get_itt(task->itt));
1119         fw_login_req->cid = qedi_conn->iscsi_conn_id;
1120         fw_login_req->cmd_sn = be32_to_cpu(login_hdr->cmdsn);
1121         fw_login_req->exp_stat_sn = be32_to_cpu(login_hdr->exp_statsn);
1122         fw_login_req->exp_stat_sn = 0;
1123
1124         if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) {
1125                 ptu_invalidate = 1;
1126                 qedi->tid_reuse_count[tid] = 0;
1127         }
1128
1129         fw_task_ctx->ystorm_st_context.state.reuse_count =
1130                                                 qedi->tid_reuse_count[tid];
1131         fw_task_ctx->mstorm_st_context.reuse_count =
1132                                                 qedi->tid_reuse_count[tid]++;
1133         cached_sge =
1134                &fw_task_ctx->ystorm_st_context.state.sgl_ctx_union.cached_sge;
1135         cached_sge->sge.sge_len = req_sge->sge_len;
1136         cached_sge->sge.sge_addr.lo = (u32)(qedi_conn->gen_pdu.req_dma_addr);
1137         cached_sge->sge.sge_addr.hi =
1138                              (u32)((u64)qedi_conn->gen_pdu.req_dma_addr >> 32);
1139
1140         /* Mstorm context */
1141         single_sge = &fw_task_ctx->mstorm_st_context.sgl_union.single_sge;
1142         fw_task_ctx->mstorm_st_context.task_type = 0x2;
1143         fw_task_ctx->mstorm_ag_context.task_cid = (u16)qedi_conn->iscsi_conn_id;
1144         single_sge->sge_addr.lo = resp_sge->sge_addr.lo;
1145         single_sge->sge_addr.hi = resp_sge->sge_addr.hi;
1146         single_sge->sge_len = resp_sge->sge_len;
1147
1148         SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
1149                   ISCSI_MFLAGS_SINGLE_SGE, 1);
1150         SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
1151                   ISCSI_MFLAGS_SLOW_IO, 0);
1152         fw_task_ctx->mstorm_st_context.sgl_size = 1;
1153         fw_task_ctx->mstorm_st_context.rem_task_size = resp_sge->sge_len;
1154
1155         /* Ustorm context */
1156         fw_task_ctx->ustorm_st_context.rem_rcv_len = resp_sge->sge_len;
1157         fw_task_ctx->ustorm_st_context.exp_data_transfer_len =
1158                                                 ntoh24(login_hdr->dlength);
1159         fw_task_ctx->ustorm_st_context.exp_data_sn = 0;
1160         fw_task_ctx->ustorm_st_context.cq_rss_number = 0;
1161         fw_task_ctx->ustorm_st_context.task_type = 0x2;
1162         fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id;
1163         fw_task_ctx->ustorm_ag_context.exp_data_acked =
1164                                                  ntoh24(login_hdr->dlength);
1165         SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1,
1166                   USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1);
1167         SET_FIELD(fw_task_ctx->ustorm_st_context.flags,
1168                   USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP, 0);
1169
1170         spin_lock(&qedi_conn->list_lock);
1171         list_add_tail(&qedi_cmd->io_cmd, &qedi_conn->active_cmd_list);
1172         qedi_cmd->io_cmd_in_list = true;
1173         qedi_conn->active_cmd_count++;
1174         spin_unlock(&qedi_conn->list_lock);
1175
1176         qedi_add_to_sq(qedi_conn, task, tid, ptu_invalidate, false);
1177         qedi_ring_doorbell(qedi_conn);
1178         return 0;
1179 }
1180
1181 int qedi_send_iscsi_logout(struct qedi_conn *qedi_conn,
1182                            struct iscsi_task *task)
1183 {
1184         struct qedi_ctx *qedi = qedi_conn->qedi;
1185         struct iscsi_logout_req_hdr *fw_logout_req = NULL;
1186         struct iscsi_task_context *fw_task_ctx = NULL;
1187         struct iscsi_logout *logout_hdr = NULL;
1188         struct qedi_cmd *qedi_cmd = NULL;
1189         s16  tid = 0;
1190         s16 ptu_invalidate = 0;
1191
1192         qedi_cmd = (struct qedi_cmd *)task->dd_data;
1193         logout_hdr = (struct iscsi_logout *)task->hdr;
1194
1195         tid = qedi_get_task_idx(qedi);
1196         if (tid == -1)
1197                 return -ENOMEM;
1198
1199         fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid);
1200
1201         memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context));
1202         qedi_cmd->task_id = tid;
1203
1204         /* Ystorm context */
1205         fw_logout_req = &fw_task_ctx->ystorm_st_context.pdu_hdr.logout_req;
1206         fw_logout_req->opcode = ISCSI_OPCODE_LOGOUT_REQUEST;
1207         fw_logout_req->reason_code = 0x80 | logout_hdr->flags;
1208         qedi_update_itt_map(qedi, tid, task->itt, qedi_cmd);
1209         fw_logout_req->itt = qedi_set_itt(tid, get_itt(task->itt));
1210         fw_logout_req->exp_stat_sn = be32_to_cpu(logout_hdr->exp_statsn);
1211         fw_logout_req->cmd_sn = be32_to_cpu(logout_hdr->cmdsn);
1212
1213         if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) {
1214                 ptu_invalidate = 1;
1215                 qedi->tid_reuse_count[tid] = 0;
1216         }
1217         fw_task_ctx->ystorm_st_context.state.reuse_count =
1218                                                   qedi->tid_reuse_count[tid];
1219         fw_task_ctx->mstorm_st_context.reuse_count =
1220                                                 qedi->tid_reuse_count[tid]++;
1221         fw_logout_req->cid = qedi_conn->iscsi_conn_id;
1222         fw_task_ctx->ystorm_st_context.state.buffer_offset[0] = 0;
1223
1224         /* Mstorm context */
1225         fw_task_ctx->mstorm_st_context.task_type = ISCSI_TASK_TYPE_MIDPATH;
1226         fw_task_ctx->mstorm_ag_context.task_cid = (u16)qedi_conn->iscsi_conn_id;
1227
1228         /* Ustorm context */
1229         fw_task_ctx->ustorm_st_context.rem_rcv_len = 0;
1230         fw_task_ctx->ustorm_st_context.exp_data_transfer_len = 0;
1231         fw_task_ctx->ustorm_st_context.exp_data_sn = 0;
1232         fw_task_ctx->ustorm_st_context.task_type =  ISCSI_TASK_TYPE_MIDPATH;
1233         fw_task_ctx->ustorm_st_context.cq_rss_number = 0;
1234
1235         SET_FIELD(fw_task_ctx->ustorm_st_context.flags,
1236                   USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP, 0);
1237         SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map,
1238                   ISCSI_REG1_NUM_FAST_SGES, 0);
1239
1240         fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id;
1241         SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1,
1242                   USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1);
1243
1244         spin_lock(&qedi_conn->list_lock);
1245         list_add_tail(&qedi_cmd->io_cmd, &qedi_conn->active_cmd_list);
1246         qedi_cmd->io_cmd_in_list = true;
1247         qedi_conn->active_cmd_count++;
1248         spin_unlock(&qedi_conn->list_lock);
1249
1250         qedi_add_to_sq(qedi_conn, task, tid, ptu_invalidate, false);
1251         qedi_ring_doorbell(qedi_conn);
1252
1253         return 0;
1254 }
1255
1256 int qedi_cleanup_all_io(struct qedi_ctx *qedi, struct qedi_conn *qedi_conn,
1257                         struct iscsi_task *task, bool in_recovery)
1258 {
1259         int rval;
1260         struct iscsi_task *ctask;
1261         struct qedi_cmd *cmd, *cmd_tmp;
1262         struct iscsi_tm *tmf_hdr;
1263         unsigned int lun = 0;
1264         bool lun_reset = false;
1265         struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
1266         struct iscsi_session *session = conn->session;
1267
1268         /* From recovery, task is NULL or from tmf resp valid task */
1269         if (task) {
1270                 tmf_hdr = (struct iscsi_tm *)task->hdr;
1271
1272                 if ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
1273                         ISCSI_TM_FUNC_LOGICAL_UNIT_RESET) {
1274                         lun_reset = true;
1275                         lun = scsilun_to_int(&tmf_hdr->lun);
1276                 }
1277         }
1278
1279         qedi_conn->cmd_cleanup_req = 0;
1280         qedi_conn->cmd_cleanup_cmpl = 0;
1281
1282         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
1283                   "active_cmd_count=%d, cid=0x%x, in_recovery=%d, lun_reset=%d\n",
1284                   qedi_conn->active_cmd_count, qedi_conn->iscsi_conn_id,
1285                   in_recovery, lun_reset);
1286
1287         if (lun_reset)
1288                 spin_lock_bh(&session->back_lock);
1289
1290         spin_lock(&qedi_conn->list_lock);
1291
1292         list_for_each_entry_safe(cmd, cmd_tmp, &qedi_conn->active_cmd_list,
1293                                  io_cmd) {
1294                 ctask = cmd->task;
1295                 if (ctask == task)
1296                         continue;
1297
1298                 if (lun_reset) {
1299                         if (cmd->scsi_cmd && cmd->scsi_cmd->device) {
1300                                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
1301                                           "tid=0x%x itt=0x%x scsi_cmd_ptr=%p device=%p task_state=%d cmd_state=0%x cid=0x%x\n",
1302                                           cmd->task_id, get_itt(ctask->itt),
1303                                           cmd->scsi_cmd, cmd->scsi_cmd->device,
1304                                           ctask->state, cmd->state,
1305                                           qedi_conn->iscsi_conn_id);
1306                                 if (cmd->scsi_cmd->device->lun != lun)
1307                                         continue;
1308                         }
1309                 }
1310                 qedi_conn->cmd_cleanup_req++;
1311                 qedi_iscsi_cleanup_task(ctask, true);
1312
1313                 list_del_init(&cmd->io_cmd);
1314                 qedi_conn->active_cmd_count--;
1315                 QEDI_WARN(&qedi->dbg_ctx,
1316                           "Deleted active cmd list node io_cmd=%p, cid=0x%x\n",
1317                           &cmd->io_cmd, qedi_conn->iscsi_conn_id);
1318         }
1319
1320         spin_unlock(&qedi_conn->list_lock);
1321
1322         if (lun_reset)
1323                 spin_unlock_bh(&session->back_lock);
1324
1325         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
1326                   "cmd_cleanup_req=%d, cid=0x%x\n",
1327                   qedi_conn->cmd_cleanup_req,
1328                   qedi_conn->iscsi_conn_id);
1329
1330         rval  = wait_event_interruptible_timeout(qedi_conn->wait_queue,
1331                                                  ((qedi_conn->cmd_cleanup_req ==
1332                                                  qedi_conn->cmd_cleanup_cmpl) ||
1333                                                  qedi_conn->ep),
1334                                                  5 * HZ);
1335         if (rval) {
1336                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
1337                           "i/o cmd_cleanup_req=%d, equal to cmd_cleanup_cmpl=%d, cid=0x%x\n",
1338                           qedi_conn->cmd_cleanup_req,
1339                           qedi_conn->cmd_cleanup_cmpl,
1340                           qedi_conn->iscsi_conn_id);
1341
1342                 return 0;
1343         }
1344
1345         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
1346                   "i/o cmd_cleanup_req=%d, not equal to cmd_cleanup_cmpl=%d, cid=0x%x\n",
1347                   qedi_conn->cmd_cleanup_req,
1348                   qedi_conn->cmd_cleanup_cmpl,
1349                   qedi_conn->iscsi_conn_id);
1350
1351         iscsi_host_for_each_session(qedi->shost,
1352                                     qedi_mark_device_missing);
1353         qedi_ops->common->drain(qedi->cdev);
1354
1355         /* Enable IOs for all other sessions except current.*/
1356         if (!wait_event_interruptible_timeout(qedi_conn->wait_queue,
1357                                               (qedi_conn->cmd_cleanup_req ==
1358                                                qedi_conn->cmd_cleanup_cmpl),
1359                                               5 * HZ)) {
1360                 iscsi_host_for_each_session(qedi->shost,
1361                                             qedi_mark_device_available);
1362                 return -1;
1363         }
1364
1365         iscsi_host_for_each_session(qedi->shost,
1366                                     qedi_mark_device_available);
1367
1368         return 0;
1369 }
1370
1371 void qedi_clearsq(struct qedi_ctx *qedi, struct qedi_conn *qedi_conn,
1372                   struct iscsi_task *task)
1373 {
1374         struct qedi_endpoint *qedi_ep;
1375         int rval;
1376
1377         qedi_ep = qedi_conn->ep;
1378         qedi_conn->cmd_cleanup_req = 0;
1379         qedi_conn->cmd_cleanup_cmpl = 0;
1380
1381         if (!qedi_ep) {
1382                 QEDI_WARN(&qedi->dbg_ctx,
1383                           "Cannot proceed, ep already disconnected, cid=0x%x\n",
1384                           qedi_conn->iscsi_conn_id);
1385                 return;
1386         }
1387
1388         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1389                   "Clearing SQ for cid=0x%x, conn=%p, ep=%p\n",
1390                   qedi_conn->iscsi_conn_id, qedi_conn, qedi_ep);
1391
1392         qedi_ops->clear_sq(qedi->cdev, qedi_ep->handle);
1393
1394         rval = qedi_cleanup_all_io(qedi, qedi_conn, task, true);
1395         if (rval) {
1396                 QEDI_ERR(&qedi->dbg_ctx,
1397                          "fatal error, need hard reset, cid=0x%x\n",
1398                          qedi_conn->iscsi_conn_id);
1399                 WARN_ON(1);
1400         }
1401 }
1402
1403 static int qedi_wait_for_cleanup_request(struct qedi_ctx *qedi,
1404                                          struct qedi_conn *qedi_conn,
1405                                          struct iscsi_task *task,
1406                                          struct qedi_cmd *qedi_cmd,
1407                                          struct qedi_work_map *list_work)
1408 {
1409         struct qedi_cmd *cmd = (struct qedi_cmd *)task->dd_data;
1410         int wait;
1411
1412         wait  = wait_event_interruptible_timeout(qedi_conn->wait_queue,
1413                                                  ((qedi_cmd->state ==
1414                                                    CLEANUP_RECV) ||
1415                                                  ((qedi_cmd->type == TYPEIO) &&
1416                                                   (cmd->state ==
1417                                                    RESPONSE_RECEIVED))),
1418                                                  5 * HZ);
1419         if (!wait) {
1420                 qedi_cmd->state = CLEANUP_WAIT_FAILED;
1421
1422                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
1423                           "Cleanup timedout tid=0x%x, issue connection recovery, cid=0x%x\n",
1424                           cmd->task_id, qedi_conn->iscsi_conn_id);
1425
1426                 return -1;
1427         }
1428         return 0;
1429 }
1430
1431 static void qedi_tmf_work(struct work_struct *work)
1432 {
1433         struct qedi_cmd *qedi_cmd =
1434                 container_of(work, struct qedi_cmd, tmf_work);
1435         struct qedi_conn *qedi_conn = qedi_cmd->conn;
1436         struct qedi_ctx *qedi = qedi_conn->qedi;
1437         struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
1438         struct iscsi_cls_session *cls_sess;
1439         struct qedi_work_map *list_work = NULL;
1440         struct iscsi_task *mtask;
1441         struct qedi_cmd *cmd;
1442         struct iscsi_task *ctask;
1443         struct iscsi_tm *tmf_hdr;
1444         s16 rval = 0;
1445         s16 tid = 0;
1446
1447         mtask = qedi_cmd->task;
1448         tmf_hdr = (struct iscsi_tm *)mtask->hdr;
1449         cls_sess = iscsi_conn_to_session(qedi_conn->cls_conn);
1450         set_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags);
1451
1452         ctask = iscsi_itt_to_task(conn, tmf_hdr->rtt);
1453         if (!ctask || !ctask->sc) {
1454                 QEDI_ERR(&qedi->dbg_ctx, "Task already completed\n");
1455                 goto abort_ret;
1456         }
1457
1458         cmd = (struct qedi_cmd *)ctask->dd_data;
1459         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1460                   "Abort tmf rtt=0x%x, cmd itt=0x%x, cmd tid=0x%x, cid=0x%x\n",
1461                   get_itt(tmf_hdr->rtt), get_itt(ctask->itt), cmd->task_id,
1462                   qedi_conn->iscsi_conn_id);
1463
1464         if (qedi_do_not_recover) {
1465                 QEDI_ERR(&qedi->dbg_ctx, "DONT SEND CLEANUP/ABORT %d\n",
1466                          qedi_do_not_recover);
1467                 goto abort_ret;
1468         }
1469
1470         list_work = kzalloc(sizeof(*list_work), GFP_ATOMIC);
1471         if (!list_work) {
1472                 QEDI_ERR(&qedi->dbg_ctx, "Memory alloction failed\n");
1473                 goto abort_ret;
1474         }
1475
1476         qedi_cmd->type = TYPEIO;
1477         list_work->qedi_cmd = qedi_cmd;
1478         list_work->rtid = cmd->task_id;
1479         list_work->state = QEDI_WORK_SCHEDULED;
1480         qedi_cmd->list_tmf_work = list_work;
1481
1482         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
1483                   "Queue tmf work=%p, list node=%p, cid=0x%x, tmf flags=0x%x\n",
1484                   list_work->ptr_tmf_work, list_work, qedi_conn->iscsi_conn_id,
1485                   tmf_hdr->flags);
1486
1487         spin_lock_bh(&qedi_conn->tmf_work_lock);
1488         list_add_tail(&list_work->list, &qedi_conn->tmf_work_list);
1489         spin_unlock_bh(&qedi_conn->tmf_work_lock);
1490
1491         qedi_iscsi_cleanup_task(ctask, false);
1492
1493         rval = qedi_wait_for_cleanup_request(qedi, qedi_conn, ctask, qedi_cmd,
1494                                              list_work);
1495         if (rval == -1) {
1496                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1497                           "FW cleanup got escalated, cid=0x%x\n",
1498                           qedi_conn->iscsi_conn_id);
1499                 goto ldel_exit;
1500         }
1501
1502         tid = qedi_get_task_idx(qedi);
1503         if (tid == -1) {
1504                 QEDI_ERR(&qedi->dbg_ctx, "Invalid tid, cid=0x%x\n",
1505                          qedi_conn->iscsi_conn_id);
1506                 goto ldel_exit;
1507         }
1508
1509         qedi_cmd->task_id = tid;
1510         qedi_send_iscsi_tmf(qedi_conn, qedi_cmd->task);
1511
1512 abort_ret:
1513         clear_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags);
1514         return;
1515
1516 ldel_exit:
1517         spin_lock_bh(&qedi_conn->tmf_work_lock);
1518         if (!qedi_cmd->list_tmf_work) {
1519                 list_del_init(&list_work->list);
1520                 qedi_cmd->list_tmf_work = NULL;
1521                 kfree(list_work);
1522         }
1523         spin_unlock_bh(&qedi_conn->tmf_work_lock);
1524
1525         spin_lock(&qedi_conn->list_lock);
1526         list_del_init(&cmd->io_cmd);
1527         qedi_conn->active_cmd_count--;
1528         spin_unlock(&qedi_conn->list_lock);
1529
1530         clear_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags);
1531 }
1532
1533 static int qedi_send_iscsi_tmf(struct qedi_conn *qedi_conn,
1534                                struct iscsi_task *mtask)
1535 {
1536         struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
1537         struct qedi_ctx *qedi = qedi_conn->qedi;
1538         struct iscsi_task_context *fw_task_ctx;
1539         struct iscsi_tmf_request_hdr *fw_tmf_request;
1540         struct iscsi_sge *single_sge;
1541         struct qedi_cmd *qedi_cmd;
1542         struct qedi_cmd *cmd;
1543         struct iscsi_task *ctask;
1544         struct iscsi_tm *tmf_hdr;
1545         struct iscsi_sge *req_sge;
1546         struct iscsi_sge *resp_sge;
1547         u32 lun[2];
1548         s16 tid = 0, ptu_invalidate = 0;
1549
1550         req_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.req_bd_tbl;
1551         resp_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.resp_bd_tbl;
1552         qedi_cmd = (struct qedi_cmd *)mtask->dd_data;
1553         tmf_hdr = (struct iscsi_tm *)mtask->hdr;
1554
1555         tid = qedi_cmd->task_id;
1556         qedi_update_itt_map(qedi, tid, mtask->itt, qedi_cmd);
1557
1558         fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid);
1559         memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context));
1560
1561         fw_tmf_request = &fw_task_ctx->ystorm_st_context.pdu_hdr.tmf_request;
1562         fw_tmf_request->itt = qedi_set_itt(tid, get_itt(mtask->itt));
1563         fw_tmf_request->cmd_sn = be32_to_cpu(tmf_hdr->cmdsn);
1564
1565         memcpy(lun, &tmf_hdr->lun, sizeof(struct scsi_lun));
1566         fw_tmf_request->lun.lo = be32_to_cpu(lun[0]);
1567         fw_tmf_request->lun.hi = be32_to_cpu(lun[1]);
1568
1569         if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) {
1570                 ptu_invalidate = 1;
1571                 qedi->tid_reuse_count[tid] = 0;
1572         }
1573         fw_task_ctx->ystorm_st_context.state.reuse_count =
1574                                                 qedi->tid_reuse_count[tid];
1575         fw_task_ctx->mstorm_st_context.reuse_count =
1576                                                 qedi->tid_reuse_count[tid]++;
1577
1578         if ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
1579              ISCSI_TM_FUNC_ABORT_TASK) {
1580                 ctask = iscsi_itt_to_task(conn, tmf_hdr->rtt);
1581                 if (!ctask || !ctask->sc) {
1582                         QEDI_ERR(&qedi->dbg_ctx,
1583                                  "Could not get reference task\n");
1584                         return 0;
1585                 }
1586                 cmd = (struct qedi_cmd *)ctask->dd_data;
1587                 fw_tmf_request->rtt =
1588                                 qedi_set_itt(cmd->task_id,
1589                                              get_itt(tmf_hdr->rtt));
1590         } else {
1591                 fw_tmf_request->rtt = ISCSI_RESERVED_TAG;
1592         }
1593
1594         fw_tmf_request->opcode = tmf_hdr->opcode;
1595         fw_tmf_request->function = tmf_hdr->flags;
1596         fw_tmf_request->hdr_second_dword = ntoh24(tmf_hdr->dlength);
1597         fw_tmf_request->ref_cmd_sn = be32_to_cpu(tmf_hdr->refcmdsn);
1598
1599         single_sge = &fw_task_ctx->mstorm_st_context.sgl_union.single_sge;
1600         fw_task_ctx->mstorm_st_context.task_type = ISCSI_TASK_TYPE_MIDPATH;
1601         fw_task_ctx->mstorm_ag_context.task_cid = (u16)qedi_conn->iscsi_conn_id;
1602         single_sge->sge_addr.lo = resp_sge->sge_addr.lo;
1603         single_sge->sge_addr.hi = resp_sge->sge_addr.hi;
1604         single_sge->sge_len = resp_sge->sge_len;
1605
1606         SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
1607                   ISCSI_MFLAGS_SINGLE_SGE, 1);
1608         SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
1609                   ISCSI_MFLAGS_SLOW_IO, 0);
1610         fw_task_ctx->mstorm_st_context.sgl_size = 1;
1611         fw_task_ctx->mstorm_st_context.rem_task_size = resp_sge->sge_len;
1612
1613         /* Ustorm context */
1614         fw_task_ctx->ustorm_st_context.rem_rcv_len = 0;
1615         fw_task_ctx->ustorm_st_context.exp_data_transfer_len = 0;
1616         fw_task_ctx->ustorm_st_context.exp_data_sn = 0;
1617         fw_task_ctx->ustorm_st_context.task_type =  ISCSI_TASK_TYPE_MIDPATH;
1618         fw_task_ctx->ustorm_st_context.cq_rss_number = 0;
1619
1620         SET_FIELD(fw_task_ctx->ustorm_st_context.flags,
1621                   USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP, 0);
1622         SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map,
1623                   ISCSI_REG1_NUM_FAST_SGES, 0);
1624
1625         fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id;
1626         SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1,
1627                   USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1);
1628         fw_task_ctx->ustorm_st_context.lun.lo = be32_to_cpu(lun[0]);
1629         fw_task_ctx->ustorm_st_context.lun.hi = be32_to_cpu(lun[1]);
1630
1631         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
1632                   "Add TMF to SQ, tmf tid=0x%x, itt=0x%x, cid=0x%x\n",
1633                   tid,  mtask->itt, qedi_conn->iscsi_conn_id);
1634
1635         spin_lock(&qedi_conn->list_lock);
1636         list_add_tail(&qedi_cmd->io_cmd, &qedi_conn->active_cmd_list);
1637         qedi_cmd->io_cmd_in_list = true;
1638         qedi_conn->active_cmd_count++;
1639         spin_unlock(&qedi_conn->list_lock);
1640
1641         qedi_add_to_sq(qedi_conn, mtask, tid, ptu_invalidate, false);
1642         qedi_ring_doorbell(qedi_conn);
1643         return 0;
1644 }
1645
1646 int qedi_iscsi_abort_work(struct qedi_conn *qedi_conn,
1647                           struct iscsi_task *mtask)
1648 {
1649         struct qedi_ctx *qedi = qedi_conn->qedi;
1650         struct iscsi_tm *tmf_hdr;
1651         struct qedi_cmd *qedi_cmd = (struct qedi_cmd *)mtask->dd_data;
1652         s16 tid = 0;
1653
1654         tmf_hdr = (struct iscsi_tm *)mtask->hdr;
1655         qedi_cmd->task = mtask;
1656
1657         /* If abort task then schedule the work and return */
1658         if ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
1659             ISCSI_TM_FUNC_ABORT_TASK) {
1660                 qedi_cmd->state = CLEANUP_WAIT;
1661                 INIT_WORK(&qedi_cmd->tmf_work, qedi_tmf_work);
1662                 queue_work(qedi->tmf_thread, &qedi_cmd->tmf_work);
1663
1664         } else if (((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
1665                     ISCSI_TM_FUNC_LOGICAL_UNIT_RESET) ||
1666                    ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
1667                     ISCSI_TM_FUNC_TARGET_WARM_RESET) ||
1668                    ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) ==
1669                     ISCSI_TM_FUNC_TARGET_COLD_RESET)) {
1670                 tid = qedi_get_task_idx(qedi);
1671                 if (tid == -1) {
1672                         QEDI_ERR(&qedi->dbg_ctx, "Invalid tid, cid=0x%x\n",
1673                                  qedi_conn->iscsi_conn_id);
1674                         return -1;
1675                 }
1676                 qedi_cmd->task_id = tid;
1677
1678                 qedi_send_iscsi_tmf(qedi_conn, qedi_cmd->task);
1679
1680         } else {
1681                 QEDI_ERR(&qedi->dbg_ctx, "Invalid tmf, cid=0x%x\n",
1682                          qedi_conn->iscsi_conn_id);
1683                 return -1;
1684         }
1685
1686         return 0;
1687 }
1688
1689 int qedi_send_iscsi_text(struct qedi_conn *qedi_conn,
1690                          struct iscsi_task *task)
1691 {
1692         struct qedi_ctx *qedi = qedi_conn->qedi;
1693         struct iscsi_task_context *fw_task_ctx;
1694         struct iscsi_text_request_hdr *fw_text_request;
1695         struct iscsi_cached_sge_ctx *cached_sge;
1696         struct iscsi_sge *single_sge;
1697         struct qedi_cmd *qedi_cmd;
1698         /* For 6.5 hdr iscsi_hdr */
1699         struct iscsi_text *text_hdr;
1700         struct iscsi_sge *req_sge;
1701         struct iscsi_sge *resp_sge;
1702         s16 ptu_invalidate = 0;
1703         s16 tid = 0;
1704
1705         req_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.req_bd_tbl;
1706         resp_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.resp_bd_tbl;
1707         qedi_cmd = (struct qedi_cmd *)task->dd_data;
1708         text_hdr = (struct iscsi_text *)task->hdr;
1709
1710         tid = qedi_get_task_idx(qedi);
1711         if (tid == -1)
1712                 return -ENOMEM;
1713
1714         fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid);
1715         memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context));
1716
1717         qedi_cmd->task_id = tid;
1718
1719         /* Ystorm context */
1720         fw_text_request =
1721                         &fw_task_ctx->ystorm_st_context.pdu_hdr.text_request;
1722         fw_text_request->opcode = text_hdr->opcode;
1723         fw_text_request->flags_attr = text_hdr->flags;
1724
1725         qedi_update_itt_map(qedi, tid, task->itt, qedi_cmd);
1726         fw_text_request->itt = qedi_set_itt(tid, get_itt(task->itt));
1727         fw_text_request->ttt = text_hdr->ttt;
1728         fw_text_request->cmd_sn = be32_to_cpu(text_hdr->cmdsn);
1729         fw_text_request->exp_stat_sn = be32_to_cpu(text_hdr->exp_statsn);
1730         fw_text_request->hdr_second_dword = ntoh24(text_hdr->dlength);
1731
1732         if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) {
1733                 ptu_invalidate = 1;
1734                 qedi->tid_reuse_count[tid] = 0;
1735         }
1736         fw_task_ctx->ystorm_st_context.state.reuse_count =
1737                                                      qedi->tid_reuse_count[tid];
1738         fw_task_ctx->mstorm_st_context.reuse_count =
1739                                                    qedi->tid_reuse_count[tid]++;
1740
1741         cached_sge =
1742                &fw_task_ctx->ystorm_st_context.state.sgl_ctx_union.cached_sge;
1743         cached_sge->sge.sge_len = req_sge->sge_len;
1744         cached_sge->sge.sge_addr.lo = (u32)(qedi_conn->gen_pdu.req_dma_addr);
1745         cached_sge->sge.sge_addr.hi =
1746                               (u32)((u64)qedi_conn->gen_pdu.req_dma_addr >> 32);
1747
1748         /* Mstorm context */
1749         single_sge = &fw_task_ctx->mstorm_st_context.sgl_union.single_sge;
1750         fw_task_ctx->mstorm_st_context.task_type = 0x2;
1751         fw_task_ctx->mstorm_ag_context.task_cid = (u16)qedi_conn->iscsi_conn_id;
1752         single_sge->sge_addr.lo = resp_sge->sge_addr.lo;
1753         single_sge->sge_addr.hi = resp_sge->sge_addr.hi;
1754         single_sge->sge_len = resp_sge->sge_len;
1755
1756         SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
1757                   ISCSI_MFLAGS_SINGLE_SGE, 1);
1758         SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
1759                   ISCSI_MFLAGS_SLOW_IO, 0);
1760         fw_task_ctx->mstorm_st_context.sgl_size = 1;
1761         fw_task_ctx->mstorm_st_context.rem_task_size = resp_sge->sge_len;
1762
1763         /* Ustorm context */
1764         fw_task_ctx->ustorm_ag_context.exp_data_acked =
1765                                                       ntoh24(text_hdr->dlength);
1766         fw_task_ctx->ustorm_st_context.rem_rcv_len = resp_sge->sge_len;
1767         fw_task_ctx->ustorm_st_context.exp_data_transfer_len =
1768                                                       ntoh24(text_hdr->dlength);
1769         fw_task_ctx->ustorm_st_context.exp_data_sn =
1770                                               be32_to_cpu(text_hdr->exp_statsn);
1771         fw_task_ctx->ustorm_st_context.cq_rss_number = 0;
1772         fw_task_ctx->ustorm_st_context.task_type = 0x2;
1773         fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id;
1774         SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1,
1775                   USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1);
1776
1777         /*  Add command in active command list */
1778         spin_lock(&qedi_conn->list_lock);
1779         list_add_tail(&qedi_cmd->io_cmd, &qedi_conn->active_cmd_list);
1780         qedi_cmd->io_cmd_in_list = true;
1781         qedi_conn->active_cmd_count++;
1782         spin_unlock(&qedi_conn->list_lock);
1783
1784         qedi_add_to_sq(qedi_conn, task, tid, ptu_invalidate, false);
1785         qedi_ring_doorbell(qedi_conn);
1786
1787         return 0;
1788 }
1789
1790 int qedi_send_iscsi_nopout(struct qedi_conn *qedi_conn,
1791                            struct iscsi_task *task,
1792                            char *datap, int data_len, int unsol)
1793 {
1794         struct qedi_ctx *qedi = qedi_conn->qedi;
1795         struct iscsi_task_context *fw_task_ctx;
1796         struct iscsi_nop_out_hdr *fw_nop_out;
1797         struct qedi_cmd *qedi_cmd;
1798         /* For 6.5 hdr iscsi_hdr */
1799         struct iscsi_nopout *nopout_hdr;
1800         struct iscsi_cached_sge_ctx *cached_sge;
1801         struct iscsi_sge *single_sge;
1802         struct iscsi_sge *req_sge;
1803         struct iscsi_sge *resp_sge;
1804         u32 lun[2];
1805         s16 ptu_invalidate = 0;
1806         s16 tid = 0;
1807
1808         req_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.req_bd_tbl;
1809         resp_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.resp_bd_tbl;
1810         qedi_cmd = (struct qedi_cmd *)task->dd_data;
1811         nopout_hdr = (struct iscsi_nopout *)task->hdr;
1812
1813         tid = qedi_get_task_idx(qedi);
1814         if (tid == -1) {
1815                 QEDI_WARN(&qedi->dbg_ctx, "Invalid tid\n");
1816                 return -ENOMEM;
1817         }
1818
1819         fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid);
1820
1821         memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context));
1822         qedi_cmd->task_id = tid;
1823
1824         /* Ystorm context */
1825         fw_nop_out = &fw_task_ctx->ystorm_st_context.pdu_hdr.nop_out;
1826         SET_FIELD(fw_nop_out->flags_attr, ISCSI_NOP_OUT_HDR_CONST1, 1);
1827         SET_FIELD(fw_nop_out->flags_attr, ISCSI_NOP_OUT_HDR_RSRV, 0);
1828
1829         memcpy(lun, &nopout_hdr->lun, sizeof(struct scsi_lun));
1830         fw_nop_out->lun.lo = be32_to_cpu(lun[0]);
1831         fw_nop_out->lun.hi = be32_to_cpu(lun[1]);
1832
1833         qedi_update_itt_map(qedi, tid, task->itt, qedi_cmd);
1834
1835         if (nopout_hdr->ttt != ISCSI_TTT_ALL_ONES) {
1836                 fw_nop_out->itt = be32_to_cpu(nopout_hdr->itt);
1837                 fw_nop_out->ttt = be32_to_cpu(nopout_hdr->ttt);
1838                 fw_task_ctx->ystorm_st_context.state.buffer_offset[0] = 0;
1839                 fw_task_ctx->ystorm_st_context.state.local_comp = 1;
1840                 SET_FIELD(fw_task_ctx->ustorm_st_context.flags,
1841                           USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP, 1);
1842         } else {
1843                 fw_nop_out->itt = qedi_set_itt(tid, get_itt(task->itt));
1844                 fw_nop_out->ttt = ISCSI_TTT_ALL_ONES;
1845                 fw_task_ctx->ystorm_st_context.state.buffer_offset[0] = 0;
1846
1847                 spin_lock(&qedi_conn->list_lock);
1848                 list_add_tail(&qedi_cmd->io_cmd, &qedi_conn->active_cmd_list);
1849                 qedi_cmd->io_cmd_in_list = true;
1850                 qedi_conn->active_cmd_count++;
1851                 spin_unlock(&qedi_conn->list_lock);
1852         }
1853
1854         fw_nop_out->opcode = ISCSI_OPCODE_NOP_OUT;
1855         fw_nop_out->cmd_sn = be32_to_cpu(nopout_hdr->cmdsn);
1856         fw_nop_out->exp_stat_sn = be32_to_cpu(nopout_hdr->exp_statsn);
1857
1858         cached_sge =
1859                &fw_task_ctx->ystorm_st_context.state.sgl_ctx_union.cached_sge;
1860         cached_sge->sge.sge_len = req_sge->sge_len;
1861         cached_sge->sge.sge_addr.lo = (u32)(qedi_conn->gen_pdu.req_dma_addr);
1862         cached_sge->sge.sge_addr.hi =
1863                         (u32)((u64)qedi_conn->gen_pdu.req_dma_addr >> 32);
1864
1865         /* Mstorm context */
1866         fw_task_ctx->mstorm_st_context.task_type = ISCSI_TASK_TYPE_MIDPATH;
1867         fw_task_ctx->mstorm_ag_context.task_cid = (u16)qedi_conn->iscsi_conn_id;
1868
1869         single_sge = &fw_task_ctx->mstorm_st_context.sgl_union.single_sge;
1870         single_sge->sge_addr.lo = resp_sge->sge_addr.lo;
1871         single_sge->sge_addr.hi = resp_sge->sge_addr.hi;
1872         single_sge->sge_len = resp_sge->sge_len;
1873         fw_task_ctx->mstorm_st_context.rem_task_size = resp_sge->sge_len;
1874
1875         if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) {
1876                 ptu_invalidate = 1;
1877                 qedi->tid_reuse_count[tid] = 0;
1878         }
1879         fw_task_ctx->ystorm_st_context.state.reuse_count =
1880                                                 qedi->tid_reuse_count[tid];
1881         fw_task_ctx->mstorm_st_context.reuse_count =
1882                                                 qedi->tid_reuse_count[tid]++;
1883         /* Ustorm context */
1884         fw_task_ctx->ustorm_st_context.rem_rcv_len = resp_sge->sge_len;
1885         fw_task_ctx->ustorm_st_context.exp_data_transfer_len = data_len;
1886         fw_task_ctx->ustorm_st_context.exp_data_sn = 0;
1887         fw_task_ctx->ustorm_st_context.task_type =  ISCSI_TASK_TYPE_MIDPATH;
1888         fw_task_ctx->ustorm_st_context.cq_rss_number = 0;
1889
1890         SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map,
1891                   ISCSI_REG1_NUM_FAST_SGES, 0);
1892
1893         fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id;
1894         SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1,
1895                   USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1);
1896
1897         fw_task_ctx->ustorm_st_context.lun.lo = be32_to_cpu(lun[0]);
1898         fw_task_ctx->ustorm_st_context.lun.hi = be32_to_cpu(lun[1]);
1899
1900         qedi_add_to_sq(qedi_conn, task, tid, ptu_invalidate, false);
1901         qedi_ring_doorbell(qedi_conn);
1902         return 0;
1903 }
1904
1905 static int qedi_split_bd(struct qedi_cmd *cmd, u64 addr, int sg_len,
1906                          int bd_index)
1907 {
1908         struct iscsi_sge *bd = cmd->io_tbl.sge_tbl;
1909         int frag_size, sg_frags;
1910
1911         sg_frags = 0;
1912
1913         while (sg_len) {
1914                 if (addr % QEDI_PAGE_SIZE)
1915                         frag_size =
1916                                    (QEDI_PAGE_SIZE - (addr % QEDI_PAGE_SIZE));
1917                 else
1918                         frag_size = (sg_len > QEDI_BD_SPLIT_SZ) ? 0 :
1919                                     (sg_len % QEDI_BD_SPLIT_SZ);
1920
1921                 if (frag_size == 0)
1922                         frag_size = QEDI_BD_SPLIT_SZ;
1923
1924                 bd[bd_index + sg_frags].sge_addr.lo = (addr & 0xffffffff);
1925                 bd[bd_index + sg_frags].sge_addr.hi = (addr >> 32);
1926                 bd[bd_index + sg_frags].sge_len = (u16)frag_size;
1927                 QEDI_INFO(&cmd->conn->qedi->dbg_ctx, QEDI_LOG_IO,
1928                           "split sge %d: addr=%llx, len=%x",
1929                           (bd_index + sg_frags), addr, frag_size);
1930
1931                 addr += (u64)frag_size;
1932                 sg_frags++;
1933                 sg_len -= frag_size;
1934         }
1935         return sg_frags;
1936 }
1937
1938 static int qedi_map_scsi_sg(struct qedi_ctx *qedi, struct qedi_cmd *cmd)
1939 {
1940         struct scsi_cmnd *sc = cmd->scsi_cmd;
1941         struct iscsi_sge *bd = cmd->io_tbl.sge_tbl;
1942         struct scatterlist *sg;
1943         int byte_count = 0;
1944         int bd_count = 0;
1945         int sg_count;
1946         int sg_len;
1947         int sg_frags;
1948         u64 addr, end_addr;
1949         int i;
1950
1951         WARN_ON(scsi_sg_count(sc) > QEDI_ISCSI_MAX_BDS_PER_CMD);
1952
1953         sg_count = dma_map_sg(&qedi->pdev->dev, scsi_sglist(sc),
1954                               scsi_sg_count(sc), sc->sc_data_direction);
1955
1956         /*
1957          * New condition to send single SGE as cached-SGL.
1958          * Single SGE with length less than 64K.
1959          */
1960         sg = scsi_sglist(sc);
1961         if ((sg_count == 1) && (sg_dma_len(sg) <= MAX_SGLEN_FOR_CACHESGL)) {
1962                 sg_len = sg_dma_len(sg);
1963                 addr = (u64)sg_dma_address(sg);
1964
1965                 bd[bd_count].sge_addr.lo = (addr & 0xffffffff);
1966                 bd[bd_count].sge_addr.hi = (addr >> 32);
1967                 bd[bd_count].sge_len = (u16)sg_len;
1968
1969                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO,
1970                           "single-cashed-sgl: bd_count:%d addr=%llx, len=%x",
1971                           sg_count, addr, sg_len);
1972
1973                 return ++bd_count;
1974         }
1975
1976         scsi_for_each_sg(sc, sg, sg_count, i) {
1977                 sg_len = sg_dma_len(sg);
1978                 addr = (u64)sg_dma_address(sg);
1979                 end_addr = (addr + sg_len);
1980
1981                 /*
1982                  * first sg elem in the 'list',
1983                  * check if end addr is page-aligned.
1984                  */
1985                 if ((i == 0) && (sg_count > 1) && (end_addr % QEDI_PAGE_SIZE))
1986                         cmd->use_slowpath = true;
1987
1988                 /*
1989                  * last sg elem in the 'list',
1990                  * check if start addr is page-aligned.
1991                  */
1992                 else if ((i == (sg_count - 1)) &&
1993                          (sg_count > 1) && (addr % QEDI_PAGE_SIZE))
1994                         cmd->use_slowpath = true;
1995
1996                 /*
1997                  * middle sg elements in list,
1998                  * check if start and end addr is page-aligned
1999                  */
2000                 else if ((i != 0) && (i != (sg_count - 1)) &&
2001                          ((addr % QEDI_PAGE_SIZE) ||
2002                          (end_addr % QEDI_PAGE_SIZE)))
2003                         cmd->use_slowpath = true;
2004
2005                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO, "sg[%d] size=0x%x",
2006                           i, sg_len);
2007
2008                 if (sg_len > QEDI_BD_SPLIT_SZ) {
2009                         sg_frags = qedi_split_bd(cmd, addr, sg_len, bd_count);
2010                 } else {
2011                         sg_frags = 1;
2012                         bd[bd_count].sge_addr.lo = addr & 0xffffffff;
2013                         bd[bd_count].sge_addr.hi = addr >> 32;
2014                         bd[bd_count].sge_len = sg_len;
2015                 }
2016                 byte_count += sg_len;
2017                 bd_count += sg_frags;
2018         }
2019
2020         if (byte_count != scsi_bufflen(sc))
2021                 QEDI_ERR(&qedi->dbg_ctx,
2022                          "byte_count = %d != scsi_bufflen = %d\n", byte_count,
2023                          scsi_bufflen(sc));
2024         else
2025                 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO, "byte_count = %d\n",
2026                           byte_count);
2027
2028         WARN_ON(byte_count != scsi_bufflen(sc));
2029
2030         return bd_count;
2031 }
2032
2033 static void qedi_iscsi_map_sg_list(struct qedi_cmd *cmd)
2034 {
2035         int bd_count;
2036         struct scsi_cmnd *sc = cmd->scsi_cmd;
2037
2038         if (scsi_sg_count(sc)) {
2039                 bd_count  = qedi_map_scsi_sg(cmd->conn->qedi, cmd);
2040                 if (bd_count == 0)
2041                         return;
2042         } else {
2043                 struct iscsi_sge *bd = cmd->io_tbl.sge_tbl;
2044
2045                 bd[0].sge_addr.lo = 0;
2046                 bd[0].sge_addr.hi = 0;
2047                 bd[0].sge_len = 0;
2048                 bd_count = 0;
2049         }
2050         cmd->io_tbl.sge_valid = bd_count;
2051 }
2052
2053 static void qedi_cpy_scsi_cdb(struct scsi_cmnd *sc, u32 *dstp)
2054 {
2055         u32 dword;
2056         int lpcnt;
2057         u8 *srcp;
2058
2059         lpcnt = sc->cmd_len / sizeof(dword);
2060         srcp = (u8 *)sc->cmnd;
2061         while (lpcnt--) {
2062                 memcpy(&dword, (const void *)srcp, 4);
2063                 *dstp = cpu_to_be32(dword);
2064                 srcp += 4;
2065                 dstp++;
2066         }
2067         if (sc->cmd_len & 0x3) {
2068                 dword = (u32)srcp[0] | ((u32)srcp[1] << 8);
2069                 *dstp = cpu_to_be32(dword);
2070         }
2071 }
2072
2073 void qedi_trace_io(struct qedi_ctx *qedi, struct iscsi_task *task,
2074                    u16 tid, int8_t direction)
2075 {
2076         struct qedi_io_log *io_log;
2077         struct iscsi_conn *conn = task->conn;
2078         struct qedi_conn *qedi_conn = conn->dd_data;
2079         struct scsi_cmnd *sc_cmd = task->sc;
2080         unsigned long flags;
2081         u8 op;
2082
2083         spin_lock_irqsave(&qedi->io_trace_lock, flags);
2084
2085         io_log = &qedi->io_trace_buf[qedi->io_trace_idx];
2086         io_log->direction = direction;
2087         io_log->task_id = tid;
2088         io_log->cid = qedi_conn->iscsi_conn_id;
2089         io_log->lun = sc_cmd->device->lun;
2090         io_log->op = sc_cmd->cmnd[0];
2091         op = sc_cmd->cmnd[0];
2092         io_log->lba[0] = sc_cmd->cmnd[2];
2093         io_log->lba[1] = sc_cmd->cmnd[3];
2094         io_log->lba[2] = sc_cmd->cmnd[4];
2095         io_log->lba[3] = sc_cmd->cmnd[5];
2096         io_log->bufflen = scsi_bufflen(sc_cmd);
2097         io_log->sg_count = scsi_sg_count(sc_cmd);
2098         io_log->fast_sgs = qedi->fast_sgls;
2099         io_log->cached_sgs = qedi->cached_sgls;
2100         io_log->slow_sgs = qedi->slow_sgls;
2101         io_log->cached_sge = qedi->use_cached_sge;
2102         io_log->slow_sge = qedi->use_slow_sge;
2103         io_log->fast_sge = qedi->use_fast_sge;
2104         io_log->result = sc_cmd->result;
2105         io_log->jiffies = jiffies;
2106         io_log->blk_req_cpu = smp_processor_id();
2107
2108         if (direction == QEDI_IO_TRACE_REQ) {
2109                 /* For requests we only care about the submission CPU */
2110                 io_log->req_cpu = smp_processor_id() % qedi->num_queues;
2111                 io_log->intr_cpu = 0;
2112                 io_log->blk_rsp_cpu = 0;
2113         } else if (direction == QEDI_IO_TRACE_RSP) {
2114                 io_log->req_cpu = smp_processor_id() % qedi->num_queues;
2115                 io_log->intr_cpu = qedi->intr_cpu;
2116                 io_log->blk_rsp_cpu = smp_processor_id();
2117         }
2118
2119         qedi->io_trace_idx++;
2120         if (qedi->io_trace_idx == QEDI_IO_TRACE_SIZE)
2121                 qedi->io_trace_idx = 0;
2122
2123         qedi->use_cached_sge = false;
2124         qedi->use_slow_sge = false;
2125         qedi->use_fast_sge = false;
2126
2127         spin_unlock_irqrestore(&qedi->io_trace_lock, flags);
2128 }
2129
2130 int qedi_iscsi_send_ioreq(struct iscsi_task *task)
2131 {
2132         struct iscsi_conn *conn = task->conn;
2133         struct iscsi_session *session = conn->session;
2134         struct Scsi_Host *shost = iscsi_session_to_shost(session->cls_session);
2135         struct qedi_ctx *qedi = iscsi_host_priv(shost);
2136         struct qedi_conn *qedi_conn = conn->dd_data;
2137         struct qedi_cmd *cmd = task->dd_data;
2138         struct scsi_cmnd *sc = task->sc;
2139         struct iscsi_task_context *fw_task_ctx;
2140         struct iscsi_cached_sge_ctx *cached_sge;
2141         struct iscsi_phys_sgl_ctx *phys_sgl;
2142         struct iscsi_virt_sgl_ctx *virt_sgl;
2143         struct ystorm_iscsi_task_st_ctx *yst_cxt;
2144         struct mstorm_iscsi_task_st_ctx *mst_cxt;
2145         struct iscsi_sgl *sgl_struct;
2146         struct iscsi_sge *single_sge;
2147         struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr;
2148         struct iscsi_sge *bd = cmd->io_tbl.sge_tbl;
2149         enum iscsi_task_type task_type;
2150         struct iscsi_cmd_hdr *fw_cmd;
2151         u32 lun[2];
2152         u32 exp_data;
2153         u16 cq_idx = smp_processor_id() % qedi->num_queues;
2154         s16 ptu_invalidate = 0;
2155         s16 tid = 0;
2156         u8 num_fast_sgs;
2157
2158         tid = qedi_get_task_idx(qedi);
2159         if (tid == -1)
2160                 return -ENOMEM;
2161
2162         qedi_iscsi_map_sg_list(cmd);
2163
2164         int_to_scsilun(sc->device->lun, (struct scsi_lun *)lun);
2165         fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid);
2166
2167         memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context));
2168         cmd->task_id = tid;
2169
2170         /* Ystorm context */
2171         fw_cmd = &fw_task_ctx->ystorm_st_context.pdu_hdr.cmd;
2172         SET_FIELD(fw_cmd->flags_attr, ISCSI_CMD_HDR_ATTR, ISCSI_ATTR_SIMPLE);
2173
2174         if (sc->sc_data_direction == DMA_TO_DEVICE) {
2175                 if (conn->session->initial_r2t_en) {
2176                         exp_data = min((conn->session->imm_data_en *
2177                                         conn->max_xmit_dlength),
2178                                        conn->session->first_burst);
2179                         exp_data = min(exp_data, scsi_bufflen(sc));
2180                         fw_task_ctx->ustorm_ag_context.exp_data_acked =
2181                                                           cpu_to_le32(exp_data);
2182                 } else {
2183                         fw_task_ctx->ustorm_ag_context.exp_data_acked =
2184                               min(conn->session->first_burst, scsi_bufflen(sc));
2185                 }
2186
2187                 SET_FIELD(fw_cmd->flags_attr, ISCSI_CMD_HDR_WRITE, 1);
2188                 task_type = ISCSI_TASK_TYPE_INITIATOR_WRITE;
2189         } else {
2190                 if (scsi_bufflen(sc))
2191                         SET_FIELD(fw_cmd->flags_attr, ISCSI_CMD_HDR_READ, 1);
2192                 task_type = ISCSI_TASK_TYPE_INITIATOR_READ;
2193         }
2194
2195         fw_cmd->lun.lo = be32_to_cpu(lun[0]);
2196         fw_cmd->lun.hi = be32_to_cpu(lun[1]);
2197
2198         qedi_update_itt_map(qedi, tid, task->itt, cmd);
2199         fw_cmd->itt = qedi_set_itt(tid, get_itt(task->itt));
2200         fw_cmd->expected_transfer_length = scsi_bufflen(sc);
2201         fw_cmd->cmd_sn = be32_to_cpu(hdr->cmdsn);
2202         fw_cmd->opcode = hdr->opcode;
2203         qedi_cpy_scsi_cdb(sc, (u32 *)fw_cmd->cdb);
2204
2205         /* Mstorm context */
2206         fw_task_ctx->mstorm_st_context.sense_db.lo = (u32)cmd->sense_buffer_dma;
2207         fw_task_ctx->mstorm_st_context.sense_db.hi =
2208                                         (u32)((u64)cmd->sense_buffer_dma >> 32);
2209         fw_task_ctx->mstorm_ag_context.task_cid = qedi_conn->iscsi_conn_id;
2210         fw_task_ctx->mstorm_st_context.task_type = task_type;
2211
2212         if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) {
2213                 ptu_invalidate = 1;
2214                 qedi->tid_reuse_count[tid] = 0;
2215         }
2216         fw_task_ctx->ystorm_st_context.state.reuse_count =
2217                                                      qedi->tid_reuse_count[tid];
2218         fw_task_ctx->mstorm_st_context.reuse_count =
2219                                                    qedi->tid_reuse_count[tid]++;
2220
2221         /* Ustorm context */
2222         fw_task_ctx->ustorm_st_context.rem_rcv_len = scsi_bufflen(sc);
2223         fw_task_ctx->ustorm_st_context.exp_data_transfer_len = scsi_bufflen(sc);
2224         fw_task_ctx->ustorm_st_context.exp_data_sn =
2225                                                    be32_to_cpu(hdr->exp_statsn);
2226         fw_task_ctx->ustorm_st_context.task_type = task_type;
2227         fw_task_ctx->ustorm_st_context.cq_rss_number = cq_idx;
2228         fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id;
2229
2230         SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1,
2231                   USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1);
2232         SET_FIELD(fw_task_ctx->ustorm_st_context.flags,
2233                   USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP, 0);
2234
2235         num_fast_sgs = (cmd->io_tbl.sge_valid ?
2236                         min((u16)QEDI_FAST_SGE_COUNT,
2237                             (u16)cmd->io_tbl.sge_valid) : 0);
2238         SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map,
2239                   ISCSI_REG1_NUM_FAST_SGES, num_fast_sgs);
2240
2241         fw_task_ctx->ustorm_st_context.lun.lo = be32_to_cpu(lun[0]);
2242         fw_task_ctx->ustorm_st_context.lun.hi = be32_to_cpu(lun[1]);
2243
2244         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO, "Total sge count [%d]\n",
2245                   cmd->io_tbl.sge_valid);
2246
2247         yst_cxt = &fw_task_ctx->ystorm_st_context;
2248         mst_cxt = &fw_task_ctx->mstorm_st_context;
2249         /* Tx path */
2250         if (task_type == ISCSI_TASK_TYPE_INITIATOR_WRITE) {
2251                 /* not considering  superIO or FastIO */
2252                 if (cmd->io_tbl.sge_valid == 1) {
2253                         cached_sge = &yst_cxt->state.sgl_ctx_union.cached_sge;
2254                         cached_sge->sge.sge_addr.lo = bd[0].sge_addr.lo;
2255                         cached_sge->sge.sge_addr.hi = bd[0].sge_addr.hi;
2256                         cached_sge->sge.sge_len = bd[0].sge_len;
2257                         qedi->cached_sgls++;
2258                 } else if ((cmd->io_tbl.sge_valid != 1) && cmd->use_slowpath) {
2259                         SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
2260                                   ISCSI_MFLAGS_SLOW_IO, 1);
2261                         SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map,
2262                                   ISCSI_REG1_NUM_FAST_SGES, 0);
2263                         phys_sgl = &yst_cxt->state.sgl_ctx_union.phys_sgl;
2264                         phys_sgl->sgl_base.lo = (u32)(cmd->io_tbl.sge_tbl_dma);
2265                         phys_sgl->sgl_base.hi =
2266                                      (u32)((u64)cmd->io_tbl.sge_tbl_dma >> 32);
2267                         phys_sgl->sgl_size = cmd->io_tbl.sge_valid;
2268                         qedi->slow_sgls++;
2269                 } else if ((cmd->io_tbl.sge_valid != 1) && !cmd->use_slowpath) {
2270                         SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
2271                                   ISCSI_MFLAGS_SLOW_IO, 0);
2272                         SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map,
2273                                   ISCSI_REG1_NUM_FAST_SGES,
2274                                   min((u16)QEDI_FAST_SGE_COUNT,
2275                                       (u16)cmd->io_tbl.sge_valid));
2276                         virt_sgl = &yst_cxt->state.sgl_ctx_union.virt_sgl;
2277                         virt_sgl->sgl_base.lo = (u32)(cmd->io_tbl.sge_tbl_dma);
2278                         virt_sgl->sgl_base.hi =
2279                                       (u32)((u64)cmd->io_tbl.sge_tbl_dma >> 32);
2280                         virt_sgl->sgl_initial_offset =
2281                                  (u32)bd[0].sge_addr.lo & (QEDI_PAGE_SIZE - 1);
2282                         qedi->fast_sgls++;
2283                 }
2284                 fw_task_ctx->mstorm_st_context.sgl_size = cmd->io_tbl.sge_valid;
2285                 fw_task_ctx->mstorm_st_context.rem_task_size = scsi_bufflen(sc);
2286         } else {
2287         /* Rx path */
2288                 if (cmd->io_tbl.sge_valid == 1) {
2289                         SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
2290                                   ISCSI_MFLAGS_SLOW_IO, 0);
2291                         SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
2292                                   ISCSI_MFLAGS_SINGLE_SGE, 1);
2293                         single_sge = &mst_cxt->sgl_union.single_sge;
2294                         single_sge->sge_addr.lo = bd[0].sge_addr.lo;
2295                         single_sge->sge_addr.hi = bd[0].sge_addr.hi;
2296                         single_sge->sge_len = bd[0].sge_len;
2297                         qedi->cached_sgls++;
2298                 } else if ((cmd->io_tbl.sge_valid != 1) && cmd->use_slowpath) {
2299                         sgl_struct = &mst_cxt->sgl_union.sgl_struct;
2300                         sgl_struct->sgl_addr.lo =
2301                                                 (u32)(cmd->io_tbl.sge_tbl_dma);
2302                         sgl_struct->sgl_addr.hi =
2303                                      (u32)((u64)cmd->io_tbl.sge_tbl_dma >> 32);
2304                         SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
2305                                   ISCSI_MFLAGS_SLOW_IO, 1);
2306                         SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map,
2307                                   ISCSI_REG1_NUM_FAST_SGES, 0);
2308                         sgl_struct->updated_sge_size = 0;
2309                         sgl_struct->updated_sge_offset = 0;
2310                         qedi->slow_sgls++;
2311                 } else if ((cmd->io_tbl.sge_valid != 1) && !cmd->use_slowpath) {
2312                         sgl_struct = &mst_cxt->sgl_union.sgl_struct;
2313                         sgl_struct->sgl_addr.lo =
2314                                                 (u32)(cmd->io_tbl.sge_tbl_dma);
2315                         sgl_struct->sgl_addr.hi =
2316                                      (u32)((u64)cmd->io_tbl.sge_tbl_dma >> 32);
2317                         sgl_struct->byte_offset =
2318                                 (u32)bd[0].sge_addr.lo & (QEDI_PAGE_SIZE - 1);
2319                         SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags,
2320                                   ISCSI_MFLAGS_SLOW_IO, 0);
2321                         SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map,
2322                                   ISCSI_REG1_NUM_FAST_SGES, 0);
2323                         sgl_struct->updated_sge_size = 0;
2324                         sgl_struct->updated_sge_offset = 0;
2325                         qedi->fast_sgls++;
2326                 }
2327                 fw_task_ctx->mstorm_st_context.sgl_size = cmd->io_tbl.sge_valid;
2328                 fw_task_ctx->mstorm_st_context.rem_task_size = scsi_bufflen(sc);
2329         }
2330
2331         if (cmd->io_tbl.sge_valid == 1)
2332                 /* Singel-SGL */
2333                 qedi->use_cached_sge = true;
2334         else {
2335                 if (cmd->use_slowpath)
2336                         qedi->use_slow_sge = true;
2337                 else
2338                         qedi->use_fast_sge = true;
2339         }
2340         QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO,
2341                   "%s: %s-SGL: num_sges=0x%x first-sge-lo=0x%x first-sge-hi=0x%x",
2342                   (task_type == ISCSI_TASK_TYPE_INITIATOR_WRITE) ?
2343                   "Write " : "Read ", (cmd->io_tbl.sge_valid == 1) ?
2344                   "Single" : (cmd->use_slowpath ? "SLOW" : "FAST"),
2345                   (u16)cmd->io_tbl.sge_valid, (u32)(cmd->io_tbl.sge_tbl_dma),
2346                   (u32)((u64)cmd->io_tbl.sge_tbl_dma >> 32));
2347
2348         /*  Add command in active command list */
2349         spin_lock(&qedi_conn->list_lock);
2350         list_add_tail(&cmd->io_cmd, &qedi_conn->active_cmd_list);
2351         cmd->io_cmd_in_list = true;
2352         qedi_conn->active_cmd_count++;
2353         spin_unlock(&qedi_conn->list_lock);
2354
2355         qedi_add_to_sq(qedi_conn, task, tid, ptu_invalidate, false);
2356         qedi_ring_doorbell(qedi_conn);
2357         if (qedi_io_tracing)
2358                 qedi_trace_io(qedi, task, tid, QEDI_IO_TRACE_REQ);
2359
2360         return 0;
2361 }
2362
2363 int qedi_iscsi_cleanup_task(struct iscsi_task *task, bool mark_cmd_node_deleted)
2364 {
2365         struct iscsi_conn *conn = task->conn;
2366         struct qedi_conn *qedi_conn = conn->dd_data;
2367         struct qedi_cmd *cmd = task->dd_data;
2368         s16 ptu_invalidate = 0;
2369
2370         QEDI_INFO(&qedi_conn->qedi->dbg_ctx, QEDI_LOG_SCSI_TM,
2371                   "issue cleanup tid=0x%x itt=0x%x task_state=%d cmd_state=0%x cid=0x%x\n",
2372                   cmd->task_id, get_itt(task->itt), task->state,
2373                   cmd->state, qedi_conn->iscsi_conn_id);
2374
2375         qedi_add_to_sq(qedi_conn, task, cmd->task_id, ptu_invalidate, true);
2376         qedi_ring_doorbell(qedi_conn);
2377
2378         return 0;
2379 }