drivers/scsi/qedi/qedi_fw_api.c

   1 /* QLogic iSCSI Offload Driver
   2  * Copyright (c) 2016 Cavium Inc.
   3  *
   4  * This software is available under the terms of the GNU General Public License
   5  * (GPL) Version 2, available from the file COPYING in the main directory of
   6  * this source tree.
   7  */
   8
   9 #include <linux/types.h>
  10 #include <asm/byteorder.h>
  11 #include "qedi_hsi.h"
  12 #include <linux/qed/qed_if.h>
  13
  14 #include "qedi_fw_iscsi.h"
  15 #include "qedi_fw_scsi.h"
  16
  17 #define SCSI_NUM_SGES_IN_CACHE 0x4
  18
  19 static bool scsi_is_slow_sgl(u16 num_sges, bool small_mid_sge)
  20 {
  21         return (num_sges > SCSI_NUM_SGES_SLOW_SGL_THR && small_mid_sge);
  22 }
  23
  24 static
  25 void init_scsi_sgl_context(struct scsi_sgl_params *ctx_sgl_params,
  26                            struct scsi_cached_sges *ctx_data_desc,
  27                            struct scsi_sgl_task_params *sgl_task_params)
  28 {
  29         u8 sge_index;
  30         u8 num_sges;
  31         u32 val;
  32
  33         num_sges = (sgl_task_params->num_sges > SCSI_NUM_SGES_IN_CACHE) ?
  34                              SCSI_NUM_SGES_IN_CACHE : sgl_task_params->num_sges;
  35
  36         /* sgl params */
  37         val = cpu_to_le32(sgl_task_params->sgl_phys_addr.lo);
  38         ctx_sgl_params->sgl_addr.lo = val;
  39         val = cpu_to_le32(sgl_task_params->sgl_phys_addr.hi);
  40         ctx_sgl_params->sgl_addr.hi = val;
  41         val = cpu_to_le32(sgl_task_params->total_buffer_size);
  42         ctx_sgl_params->sgl_total_length = val;
  43         ctx_sgl_params->sgl_num_sges = cpu_to_le16(sgl_task_params->num_sges);
  44
  45         for (sge_index = 0; sge_index < num_sges; sge_index++) {
  46                 val = cpu_to_le32(sgl_task_params->sgl[sge_index].sge_addr.lo);
  47                 ctx_data_desc->sge[sge_index].sge_addr.lo = val;
  48                 val = cpu_to_le32(sgl_task_params->sgl[sge_index].sge_addr.hi);
  49                 ctx_data_desc->sge[sge_index].sge_addr.hi = val;
  50                 val = cpu_to_le32(sgl_task_params->sgl[sge_index].sge_len);
  51                 ctx_data_desc->sge[sge_index].sge_len = val;
  52         }
  53 }
  54
  55 static u32 calc_rw_task_size(struct iscsi_task_params *task_params,
  56                              enum iscsi_task_type task_type,
  57                              struct scsi_sgl_task_params *sgl_task_params,
  58                              struct scsi_dif_task_params *dif_task_params)
  59 {
  60         u32 io_size;
  61
  62         if (task_type == ISCSI_TASK_TYPE_INITIATOR_WRITE ||
  63             task_type == ISCSI_TASK_TYPE_TARGET_READ)
  64                 io_size = task_params->tx_io_size;
  65         else
  66                 io_size = task_params->rx_io_size;
  67
  68         if (!io_size)
  69                 return 0;
  70
  71         if (!dif_task_params)
  72                 return io_size;
  73
  74         return !dif_task_params->dif_on_network ?
  75                io_size : sgl_task_params->total_buffer_size;
  76 }
  77
  78 static void
  79 init_dif_context_flags(struct iscsi_dif_flags *ctx_dif_flags,
  80                        struct scsi_dif_task_params *dif_task_params)
  81 {
  82         if (!dif_task_params)
  83                 return;
  84
  85         SET_FIELD(ctx_dif_flags->flags, ISCSI_DIF_FLAGS_PROT_INTERVAL_SIZE_LOG,
  86                   dif_task_params->dif_block_size_log);
  87         SET_FIELD(ctx_dif_flags->flags, ISCSI_DIF_FLAGS_DIF_TO_PEER,
  88                   dif_task_params->dif_on_network ? 1 : 0);
  89         SET_FIELD(ctx_dif_flags->flags, ISCSI_DIF_FLAGS_HOST_INTERFACE,
  90                   dif_task_params->dif_on_host ? 1 : 0);
  91 }
  92
  93 static void init_sqe(struct iscsi_task_params *task_params,
  94                      struct scsi_sgl_task_params *sgl_task_params,
  95                      struct scsi_dif_task_params *dif_task_params,
  96                      struct iscsi_common_hdr *pdu_header,
  97                      struct scsi_initiator_cmd_params *cmd_params,
  98                      enum iscsi_task_type task_type,
  99                      bool is_cleanup)
 100 {
 101         if (!task_params->sqe)
 102                 return;
 103
 104         memset(task_params->sqe, 0, sizeof(*task_params->sqe));
 105         task_params->sqe->task_id = cpu_to_le16(task_params->itid);
 106         if (is_cleanup) {
 107                 SET_FIELD(task_params->sqe->flags, ISCSI_WQE_WQE_TYPE,
 108                           ISCSI_WQE_TYPE_TASK_CLEANUP);
 109                 return;
 110         }
 111
 112         switch (task_type) {
 113         case ISCSI_TASK_TYPE_INITIATOR_WRITE:
 114         {
 115                 u32 buf_size = 0;
 116                 u32 num_sges = 0;
 117
 118                 init_dif_context_flags(&task_params->sqe->prot_flags,
 119                                        dif_task_params);
 120
 121                 SET_FIELD(task_params->sqe->flags, ISCSI_WQE_WQE_TYPE,
 122                           ISCSI_WQE_TYPE_NORMAL);
 123
 124                 if (task_params->tx_io_size) {
 125                         buf_size = calc_rw_task_size(task_params, task_type,
 126                                                      sgl_task_params,
 127                                                      dif_task_params);
 128
 129                 if (scsi_is_slow_sgl(sgl_task_params->num_sges,
 130                                      sgl_task_params->small_mid_sge))
 131                         num_sges = ISCSI_WQE_NUM_SGES_SLOWIO;
 132                 else
 133                         num_sges = min(sgl_task_params->num_sges,
 134                                        (u16)SCSI_NUM_SGES_SLOW_SGL_THR);
 135         }
 136
 137         SET_FIELD(task_params->sqe->flags, ISCSI_WQE_NUM_SGES, num_sges);
 138         SET_FIELD(task_params->sqe->contlen_cdbsize, ISCSI_WQE_CONT_LEN,
 139                   buf_size);
 140
 141         if (GET_FIELD(pdu_header->hdr_second_dword,
 142                       ISCSI_CMD_HDR_TOTAL_AHS_LEN))
 143                 SET_FIELD(task_params->sqe->contlen_cdbsize, ISCSI_WQE_CDB_SIZE,
 144                           cmd_params->extended_cdb_sge.sge_len);
 145         }
 146                 break;
 147         case ISCSI_TASK_TYPE_INITIATOR_READ:
 148                 SET_FIELD(task_params->sqe->flags, ISCSI_WQE_WQE_TYPE,
 149                           ISCSI_WQE_TYPE_NORMAL);
 150
 151                 if (GET_FIELD(pdu_header->hdr_second_dword,
 152                               ISCSI_CMD_HDR_TOTAL_AHS_LEN))
 153                         SET_FIELD(task_params->sqe->contlen_cdbsize,
 154                                   ISCSI_WQE_CDB_SIZE,
 155                                   cmd_params->extended_cdb_sge.sge_len);
 156                 break;
 157         case ISCSI_TASK_TYPE_LOGIN_RESPONSE:
 158         case ISCSI_TASK_TYPE_MIDPATH:
 159         {
 160                 bool advance_statsn = true;
 161
 162                 if (task_type == ISCSI_TASK_TYPE_LOGIN_RESPONSE)
 163                         SET_FIELD(task_params->sqe->flags, ISCSI_WQE_WQE_TYPE,
 164                                   ISCSI_WQE_TYPE_LOGIN);
 165                 else
 166                         SET_FIELD(task_params->sqe->flags, ISCSI_WQE_WQE_TYPE,
 167                                   ISCSI_WQE_TYPE_MIDDLE_PATH);
 168
 169                 if (task_type == ISCSI_TASK_TYPE_MIDPATH) {
 170                         u8 opcode = GET_FIELD(pdu_header->hdr_first_byte,
 171                                               ISCSI_COMMON_HDR_OPCODE);
 172
 173                         if (opcode != ISCSI_OPCODE_TEXT_RESPONSE &&
 174                             (opcode != ISCSI_OPCODE_NOP_IN ||
 175                             pdu_header->itt == ISCSI_TTT_ALL_ONES))
 176                                 advance_statsn = false;
 177                 }
 178
 179                 SET_FIELD(task_params->sqe->flags, ISCSI_WQE_RESPONSE,
 180                           advance_statsn ? 1 : 0);
 181
 182                 if (task_params->tx_io_size) {
 183                         SET_FIELD(task_params->sqe->contlen_cdbsize,
 184                                   ISCSI_WQE_CONT_LEN, task_params->tx_io_size);
 185
 186                 if (scsi_is_slow_sgl(sgl_task_params->num_sges,
 187                                      sgl_task_params->small_mid_sge))
 188                         SET_FIELD(task_params->sqe->flags, ISCSI_WQE_NUM_SGES,
 189                                   ISCSI_WQE_NUM_SGES_SLOWIO);
 190                 else
 191                         SET_FIELD(task_params->sqe->flags, ISCSI_WQE_NUM_SGES,
 192                                   min(sgl_task_params->num_sges,
 193                                       (u16)SCSI_NUM_SGES_SLOW_SGL_THR));
 194                 }
 195         }
 196                 break;
 197         default:
 198                 break;
 199         }
 200 }
 201
 202 static void init_default_iscsi_task(struct iscsi_task_params *task_params,
 203                                     struct data_hdr *pdu_header,
 204                                     enum iscsi_task_type task_type)
 205 {
 206         struct iscsi_task_context *context;
 207         u16 index;
 208         u32 val;
 209
 210         context = task_params->context;
 211         memset(context, 0, sizeof(*context));
 212
 213         for (index = 0; index <
 214              ARRAY_SIZE(context->ystorm_st_context.pdu_hdr.data.data);
 215              index++) {
 216                 val = cpu_to_le32(pdu_header->data[index]);
 217                 context->ystorm_st_context.pdu_hdr.data.data[index] = val;
 218         }
 219
 220         context->mstorm_st_context.task_type = task_type;
 221         context->mstorm_ag_context.task_cid =
 222                                             cpu_to_le16(task_params->conn_icid);
 223
 224         SET_FIELD(context->ustorm_ag_context.flags1,
 225                   USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1);
 226
 227         context->ustorm_st_context.task_type = task_type;
 228         context->ustorm_st_context.cq_rss_number = task_params->cq_rss_number;
 229         context->ustorm_ag_context.icid = cpu_to_le16(task_params->conn_icid);
 230 }
 231
 232 static
 233 void init_initiator_rw_cdb_ystorm_context(struct ystorm_iscsi_task_st_ctx *ystc,
 234                                           struct scsi_initiator_cmd_params *cmd)
 235 {
 236         union iscsi_task_hdr *ctx_pdu_hdr = &ystc->pdu_hdr;
 237         u32 val;
 238
 239         if (!cmd->extended_cdb_sge.sge_len)
 240                 return;
 241
 242         SET_FIELD(ctx_pdu_hdr->ext_cdb_cmd.hdr_second_dword,
 243                   ISCSI_EXT_CDB_CMD_HDR_CDB_SIZE,
 244                   cmd->extended_cdb_sge.sge_len);
 245         val = cpu_to_le32(cmd->extended_cdb_sge.sge_addr.lo);
 246         ctx_pdu_hdr->ext_cdb_cmd.cdb_sge.sge_addr.lo = val;
 247         val = cpu_to_le32(cmd->extended_cdb_sge.sge_addr.hi);
 248         ctx_pdu_hdr->ext_cdb_cmd.cdb_sge.sge_addr.hi = val;
 249         val = cpu_to_le32(cmd->extended_cdb_sge.sge_len);
 250         ctx_pdu_hdr->ext_cdb_cmd.cdb_sge.sge_len  = val;
 251 }
 252
 253 static
 254 void init_ustorm_task_contexts(struct ustorm_iscsi_task_st_ctx *ustorm_st_cxt,
 255                                struct ustorm_iscsi_task_ag_ctx *ustorm_ag_cxt,
 256                                u32 remaining_recv_len,
 257                                u32 expected_data_transfer_len,
 258                                u8 num_sges, bool tx_dif_conn_err_en)
 259 {
 260         u32 val;
 261
 262         ustorm_st_cxt->rem_rcv_len = cpu_to_le32(remaining_recv_len);
 263         ustorm_ag_cxt->exp_data_acked = cpu_to_le32(expected_data_transfer_len);
 264         val = cpu_to_le32(expected_data_transfer_len);
 265         ustorm_st_cxt->exp_data_transfer_len = val;
 266         SET_FIELD(ustorm_st_cxt->reg1.reg1_map, ISCSI_REG1_NUM_SGES, num_sges);
 267         SET_FIELD(ustorm_ag_cxt->flags2,
 268                   USTORM_ISCSI_TASK_AG_CTX_DIF_ERROR_CF_EN,
 269                   tx_dif_conn_err_en ? 1 : 0);
 270 }
 271
 272 static
 273 void set_rw_exp_data_acked_and_cont_len(struct iscsi_task_context *context,
 274                                         struct iscsi_conn_params  *conn_params,
 275                                         enum iscsi_task_type task_type,
 276                                         u32 task_size,
 277                                         u32 exp_data_transfer_len,
 278                                         u8 total_ahs_length)
 279 {
 280         u32 max_unsolicited_data = 0, val;
 281
 282         if (total_ahs_length &&
 283             (task_type == ISCSI_TASK_TYPE_INITIATOR_WRITE ||
 284              task_type == ISCSI_TASK_TYPE_INITIATOR_READ))
 285                 SET_FIELD(context->ustorm_st_context.flags2,
 286                           USTORM_ISCSI_TASK_ST_CTX_AHS_EXIST, 1);
 287
 288         switch (task_type) {
 289         case ISCSI_TASK_TYPE_INITIATOR_WRITE:
 290                 if (!conn_params->initial_r2t)
 291                         max_unsolicited_data = conn_params->first_burst_length;
 292                 else if (conn_params->immediate_data)
 293                         max_unsolicited_data =
 294                                           min(conn_params->first_burst_length,
 295                                               conn_params->max_send_pdu_length);
 296
 297                 context->ustorm_ag_context.exp_data_acked =
 298                                    cpu_to_le32(total_ahs_length == 0 ?
 299                                                 min(exp_data_transfer_len,
 300                                                     max_unsolicited_data) :
 301                                                 ((u32)(total_ahs_length +
 302                                                        ISCSI_AHS_CNTL_SIZE)));
 303                 break;
 304         case ISCSI_TASK_TYPE_TARGET_READ:
 305                 val = cpu_to_le32(exp_data_transfer_len);
 306                 context->ustorm_ag_context.exp_data_acked = val;
 307                 break;
 308         case ISCSI_TASK_TYPE_INITIATOR_READ:
 309                 context->ustorm_ag_context.exp_data_acked =
 310                                         cpu_to_le32((total_ahs_length == 0 ? 0 :
 311                                                      total_ahs_length +
 312                                                      ISCSI_AHS_CNTL_SIZE));
 313                 break;
 314         case ISCSI_TASK_TYPE_TARGET_WRITE:
 315                 val = cpu_to_le32(task_size);
 316                 context->ustorm_ag_context.exp_cont_len = val;
 317                 break;
 318         default:
 319                 break;
 320         }
 321 }
 322
 323 static
 324 void init_rtdif_task_context(struct rdif_task_context *rdif_context,
 325                              struct tdif_task_context *tdif_context,
 326                              struct scsi_dif_task_params *dif_task_params,
 327                              enum iscsi_task_type task_type)
 328 {
 329         u32 val;
 330
 331         if (!dif_task_params->dif_on_network || !dif_task_params->dif_on_host)
 332                 return;
 333
 334         if (task_type == ISCSI_TASK_TYPE_TARGET_WRITE ||
 335             task_type == ISCSI_TASK_TYPE_INITIATOR_READ) {
 336                 rdif_context->app_tag_value =
 337                                   cpu_to_le16(dif_task_params->application_tag);
 338                 rdif_context->partial_crc_value = cpu_to_le16(0xffff);
 339                 val = cpu_to_le32(dif_task_params->initial_ref_tag);
 340                 rdif_context->initial_ref_tag = val;
 341                 rdif_context->app_tag_mask =
 342                              cpu_to_le16(dif_task_params->application_tag_mask);
 343                 SET_FIELD(rdif_context->flags0, RDIF_TASK_CONTEXT_CRC_SEED,
 344                           dif_task_params->crc_seed ? 1 : 0);
 345                 SET_FIELD(rdif_context->flags0, RDIF_TASK_CONTEXT_HOSTGUARDTYPE,
 346                           dif_task_params->host_guard_type);
 347                 SET_FIELD(rdif_context->flags0,
 348                           RDIF_TASK_CONTEXT_PROTECTIONTYPE,
 349                           dif_task_params->protection_type);
 350                 SET_FIELD(rdif_context->flags0,
 351                           RDIF_TASK_CONTEXT_INITIALREFTAGVALID, 1);
 352                 SET_FIELD(rdif_context->flags0,
 353                           RDIF_TASK_CONTEXT_KEEPREFTAGCONST,
 354                           dif_task_params->keep_ref_tag_const ? 1 : 0);
 355                 SET_FIELD(rdif_context->flags1,
 356                           RDIF_TASK_CONTEXT_VALIDATEAPPTAG,
 357                           (dif_task_params->validate_app_tag &&
 358                           dif_task_params->dif_on_network) ? 1 : 0);
 359                 SET_FIELD(rdif_context->flags1,
 360                           RDIF_TASK_CONTEXT_VALIDATEGUARD,
 361                           (dif_task_params->validate_guard &&
 362                           dif_task_params->dif_on_network) ? 1 : 0);
 363                 SET_FIELD(rdif_context->flags1,
 364                           RDIF_TASK_CONTEXT_VALIDATEREFTAG,
 365                           (dif_task_params->validate_ref_tag &&
 366                           dif_task_params->dif_on_network) ? 1 : 0);
 367                 SET_FIELD(rdif_context->flags1,
 368                           RDIF_TASK_CONTEXT_HOSTINTERFACE,
 369                           dif_task_params->dif_on_host ? 1 : 0);
 370                 SET_FIELD(rdif_context->flags1,
 371                           RDIF_TASK_CONTEXT_NETWORKINTERFACE,
 372                           dif_task_params->dif_on_network ? 1 : 0);
 373                 SET_FIELD(rdif_context->flags1,
 374                           RDIF_TASK_CONTEXT_FORWARDGUARD,
 375                           dif_task_params->forward_guard ? 1 : 0);
 376                 SET_FIELD(rdif_context->flags1,
 377                           RDIF_TASK_CONTEXT_FORWARDAPPTAG,
 378                           dif_task_params->forward_app_tag ? 1 : 0);
 379                 SET_FIELD(rdif_context->flags1,
 380                           RDIF_TASK_CONTEXT_FORWARDREFTAG,
 381                           dif_task_params->forward_ref_tag ? 1 : 0);
 382                 SET_FIELD(rdif_context->flags1,
 383                           RDIF_TASK_CONTEXT_FORWARDAPPTAGWITHMASK,
 384                           dif_task_params->forward_app_tag_with_mask ? 1 : 0);
 385                 SET_FIELD(rdif_context->flags1,
 386                           RDIF_TASK_CONTEXT_FORWARDREFTAGWITHMASK,
 387                           dif_task_params->forward_ref_tag_with_mask ? 1 : 0);
 388                 SET_FIELD(rdif_context->flags1,
 389                           RDIF_TASK_CONTEXT_INTERVALSIZE,
 390                           dif_task_params->dif_block_size_log - 9);
 391                 SET_FIELD(rdif_context->state,
 392                           RDIF_TASK_CONTEXT_REFTAGMASK,
 393                           dif_task_params->ref_tag_mask);
 394                 SET_FIELD(rdif_context->state, RDIF_TASK_CONTEXT_IGNOREAPPTAG,
 395                           dif_task_params->ignore_app_tag);
 396         }
 397
 398         if (task_type == ISCSI_TASK_TYPE_TARGET_READ ||
 399             task_type == ISCSI_TASK_TYPE_INITIATOR_WRITE) {
 400                 tdif_context->app_tag_value =
 401                                   cpu_to_le16(dif_task_params->application_tag);
 402                 tdif_context->partial_crc_valueB =
 403                        cpu_to_le16(dif_task_params->crc_seed ? 0xffff : 0x0000);
 404                 tdif_context->partial_crc_value_a =
 405                        cpu_to_le16(dif_task_params->crc_seed ? 0xffff : 0x0000);
 406                 SET_FIELD(tdif_context->flags0, TDIF_TASK_CONTEXT_CRC_SEED,
 407                           dif_task_params->crc_seed ? 1 : 0);
 408
 409                 SET_FIELD(tdif_context->flags0,
 410                           TDIF_TASK_CONTEXT_SETERRORWITHEOP,
 411                           dif_task_params->tx_dif_conn_err_en ? 1 : 0);
 412                 SET_FIELD(tdif_context->flags1, TDIF_TASK_CONTEXT_FORWARDGUARD,
 413                           dif_task_params->forward_guard   ? 1 : 0);
 414                 SET_FIELD(tdif_context->flags1, TDIF_TASK_CONTEXT_FORWARDAPPTAG,
 415                           dif_task_params->forward_app_tag ? 1 : 0);
 416                 SET_FIELD(tdif_context->flags1, TDIF_TASK_CONTEXT_FORWARDREFTAG,
 417                           dif_task_params->forward_ref_tag ? 1 : 0);
 418                 SET_FIELD(tdif_context->flags1, TDIF_TASK_CONTEXT_INTERVALSIZE,
 419                           dif_task_params->dif_block_size_log - 9);
 420                 SET_FIELD(tdif_context->flags1, TDIF_TASK_CONTEXT_HOSTINTERFACE,
 421                           dif_task_params->dif_on_host    ? 1 : 0);
 422                 SET_FIELD(tdif_context->flags1,
 423                           TDIF_TASK_CONTEXT_NETWORKINTERFACE,
 424                           dif_task_params->dif_on_network ? 1 : 0);
 425                 val = cpu_to_le32(dif_task_params->initial_ref_tag);
 426                 tdif_context->initial_ref_tag = val;
 427                 tdif_context->app_tag_mask =
 428                              cpu_to_le16(dif_task_params->application_tag_mask);
 429                 SET_FIELD(tdif_context->flags0,
 430                           TDIF_TASK_CONTEXT_HOSTGUARDTYPE,
 431                           dif_task_params->host_guard_type);
 432                 SET_FIELD(tdif_context->flags0,
 433                           TDIF_TASK_CONTEXT_PROTECTIONTYPE,
 434                           dif_task_params->protection_type);
 435                 SET_FIELD(tdif_context->flags0,
 436                           TDIF_TASK_CONTEXT_INITIALREFTAGVALID,
 437                           dif_task_params->initial_ref_tag_is_valid ? 1 : 0);
 438                 SET_FIELD(tdif_context->flags0,
 439                           TDIF_TASK_CONTEXT_KEEPREFTAGCONST,
 440                           dif_task_params->keep_ref_tag_const ? 1 : 0);
 441                 SET_FIELD(tdif_context->flags1, TDIF_TASK_CONTEXT_VALIDATEGUARD,
 442                           (dif_task_params->validate_guard &&
 443                            dif_task_params->dif_on_host) ? 1 : 0);
 444                 SET_FIELD(tdif_context->flags1,
 445                           TDIF_TASK_CONTEXT_VALIDATEAPPTAG,
 446                           (dif_task_params->validate_app_tag &&
 447                           dif_task_params->dif_on_host) ? 1 : 0);
 448                 SET_FIELD(tdif_context->flags1,
 449                           TDIF_TASK_CONTEXT_VALIDATEREFTAG,
 450                           (dif_task_params->validate_ref_tag &&
 451                            dif_task_params->dif_on_host) ? 1 : 0);
 452                 SET_FIELD(tdif_context->flags1,
 453                           TDIF_TASK_CONTEXT_FORWARDAPPTAGWITHMASK,
 454                           dif_task_params->forward_app_tag_with_mask ? 1 : 0);
 455                 SET_FIELD(tdif_context->flags1,
 456                           TDIF_TASK_CONTEXT_FORWARDREFTAGWITHMASK,
 457                           dif_task_params->forward_ref_tag_with_mask ? 1 : 0);
 458                 SET_FIELD(tdif_context->flags1,
 459                           TDIF_TASK_CONTEXT_REFTAGMASK,
 460                           dif_task_params->ref_tag_mask);
 461                 SET_FIELD(tdif_context->flags0,
 462                           TDIF_TASK_CONTEXT_IGNOREAPPTAG,
 463                           dif_task_params->ignore_app_tag ? 1 : 0);
 464         }
 465 }
 466
 467 static void set_local_completion_context(struct iscsi_task_context *context)
 468 {
 469         SET_FIELD(context->ystorm_st_context.state.flags,
 470                   YSTORM_ISCSI_TASK_STATE_LOCAL_COMP, 1);
 471         SET_FIELD(context->ustorm_st_context.flags,
 472                   USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP, 1);
 473 }
 474
 475 static int init_rw_iscsi_task(struct iscsi_task_params *task_params,
 476                               enum iscsi_task_type task_type,
 477                               struct iscsi_conn_params *conn_params,
 478                               struct iscsi_common_hdr *pdu_header,
 479                               struct scsi_sgl_task_params *sgl_task_params,
 480                               struct scsi_initiator_cmd_params *cmd_params,
 481                               struct scsi_dif_task_params *dif_task_params)
 482 {
 483         u32 exp_data_transfer_len = conn_params->max_burst_length;
 484         struct iscsi_task_context *cxt;
 485         bool slow_io = false;
 486         u32 task_size, val;
 487         u8 num_sges = 0;
 488
 489         task_size = calc_rw_task_size(task_params, task_type, sgl_task_params,
 490                                       dif_task_params);
 491
 492         init_default_iscsi_task(task_params, (struct data_hdr *)pdu_header,
 493                                 task_type);
 494
 495         cxt = task_params->context;
 496
 497         val = cpu_to_le32(task_size);
 498         cxt->ystorm_st_context.pdu_hdr.cmd.expected_transfer_length = val;
 499         init_initiator_rw_cdb_ystorm_context(&cxt->ystorm_st_context,
 500                                              cmd_params);
 501         val = cpu_to_le32(cmd_params->sense_data_buffer_phys_addr.lo);
 502         cxt->mstorm_st_context.sense_db.lo = val;
 503
 504         val = cpu_to_le32(cmd_params->sense_data_buffer_phys_addr.hi);
 505         cxt->mstorm_st_context.sense_db.hi = val;
 506
 507         if (task_params->tx_io_size) {
 508                 init_dif_context_flags(&cxt->ystorm_st_context.state.dif_flags,
 509                                        dif_task_params);
 510                 init_scsi_sgl_context(&cxt->ystorm_st_context.state.sgl_params,
 511                                       &cxt->ystorm_st_context.state.data_desc,
 512                                       sgl_task_params);
 513
 514                 slow_io = scsi_is_slow_sgl(sgl_task_params->num_sges,
 515                                            sgl_task_params->small_mid_sge);
 516
 517                 num_sges = !slow_io ? min_t(u16, sgl_task_params->num_sges,
 518                                             (u16)SCSI_NUM_SGES_SLOW_SGL_THR) :
 519                                       ISCSI_WQE_NUM_SGES_SLOWIO;
 520
 521                 if (slow_io) {
 522                         SET_FIELD(cxt->ystorm_st_context.state.flags,
 523                                   YSTORM_ISCSI_TASK_STATE_SLOW_IO, 1);
 524                 }
 525         } else if (task_params->rx_io_size) {
 526                 init_dif_context_flags(&cxt->mstorm_st_context.dif_flags,
 527                                        dif_task_params);
 528                 init_scsi_sgl_context(&cxt->mstorm_st_context.sgl_params,
 529                                       &cxt->mstorm_st_context.data_desc,
 530                                       sgl_task_params);
 531                 num_sges = !scsi_is_slow_sgl(sgl_task_params->num_sges,
 532                                 sgl_task_params->small_mid_sge) ?
 533                                 min_t(u16, sgl_task_params->num_sges,
 534                                       (u16)SCSI_NUM_SGES_SLOW_SGL_THR) :
 535                                 ISCSI_WQE_NUM_SGES_SLOWIO;
 536                 cxt->mstorm_st_context.rem_task_size = cpu_to_le32(task_size);
 537         }
 538
 539         if (exp_data_transfer_len > task_size  ||
 540             task_type != ISCSI_TASK_TYPE_TARGET_WRITE)
 541                 exp_data_transfer_len = task_size;
 542
 543         init_ustorm_task_contexts(&task_params->context->ustorm_st_context,
 544                                   &task_params->context->ustorm_ag_context,
 545                                   task_size, exp_data_transfer_len, num_sges,
 546                                   dif_task_params ?
 547                                   dif_task_params->tx_dif_conn_err_en : false);
 548
 549         set_rw_exp_data_acked_and_cont_len(task_params->context, conn_params,
 550                                            task_type, task_size,
 551                                            exp_data_transfer_len,
 552                                         GET_FIELD(pdu_header->hdr_second_dword,
 553                                                   ISCSI_CMD_HDR_TOTAL_AHS_LEN));
 554
 555         if (dif_task_params)
 556                 init_rtdif_task_context(&task_params->context->rdif_context,
 557                                         &task_params->context->tdif_context,
 558                                         dif_task_params, task_type);
 559
 560         init_sqe(task_params, sgl_task_params, dif_task_params, pdu_header,
 561                  cmd_params, task_type, false);
 562
 563         return 0;
 564 }
 565
 566 int init_initiator_rw_iscsi_task(struct iscsi_task_params *task_params,
 567                                  struct iscsi_conn_params *conn_params,
 568                                  struct scsi_initiator_cmd_params *cmd_params,
 569                                  struct iscsi_cmd_hdr *cmd_header,
 570                                  struct scsi_sgl_task_params *tx_sgl_params,
 571                                  struct scsi_sgl_task_params *rx_sgl_params,
 572                                  struct scsi_dif_task_params *dif_task_params)
 573 {
 574         if (GET_FIELD(cmd_header->flags_attr, ISCSI_CMD_HDR_WRITE))
 575                 return init_rw_iscsi_task(task_params,
 576                                           ISCSI_TASK_TYPE_INITIATOR_WRITE,
 577                                           conn_params,
 578                                           (struct iscsi_common_hdr *)cmd_header,
 579                                           tx_sgl_params, cmd_params,
 580                                           dif_task_params);
 581         else if (GET_FIELD(cmd_header->flags_attr, ISCSI_CMD_HDR_READ) ||
 582                  (task_params->rx_io_size == 0 && task_params->tx_io_size == 0))
 583                 return init_rw_iscsi_task(task_params,
 584                                           ISCSI_TASK_TYPE_INITIATOR_READ,
 585                                           conn_params,
 586                                           (struct iscsi_common_hdr *)cmd_header,
 587                                           rx_sgl_params, cmd_params,
 588                                           dif_task_params);
 589         else
 590                 return -1;
 591 }
 592
 593 int init_initiator_login_request_task(struct iscsi_task_params *task_params,
 594                                       struct iscsi_login_req_hdr  *login_header,
 595                                       struct scsi_sgl_task_params *tx_params,
 596                                       struct scsi_sgl_task_params *rx_params)
 597 {
 598         struct iscsi_task_context *cxt;
 599
 600         cxt = task_params->context;
 601
 602         init_default_iscsi_task(task_params,
 603                                 (struct data_hdr *)login_header,
 604                                 ISCSI_TASK_TYPE_MIDPATH);
 605
 606         init_ustorm_task_contexts(&cxt->ustorm_st_context,
 607                                   &cxt->ustorm_ag_context,
 608                                   task_params->rx_io_size ?
 609                                   rx_params->total_buffer_size : 0,
 610                                   task_params->tx_io_size ?
 611                                   tx_params->total_buffer_size : 0, 0,
 612                                   0);
 613
 614         if (task_params->tx_io_size)
 615                 init_scsi_sgl_context(&cxt->ystorm_st_context.state.sgl_params,
 616                                       &cxt->ystorm_st_context.state.data_desc,
 617                                       tx_params);
 618
 619         if (task_params->rx_io_size)
 620                 init_scsi_sgl_context(&cxt->mstorm_st_context.sgl_params,
 621                                       &cxt->mstorm_st_context.data_desc,
 622                                       rx_params);
 623
 624         cxt->mstorm_st_context.rem_task_size =
 625                         cpu_to_le32(task_params->rx_io_size ?
 626                                     rx_params->total_buffer_size : 0);
 627
 628         init_sqe(task_params, tx_params, NULL,
 629                  (struct iscsi_common_hdr *)login_header, NULL,
 630                  ISCSI_TASK_TYPE_MIDPATH, false);
 631
 632         return 0;
 633 }
 634
 635 int init_initiator_nop_out_task(struct iscsi_task_params *task_params,
 636                                 struct iscsi_nop_out_hdr *nop_out_pdu_header,
 637                                 struct scsi_sgl_task_params *tx_sgl_task_params,
 638                                 struct scsi_sgl_task_params *rx_sgl_task_params)
 639 {
 640         struct iscsi_task_context *cxt;
 641
 642         cxt = task_params->context;
 643
 644         init_default_iscsi_task(task_params,
 645                                 (struct data_hdr *)nop_out_pdu_header,
 646                                 ISCSI_TASK_TYPE_MIDPATH);
 647
 648         if (nop_out_pdu_header->itt == ISCSI_ITT_ALL_ONES)
 649                 set_local_completion_context(task_params->context);
 650
 651         if (task_params->tx_io_size)
 652                 init_scsi_sgl_context(&cxt->ystorm_st_context.state.sgl_params,
 653                                       &cxt->ystorm_st_context.state.data_desc,
 654                                       tx_sgl_task_params);
 655
 656         if (task_params->rx_io_size)
 657                 init_scsi_sgl_context(&cxt->mstorm_st_context.sgl_params,
 658                                       &cxt->mstorm_st_context.data_desc,
 659                                       rx_sgl_task_params);
 660
 661         init_ustorm_task_contexts(&cxt->ustorm_st_context,
 662                                   &cxt->ustorm_ag_context,
 663                                   task_params->rx_io_size ?
 664                                   rx_sgl_task_params->total_buffer_size : 0,
 665                                   task_params->tx_io_size ?
 666                                   tx_sgl_task_params->total_buffer_size : 0,
 667                                   0, 0);
 668
 669         cxt->mstorm_st_context.rem_task_size =
 670                                 cpu_to_le32(task_params->rx_io_size ?
 671                                         rx_sgl_task_params->total_buffer_size :
 672                                         0);
 673
 674         init_sqe(task_params, tx_sgl_task_params, NULL,
 675                  (struct iscsi_common_hdr *)nop_out_pdu_header, NULL,
 676                  ISCSI_TASK_TYPE_MIDPATH, false);
 677
 678         return 0;
 679 }
 680
 681 int init_initiator_logout_request_task(struct iscsi_task_params *task_params,
 682                                        struct iscsi_logout_req_hdr *logout_hdr,
 683                                        struct scsi_sgl_task_params *tx_params,
 684                                        struct scsi_sgl_task_params *rx_params)
 685 {
 686         struct iscsi_task_context *cxt;
 687
 688         cxt = task_params->context;
 689
 690         init_default_iscsi_task(task_params,
 691                                 (struct data_hdr *)logout_hdr,
 692                                 ISCSI_TASK_TYPE_MIDPATH);
 693
 694         if (task_params->tx_io_size)
 695                 init_scsi_sgl_context(&cxt->ystorm_st_context.state.sgl_params,
 696                                       &cxt->ystorm_st_context.state.data_desc,
 697                                       tx_params);
 698
 699         if (task_params->rx_io_size)
 700                 init_scsi_sgl_context(&cxt->mstorm_st_context.sgl_params,
 701                                       &cxt->mstorm_st_context.data_desc,
 702                                       rx_params);
 703
 704         init_ustorm_task_contexts(&cxt->ustorm_st_context,
 705                                   &cxt->ustorm_ag_context,
 706                                   task_params->rx_io_size ?
 707                                   rx_params->total_buffer_size : 0,
 708                                   task_params->tx_io_size ?
 709                                   tx_params->total_buffer_size : 0,
 710                                   0, 0);
 711
 712         cxt->mstorm_st_context.rem_task_size =
 713                                         cpu_to_le32(task_params->rx_io_size ?
 714                                         rx_params->total_buffer_size : 0);
 715
 716         init_sqe(task_params, tx_params, NULL,
 717                  (struct iscsi_common_hdr *)logout_hdr, NULL,
 718                  ISCSI_TASK_TYPE_MIDPATH, false);
 719
 720         return 0;
 721 }
 722
 723 int init_initiator_tmf_request_task(struct iscsi_task_params *task_params,
 724                                     struct iscsi_tmf_request_hdr *tmf_header)
 725 {
 726         init_default_iscsi_task(task_params, (struct data_hdr *)tmf_header,
 727                                 ISCSI_TASK_TYPE_MIDPATH);
 728
 729         init_sqe(task_params, NULL, NULL,
 730                  (struct iscsi_common_hdr *)tmf_header, NULL,
 731                  ISCSI_TASK_TYPE_MIDPATH, false);
 732
 733         return 0;
 734 }
 735
 736 int init_initiator_text_request_task(struct iscsi_task_params *task_params,
 737                                      struct iscsi_text_request_hdr *text_header,
 738                                      struct scsi_sgl_task_params *tx_params,
 739                                      struct scsi_sgl_task_params *rx_params)
 740 {
 741         struct iscsi_task_context *cxt;
 742
 743         cxt = task_params->context;
 744
 745         init_default_iscsi_task(task_params,
 746                                 (struct data_hdr *)text_header,
 747                                 ISCSI_TASK_TYPE_MIDPATH);
 748
 749         if (task_params->tx_io_size)
 750                 init_scsi_sgl_context(&cxt->ystorm_st_context.state.sgl_params,
 751                                       &cxt->ystorm_st_context.state.data_desc,
 752                                       tx_params);
 753
 754         if (task_params->rx_io_size)
 755                 init_scsi_sgl_context(&cxt->mstorm_st_context.sgl_params,
 756                                       &cxt->mstorm_st_context.data_desc,
 757                                       rx_params);
 758
 759         cxt->mstorm_st_context.rem_task_size =
 760                                 cpu_to_le32(task_params->rx_io_size ?
 761                                         rx_params->total_buffer_size : 0);
 762
 763         init_ustorm_task_contexts(&cxt->ustorm_st_context,
 764                                   &cxt->ustorm_ag_context,
 765                                   task_params->rx_io_size ?
 766                                   rx_params->total_buffer_size : 0,
 767                                   task_params->tx_io_size ?
 768                                   tx_params->total_buffer_size : 0, 0, 0);
 769
 770         init_sqe(task_params, tx_params, NULL,
 771                  (struct iscsi_common_hdr *)text_header, NULL,
 772                  ISCSI_TASK_TYPE_MIDPATH, false);
 773
 774         return 0;
 775 }
 776
 777 int init_cleanup_task(struct iscsi_task_params *task_params)
 778 {
 779         init_sqe(task_params, NULL, NULL, NULL, NULL, ISCSI_TASK_TYPE_MIDPATH,
 780                  true);
 781         return 0;
 782 }