drivers/target/iscsi/cxgbit/cxgbit_target.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2016 Chelsio Communications, Inc.
   4  */
   5
   6 #include <linux/workqueue.h>
   7 #include <linux/kthread.h>
   8 #include <linux/sched/signal.h>
   9
  10 #include <asm/unaligned.h>
  11 #include <net/tcp.h>
  12 #include <target/target_core_base.h>
  13 #include <target/target_core_fabric.h>
  14 #include "cxgbit.h"
  15
  16 struct sge_opaque_hdr {
  17         void *dev;
  18         dma_addr_t addr[MAX_SKB_FRAGS + 1];
  19 };
  20
  21 static const u8 cxgbit_digest_len[] = {0, 4, 4, 8};
  22
  23 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \
  24                     sizeof(struct fw_ofld_tx_data_wr))
  25
  26 static struct sk_buff *
  27 __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso)
  28 {
  29         struct sk_buff *skb = NULL;
  30         u8 submode = 0;
  31         int errcode;
  32         static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN;
  33
  34         if (len) {
  35                 skb = alloc_skb_with_frags(hdr_len, len,
  36                                            0, &errcode,
  37                                            GFP_KERNEL);
  38                 if (!skb)
  39                         return NULL;
  40
  41                 skb_reserve(skb, TX_HDR_LEN);
  42                 skb_reset_transport_header(skb);
  43                 __skb_put(skb, ISCSI_HDR_LEN);
  44                 skb->data_len = len;
  45                 skb->len += len;
  46                 submode |= (csk->submode & CXGBIT_SUBMODE_DCRC);
  47
  48         } else {
  49                 u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0;
  50
  51                 skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL);
  52                 if (!skb)
  53                         return NULL;
  54
  55                 skb_reserve(skb, TX_HDR_LEN + iso_len);
  56                 skb_reset_transport_header(skb);
  57                 __skb_put(skb, ISCSI_HDR_LEN);
  58         }
  59
  60         submode |= (csk->submode & CXGBIT_SUBMODE_HCRC);
  61         cxgbit_skcb_submode(skb) = submode;
  62         cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode];
  63         cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR;
  64         return skb;
  65 }
  66
  67 static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len)
  68 {
  69         return __cxgbit_alloc_skb(csk, len, false);
  70 }
  71
  72 /*
  73  * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data
  74  * @skb: the packet
  75  *
  76  * Returns true if a packet can be sent as an offload WR with immediate
  77  * data.  We currently use the same limit as for Ethernet packets.
  78  */
  79 static int cxgbit_is_ofld_imm(const struct sk_buff *skb)
  80 {
  81         int length = skb->len;
  82
  83         if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
  84                 length += sizeof(struct fw_ofld_tx_data_wr);
  85
  86         if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO))
  87                 length += sizeof(struct cpl_tx_data_iso);
  88
  89 #define MAX_IMM_TX_PKT_LEN      256
  90         return length <= MAX_IMM_TX_PKT_LEN;
  91 }
  92
  93 /*
  94  * cxgbit_sgl_len - calculates the size of an SGL of the given capacity
  95  * @n: the number of SGL entries
  96  * Calculates the number of flits needed for a scatter/gather list that
  97  * can hold the given number of entries.
  98  */
  99 static inline unsigned int cxgbit_sgl_len(unsigned int n)
 100 {
 101         n--;
 102         return (3 * n) / 2 + (n & 1) + 2;
 103 }
 104
 105 /*
 106  * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet
 107  * @skb: the packet
 108  *
 109  * Returns the number of flits needed for the given offload packet.
 110  * These packets are already fully constructed and no additional headers
 111  * will be added.
 112  */
 113 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb)
 114 {
 115         unsigned int flits, cnt;
 116
 117         if (cxgbit_is_ofld_imm(skb))
 118                 return DIV_ROUND_UP(skb->len, 8);
 119         flits = skb_transport_offset(skb) / 8;
 120         cnt = skb_shinfo(skb)->nr_frags;
 121         if (skb_tail_pointer(skb) != skb_transport_header(skb))
 122                 cnt++;
 123         return flits + cxgbit_sgl_len(cnt);
 124 }
 125
 126 #define CXGBIT_ISO_FSLICE 0x1
 127 #define CXGBIT_ISO_LSLICE 0x2
 128 static void
 129 cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info)
 130 {
 131         struct cpl_tx_data_iso *cpl;
 132         unsigned int submode = cxgbit_skcb_submode(skb);
 133         unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE);
 134         unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE);
 135
 136         cpl = __skb_push(skb, sizeof(*cpl));
 137
 138         cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) |
 139                         CPL_TX_DATA_ISO_FIRST_V(fslice) |
 140                         CPL_TX_DATA_ISO_LAST_V(lslice) |
 141                         CPL_TX_DATA_ISO_CPLHDRLEN_V(0) |
 142                         CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) |
 143                         CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) |
 144                         CPL_TX_DATA_ISO_IMMEDIATE_V(0) |
 145                         CPL_TX_DATA_ISO_SCSI_V(2));
 146
 147         cpl->ahs_len = 0;
 148         cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4));
 149         cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4));
 150         cpl->len = htonl(iso_info->len);
 151         cpl->reserved2_seglen_offset = htonl(0);
 152         cpl->datasn_offset = htonl(0);
 153         cpl->buffer_offset = htonl(0);
 154         cpl->reserved3 = 0;
 155
 156         __skb_pull(skb, sizeof(*cpl));
 157 }
 158
 159 static void
 160 cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen,
 161                   u32 len, u32 credits, u32 compl)
 162 {
 163         struct fw_ofld_tx_data_wr *req;
 164         const struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
 165         u32 submode = cxgbit_skcb_submode(skb);
 166         u32 wr_ulp_mode = 0;
 167         u32 hdr_size = sizeof(*req);
 168         u32 opcode = FW_OFLD_TX_DATA_WR;
 169         u32 immlen = 0;
 170         u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) :
 171                     T6_TX_FORCE_F;
 172
 173         if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) {
 174                 opcode = FW_ISCSI_TX_DATA_WR;
 175                 immlen += sizeof(struct cpl_tx_data_iso);
 176                 hdr_size += sizeof(struct cpl_tx_data_iso);
 177                 submode |= 8;
 178         }
 179
 180         if (cxgbit_is_ofld_imm(skb))
 181                 immlen += dlen;
 182
 183         req = __skb_push(skb, hdr_size);
 184         req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) |
 185                                         FW_WR_COMPL_V(compl) |
 186                                         FW_WR_IMMDLEN_V(immlen));
 187         req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
 188                                         FW_WR_LEN16_V(credits));
 189         req->plen = htonl(len);
 190         wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) |
 191                                 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
 192
 193         req->tunnel_to_proxy = htonl((wr_ulp_mode) | force |
 194                  FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1));
 195 }
 196
 197 static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb)
 198 {
 199         kfree_skb(skb);
 200 }
 201
 202 void cxgbit_push_tx_frames(struct cxgbit_sock *csk)
 203 {
 204         struct sk_buff *skb;
 205
 206         while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) {
 207                 u32 dlen = skb->len;
 208                 u32 len = skb->len;
 209                 u32 credits_needed;
 210                 u32 compl = 0;
 211                 u32 flowclen16 = 0;
 212                 u32 iso_cpl_len = 0;
 213
 214                 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)
 215                         iso_cpl_len = sizeof(struct cpl_tx_data_iso);
 216
 217                 if (cxgbit_is_ofld_imm(skb))
 218                         credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16);
 219                 else
 220                         credits_needed = DIV_ROUND_UP((8 *
 221                                         cxgbit_calc_tx_flits_ofld(skb)) +
 222                                         iso_cpl_len, 16);
 223
 224                 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
 225                         credits_needed += DIV_ROUND_UP(
 226                                 sizeof(struct fw_ofld_tx_data_wr), 16);
 227                 /*
 228                  * Assumes the initial credits is large enough to support
 229                  * fw_flowc_wr plus largest possible first payload
 230                  */
 231
 232                 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) {
 233                         flowclen16 = cxgbit_send_tx_flowc_wr(csk);
 234                         csk->wr_cred -= flowclen16;
 235                         csk->wr_una_cred += flowclen16;
 236                 }
 237
 238                 if (csk->wr_cred < credits_needed) {
 239                         pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n",
 240                                  csk, skb->len, skb->data_len,
 241                                  credits_needed, csk->wr_cred);
 242                         break;
 243                 }
 244                 __skb_unlink(skb, &csk->txq);
 245                 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
 246                 skb->csum = (__force __wsum)(credits_needed + flowclen16);
 247                 csk->wr_cred -= credits_needed;
 248                 csk->wr_una_cred += credits_needed;
 249
 250                 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
 251                          csk, skb->len, skb->data_len, credits_needed,
 252                          csk->wr_cred, csk->wr_una_cred);
 253
 254                 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) {
 255                         len += cxgbit_skcb_tx_extralen(skb);
 256
 257                         if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) ||
 258                             (!before(csk->write_seq,
 259                                      csk->snd_una + csk->snd_win))) {
 260                                 compl = 1;
 261                                 csk->wr_una_cred = 0;
 262                         }
 263
 264                         cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed,
 265                                           compl);
 266                         csk->snd_nxt += len;
 267
 268                 } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) ||
 269                            (csk->wr_una_cred >= (csk->wr_max_cred / 2))) {
 270                         struct cpl_close_con_req *req =
 271                                 (struct cpl_close_con_req *)skb->data;
 272                         req->wr.wr_hi |= htonl(FW_WR_COMPL_F);
 273                         csk->wr_una_cred = 0;
 274                 }
 275
 276                 cxgbit_sock_enqueue_wr(csk, skb);
 277                 t4_set_arp_err_handler(skb, csk,
 278                                        cxgbit_arp_failure_skb_discard);
 279
 280                 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n",
 281                          csk, csk->tid, skb, len);
 282
 283                 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
 284         }
 285 }
 286
 287 static bool cxgbit_lock_sock(struct cxgbit_sock *csk)
 288 {
 289         spin_lock_bh(&csk->lock);
 290
 291         if (before(csk->write_seq, csk->snd_una + csk->snd_win))
 292                 csk->lock_owner = true;
 293
 294         spin_unlock_bh(&csk->lock);
 295
 296         return csk->lock_owner;
 297 }
 298
 299 static void cxgbit_unlock_sock(struct cxgbit_sock *csk)
 300 {
 301         struct sk_buff_head backlogq;
 302         struct sk_buff *skb;
 303         void (*fn)(struct cxgbit_sock *, struct sk_buff *);
 304
 305         skb_queue_head_init(&backlogq);
 306
 307         spin_lock_bh(&csk->lock);
 308         while (skb_queue_len(&csk->backlogq)) {
 309                 skb_queue_splice_init(&csk->backlogq, &backlogq);
 310                 spin_unlock_bh(&csk->lock);
 311
 312                 while ((skb = __skb_dequeue(&backlogq))) {
 313                         fn = cxgbit_skcb_rx_backlog_fn(skb);
 314                         fn(csk, skb);
 315                 }
 316
 317                 spin_lock_bh(&csk->lock);
 318         }
 319
 320         csk->lock_owner = false;
 321         spin_unlock_bh(&csk->lock);
 322 }
 323
 324 static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
 325 {
 326         int ret = 0;
 327
 328         wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk));
 329
 330         if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) ||
 331                      signal_pending(current))) {
 332                 __kfree_skb(skb);
 333                 __skb_queue_purge(&csk->ppodq);
 334                 ret = -1;
 335                 spin_lock_bh(&csk->lock);
 336                 if (csk->lock_owner) {
 337                         spin_unlock_bh(&csk->lock);
 338                         goto unlock;
 339                 }
 340                 spin_unlock_bh(&csk->lock);
 341                 return ret;
 342         }
 343
 344         csk->write_seq += skb->len +
 345                           cxgbit_skcb_tx_extralen(skb);
 346
 347         skb_queue_splice_tail_init(&csk->ppodq, &csk->txq);
 348         __skb_queue_tail(&csk->txq, skb);
 349         cxgbit_push_tx_frames(csk);
 350
 351 unlock:
 352         cxgbit_unlock_sock(csk);
 353         return ret;
 354 }
 355
 356 static int
 357 cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset,
 358                u32 data_length)
 359 {
 360         u32 i = 0, nr_frags = MAX_SKB_FRAGS;
 361         u32 padding = ((-data_length) & 3);
 362         struct scatterlist *sg;
 363         struct page *page;
 364         unsigned int page_off;
 365
 366         if (padding)
 367                 nr_frags--;
 368
 369         /*
 370          * We know each entry in t_data_sg contains a page.
 371          */
 372         sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE];
 373         page_off = (data_offset % PAGE_SIZE);
 374
 375         while (data_length && (i < nr_frags)) {
 376                 u32 cur_len = min_t(u32, data_length, sg->length - page_off);
 377
 378                 page = sg_page(sg);
 379
 380                 get_page(page);
 381                 skb_fill_page_desc(skb, i, page, sg->offset + page_off,
 382                                    cur_len);
 383                 skb->data_len += cur_len;
 384                 skb->len += cur_len;
 385                 skb->truesize += cur_len;
 386
 387                 data_length -= cur_len;
 388                 page_off = 0;
 389                 sg = sg_next(sg);
 390                 i++;
 391         }
 392
 393         if (data_length)
 394                 return -1;
 395
 396         if (padding) {
 397                 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 398                 if (!page)
 399                         return -1;
 400                 skb_fill_page_desc(skb, i, page, 0, padding);
 401                 skb->data_len += padding;
 402                 skb->len += padding;
 403                 skb->truesize += padding;
 404         }
 405
 406         return 0;
 407 }
 408
 409 static int
 410 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
 411                      struct iscsi_datain_req *dr)
 412 {
 413         struct iscsi_conn *conn = csk->conn;
 414         struct sk_buff *skb;
 415         struct iscsi_datain datain;
 416         struct cxgbit_iso_info iso_info;
 417         u32 data_length = cmd->se_cmd.data_length;
 418         u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
 419         u32 num_pdu, plen, tx_data = 0;
 420         bool task_sense = !!(cmd->se_cmd.se_cmd_flags &
 421                 SCF_TRANSPORT_TASK_SENSE);
 422         bool set_statsn = false;
 423         int ret = -1;
 424
 425         while (data_length) {
 426                 num_pdu = (data_length + mrdsl - 1) / mrdsl;
 427                 if (num_pdu > csk->max_iso_npdu)
 428                         num_pdu = csk->max_iso_npdu;
 429
 430                 plen = num_pdu * mrdsl;
 431                 if (plen > data_length)
 432                         plen = data_length;
 433
 434                 skb = __cxgbit_alloc_skb(csk, 0, true);
 435                 if (unlikely(!skb))
 436                         return -ENOMEM;
 437
 438                 memset(skb->data, 0, ISCSI_HDR_LEN);
 439                 cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO;
 440                 cxgbit_skcb_submode(skb) |= (csk->submode &
 441                                 CXGBIT_SUBMODE_DCRC);
 442                 cxgbit_skcb_tx_extralen(skb) = (num_pdu *
 443                                 cxgbit_digest_len[cxgbit_skcb_submode(skb)]) +
 444                                                 ((num_pdu - 1) * ISCSI_HDR_LEN);
 445
 446                 memset(&datain, 0, sizeof(struct iscsi_datain));
 447                 memset(&iso_info, 0, sizeof(iso_info));
 448
 449                 if (!tx_data)
 450                         iso_info.flags |= CXGBIT_ISO_FSLICE;
 451
 452                 if (!(data_length - plen)) {
 453                         iso_info.flags |= CXGBIT_ISO_LSLICE;
 454                         if (!task_sense) {
 455                                 datain.flags = ISCSI_FLAG_DATA_STATUS;
 456                                 iscsit_increment_maxcmdsn(cmd, conn->sess);
 457                                 cmd->stat_sn = conn->stat_sn++;
 458                                 set_statsn = true;
 459                         }
 460                 }
 461
 462                 iso_info.burst_len = num_pdu * mrdsl;
 463                 iso_info.mpdu = mrdsl;
 464                 iso_info.len = ISCSI_HDR_LEN + plen;
 465
 466                 cxgbit_cpl_tx_data_iso(skb, &iso_info);
 467
 468                 datain.offset = tx_data;
 469                 datain.data_sn = cmd->data_sn - 1;
 470
 471                 iscsit_build_datain_pdu(cmd, conn, &datain,
 472                                         (struct iscsi_data_rsp *)skb->data,
 473                                         set_statsn);
 474
 475                 ret = cxgbit_map_skb(cmd, skb, tx_data, plen);
 476                 if (unlikely(ret)) {
 477                         __kfree_skb(skb);
 478                         goto out;
 479                 }
 480
 481                 ret = cxgbit_queue_skb(csk, skb);
 482                 if (unlikely(ret))
 483                         goto out;
 484
 485                 tx_data += plen;
 486                 data_length -= plen;
 487
 488                 cmd->read_data_done += plen;
 489                 cmd->data_sn += num_pdu;
 490         }
 491
 492         dr->dr_complete = DATAIN_COMPLETE_NORMAL;
 493
 494         return 0;
 495
 496 out:
 497         return ret;
 498 }
 499
 500 static int
 501 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
 502                  const struct iscsi_datain *datain)
 503 {
 504         struct sk_buff *skb;
 505         int ret = 0;
 506
 507         skb = cxgbit_alloc_skb(csk, 0);
 508         if (unlikely(!skb))
 509                 return -ENOMEM;
 510
 511         memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
 512
 513         if (datain->length) {
 514                 cxgbit_skcb_submode(skb) |= (csk->submode &
 515                                 CXGBIT_SUBMODE_DCRC);
 516                 cxgbit_skcb_tx_extralen(skb) =
 517                                 cxgbit_digest_len[cxgbit_skcb_submode(skb)];
 518         }
 519
 520         ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length);
 521         if (ret < 0) {
 522                 __kfree_skb(skb);
 523                 return ret;
 524         }
 525
 526         return cxgbit_queue_skb(csk, skb);
 527 }
 528
 529 static int
 530 cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 531                        struct iscsi_datain_req *dr,
 532                        const struct iscsi_datain *datain)
 533 {
 534         struct cxgbit_sock *csk = conn->context;
 535         u32 data_length = cmd->se_cmd.data_length;
 536         u32 padding = ((-data_length) & 3);
 537         u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
 538
 539         if ((data_length > mrdsl) && (!dr->recovery) &&
 540             (!padding) && (!datain->offset) && csk->max_iso_npdu) {
 541                 atomic_long_add(data_length - datain->length,
 542                                 &conn->sess->tx_data_octets);
 543                 return cxgbit_tx_datain_iso(csk, cmd, dr);
 544         }
 545
 546         return cxgbit_tx_datain(csk, cmd, datain);
 547 }
 548
 549 static int
 550 cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 551                           const void *data_buf, u32 data_buf_len)
 552 {
 553         struct cxgbit_sock *csk = conn->context;
 554         struct sk_buff *skb;
 555         u32 padding = ((-data_buf_len) & 3);
 556
 557         skb = cxgbit_alloc_skb(csk, data_buf_len + padding);
 558         if (unlikely(!skb))
 559                 return -ENOMEM;
 560
 561         memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
 562
 563         if (data_buf_len) {
 564                 u32 pad_bytes = 0;
 565
 566                 skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len);
 567
 568                 if (padding)
 569                         skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len,
 570                                        &pad_bytes, padding);
 571         }
 572
 573         cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[
 574                                        cxgbit_skcb_submode(skb)];
 575
 576         return cxgbit_queue_skb(csk, skb);
 577 }
 578
 579 int
 580 cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 581                 struct iscsi_datain_req *dr, const void *buf, u32 buf_len)
 582 {
 583         if (dr)
 584                 return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf);
 585         else
 586                 return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len);
 587 }
 588
 589 int cxgbit_validate_params(struct iscsi_conn *conn)
 590 {
 591         struct cxgbit_sock *csk = conn->context;
 592         struct cxgbit_device *cdev = csk->com.cdev;
 593         struct iscsi_param *param;
 594         u32 max_xmitdsl;
 595
 596         param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH,
 597                                           conn->param_list);
 598         if (!param)
 599                 return -1;
 600
 601         if (kstrtou32(param->value, 0, &max_xmitdsl) < 0)
 602                 return -1;
 603
 604         if (max_xmitdsl > cdev->mdsl) {
 605                 if (iscsi_change_param_sprintf(
 606                         conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl))
 607                         return -1;
 608         }
 609
 610         return 0;
 611 }
 612
 613 static int cxgbit_set_digest(struct cxgbit_sock *csk)
 614 {
 615         struct iscsi_conn *conn = csk->conn;
 616         struct iscsi_param *param;
 617
 618         param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list);
 619         if (!param) {
 620                 pr_err("param not found key %s\n", HEADERDIGEST);
 621                 return -1;
 622         }
 623
 624         if (!strcmp(param->value, CRC32C))
 625                 csk->submode |= CXGBIT_SUBMODE_HCRC;
 626
 627         param = iscsi_find_param_from_key(DATADIGEST, conn->param_list);
 628         if (!param) {
 629                 csk->submode = 0;
 630                 pr_err("param not found key %s\n", DATADIGEST);
 631                 return -1;
 632         }
 633
 634         if (!strcmp(param->value, CRC32C))
 635                 csk->submode |= CXGBIT_SUBMODE_DCRC;
 636
 637         if (cxgbit_setup_conn_digest(csk)) {
 638                 csk->submode = 0;
 639                 return -1;
 640         }
 641
 642         return 0;
 643 }
 644
 645 static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk)
 646 {
 647         struct iscsi_conn *conn = csk->conn;
 648         struct iscsi_conn_ops *conn_ops = conn->conn_ops;
 649         struct iscsi_param *param;
 650         u32 mrdsl, mbl;
 651         u32 max_npdu, max_iso_npdu;
 652         u32 max_iso_payload;
 653
 654         if (conn->login->leading_connection) {
 655                 param = iscsi_find_param_from_key(MAXBURSTLENGTH,
 656                                                   conn->param_list);
 657                 if (!param) {
 658                         pr_err("param not found key %s\n", MAXBURSTLENGTH);
 659                         return -1;
 660                 }
 661
 662                 if (kstrtou32(param->value, 0, &mbl) < 0)
 663                         return -1;
 664         } else {
 665                 mbl = conn->sess->sess_ops->MaxBurstLength;
 666         }
 667
 668         mrdsl = conn_ops->MaxRecvDataSegmentLength;
 669         max_npdu = mbl / mrdsl;
 670
 671         max_iso_payload = rounddown(CXGBIT_MAX_ISO_PAYLOAD, csk->emss);
 672
 673         max_iso_npdu = max_iso_payload /
 674                        (ISCSI_HDR_LEN + mrdsl +
 675                         cxgbit_digest_len[csk->submode]);
 676
 677         csk->max_iso_npdu = min(max_npdu, max_iso_npdu);
 678
 679         if (csk->max_iso_npdu <= 1)
 680                 csk->max_iso_npdu = 0;
 681
 682         return 0;
 683 }
 684
 685 /*
 686  * cxgbit_seq_pdu_inorder()
 687  * @csk: pointer to cxgbit socket structure
 688  *
 689  * This function checks whether data sequence and data
 690  * pdu are in order.
 691  *
 692  * Return: returns -1 on error, 0 if data sequence and
 693  * data pdu are in order, 1 if data sequence or data pdu
 694  * is not in order.
 695  */
 696 static int cxgbit_seq_pdu_inorder(struct cxgbit_sock *csk)
 697 {
 698         struct iscsi_conn *conn = csk->conn;
 699         struct iscsi_param *param;
 700
 701         if (conn->login->leading_connection) {
 702                 param = iscsi_find_param_from_key(DATASEQUENCEINORDER,
 703                                                   conn->param_list);
 704                 if (!param) {
 705                         pr_err("param not found key %s\n", DATASEQUENCEINORDER);
 706                         return -1;
 707                 }
 708
 709                 if (strcmp(param->value, YES))
 710                         return 1;
 711
 712                 param = iscsi_find_param_from_key(DATAPDUINORDER,
 713                                                   conn->param_list);
 714                 if (!param) {
 715                         pr_err("param not found key %s\n", DATAPDUINORDER);
 716                         return -1;
 717                 }
 718
 719                 if (strcmp(param->value, YES))
 720                         return 1;
 721
 722         } else {
 723                 if (!conn->sess->sess_ops->DataSequenceInOrder)
 724                         return 1;
 725                 if (!conn->sess->sess_ops->DataPDUInOrder)
 726                         return 1;
 727         }
 728
 729         return 0;
 730 }
 731
 732 static int cxgbit_set_params(struct iscsi_conn *conn)
 733 {
 734         struct cxgbit_sock *csk = conn->context;
 735         struct cxgbit_device *cdev = csk->com.cdev;
 736         struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm;
 737         struct iscsi_conn_ops *conn_ops = conn->conn_ops;
 738         struct iscsi_param *param;
 739         u8 erl;
 740
 741         if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl)
 742                 conn_ops->MaxRecvDataSegmentLength = cdev->mdsl;
 743
 744         if (cxgbit_set_digest(csk))
 745                 return -1;
 746
 747         if (conn->login->leading_connection) {
 748                 param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL,
 749                                                   conn->param_list);
 750                 if (!param) {
 751                         pr_err("param not found key %s\n", ERRORRECOVERYLEVEL);
 752                         return -1;
 753                 }
 754                 if (kstrtou8(param->value, 0, &erl) < 0)
 755                         return -1;
 756         } else {
 757                 erl = conn->sess->sess_ops->ErrorRecoveryLevel;
 758         }
 759
 760         if (!erl) {
 761                 int ret;
 762
 763                 ret = cxgbit_seq_pdu_inorder(csk);
 764                 if (ret < 0) {
 765                         return -1;
 766                 } else if (ret > 0) {
 767                         if (is_t5(cdev->lldi.adapter_type))
 768                                 goto enable_ddp;
 769                         else
 770                                 return 0;
 771                 }
 772
 773                 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) {
 774                         if (cxgbit_set_iso_npdu(csk))
 775                                 return -1;
 776                 }
 777
 778 enable_ddp:
 779                 if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) {
 780                         if (cxgbit_setup_conn_pgidx(csk,
 781                                                     ppm->tformat.pgsz_idx_dflt))
 782                                 return -1;
 783                         set_bit(CSK_DDP_ENABLE, &csk->com.flags);
 784                 }
 785         }
 786
 787         return 0;
 788 }
 789
 790 int
 791 cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
 792                     u32 length)
 793 {
 794         struct cxgbit_sock *csk = conn->context;
 795         struct sk_buff *skb;
 796         u32 padding_buf = 0;
 797         u8 padding = ((-length) & 3);
 798
 799         skb = cxgbit_alloc_skb(csk, length + padding);
 800         if (!skb)
 801                 return -ENOMEM;
 802         skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN);
 803         skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length);
 804
 805         if (padding)
 806                 skb_store_bits(skb, ISCSI_HDR_LEN + length,
 807                                &padding_buf, padding);
 808
 809         if (login->login_complete) {
 810                 if (cxgbit_set_params(conn)) {
 811                         kfree_skb(skb);
 812                         return -1;
 813                 }
 814
 815                 set_bit(CSK_LOGIN_DONE, &csk->com.flags);
 816         }
 817
 818         if (cxgbit_queue_skb(csk, skb))
 819                 return -1;
 820
 821         if ((!login->login_complete) && (!login->login_failed))
 822                 schedule_delayed_work(&conn->login_work, 0);
 823
 824         return 0;
 825 }
 826
 827 static void
 828 cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg,
 829                       unsigned int nents, u32 skip)
 830 {
 831         struct skb_seq_state st;
 832         const u8 *buf;
 833         unsigned int consumed = 0, buf_len;
 834         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb);
 835
 836         skb_prepare_seq_read(skb, pdu_cb->doffset,
 837                              pdu_cb->doffset + pdu_cb->dlen,
 838                              &st);
 839
 840         while (true) {
 841                 buf_len = skb_seq_read(consumed, &buf, &st);
 842                 if (!buf_len) {
 843                         skb_abort_seq_read(&st);
 844                         break;
 845                 }
 846
 847                 consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf,
 848                                                  buf_len, skip + consumed);
 849         }
 850 }
 851
 852 static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk)
 853 {
 854         struct iscsi_conn *conn = csk->conn;
 855         struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev);
 856         struct cxgbit_cmd *ccmd;
 857         struct iscsi_cmd *cmd;
 858
 859         cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
 860         if (!cmd) {
 861                 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n");
 862                 return NULL;
 863         }
 864
 865         ccmd = iscsit_priv_cmd(cmd);
 866         ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask;
 867         ccmd->setup_ddp = true;
 868
 869         return cmd;
 870 }
 871
 872 static int
 873 cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
 874                              u32 length)
 875 {
 876         struct iscsi_conn *conn = cmd->conn;
 877         struct cxgbit_sock *csk = conn->context;
 878         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
 879
 880         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
 881                 pr_err("ImmediateData CRC32C DataDigest error\n");
 882                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 883                         pr_err("Unable to recover from"
 884                                " Immediate Data digest failure while"
 885                                " in ERL=0.\n");
 886                         iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
 887                                           (unsigned char *)hdr);
 888                         return IMMEDIATE_DATA_CANNOT_RECOVER;
 889                 }
 890
 891                 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
 892                                   (unsigned char *)hdr);
 893                 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
 894         }
 895
 896         if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
 897                 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
 898                 struct skb_shared_info *ssi = skb_shinfo(csk->skb);
 899                 skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx];
 900
 901                 sg_init_table(&ccmd->sg, 1);
 902                 sg_set_page(&ccmd->sg, dfrag->page.p, skb_frag_size(dfrag),
 903                             dfrag->page_offset);
 904                 get_page(dfrag->page.p);
 905
 906                 cmd->se_cmd.t_data_sg = &ccmd->sg;
 907                 cmd->se_cmd.t_data_nents = 1;
 908
 909                 ccmd->release = true;
 910         } else {
 911                 struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0];
 912                 u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE));
 913
 914                 cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents, 0);
 915         }
 916
 917         cmd->write_data_done += pdu_cb->dlen;
 918
 919         if (cmd->write_data_done == cmd->se_cmd.data_length) {
 920                 spin_lock_bh(&cmd->istate_lock);
 921                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
 922                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
 923                 spin_unlock_bh(&cmd->istate_lock);
 924         }
 925
 926         return IMMEDIATE_DATA_NORMAL_OPERATION;
 927 }
 928
 929 static int
 930 cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
 931                           bool dump_payload)
 932 {
 933         struct iscsi_conn *conn = cmd->conn;
 934         int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
 935         /*
 936          * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
 937          */
 938         if (dump_payload)
 939                 goto after_immediate_data;
 940
 941         immed_ret = cxgbit_handle_immediate_data(cmd, hdr,
 942                                                  cmd->first_burst_len);
 943 after_immediate_data:
 944         if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
 945                 /*
 946                  * A PDU/CmdSN carrying Immediate Data passed
 947                  * DataCRC, check against ExpCmdSN/MaxCmdSN if
 948                  * Immediate Bit is not set.
 949                  */
 950                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
 951                                                 (unsigned char *)hdr,
 952                                                 hdr->cmdsn);
 953                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
 954                         return -1;
 955
 956                 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
 957                         target_put_sess_cmd(&cmd->se_cmd);
 958                         return 0;
 959                 } else if (cmd->unsolicited_data) {
 960                         iscsit_set_unsolicited_dataout(cmd);
 961                 }
 962
 963         } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
 964                 /*
 965                  * Immediate Data failed DataCRC and ERL>=1,
 966                  * silently drop this PDU and let the initiator
 967                  * plug the CmdSN gap.
 968                  *
 969                  * FIXME: Send Unsolicited NOPIN with reserved
 970                  * TTT here to help the initiator figure out
 971                  * the missing CmdSN, although they should be
 972                  * intelligent enough to determine the missing
 973                  * CmdSN and issue a retry to plug the sequence.
 974                  */
 975                 cmd->i_state = ISTATE_REMOVE;
 976                 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
 977         } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
 978                 return -1;
 979
 980         return 0;
 981 }
 982
 983 static int
 984 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
 985 {
 986         struct iscsi_conn *conn = csk->conn;
 987         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
 988         struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr;
 989         int rc;
 990         bool dump_payload = false;
 991
 992         rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr);
 993         if (rc < 0)
 994                 return rc;
 995
 996         if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) &&
 997             (pdu_cb->nr_dfrags == 1))
 998                 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
 999
1000         rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
1001         if (rc < 0)
1002                 return 0;
1003         else if (rc > 0)
1004                 dump_payload = true;
1005
1006         if (!pdu_cb->dlen)
1007                 return 0;
1008
1009         return cxgbit_get_immediate_data(cmd, hdr, dump_payload);
1010 }
1011
1012 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk)
1013 {
1014         struct scatterlist *sg_start;
1015         struct iscsi_conn *conn = csk->conn;
1016         struct iscsi_cmd *cmd = NULL;
1017         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1018         struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr;
1019         u32 data_offset = be32_to_cpu(hdr->offset);
1020         u32 data_len = pdu_cb->dlen;
1021         int rc, sg_nents, sg_off;
1022         bool dcrc_err = false;
1023
1024         if (pdu_cb->flags & PDUCBF_RX_DDP_CMP) {
1025                 u32 offset = be32_to_cpu(hdr->offset);
1026                 u32 ddp_data_len;
1027                 u32 payload_length = ntoh24(hdr->dlength);
1028                 bool success = false;
1029
1030                 cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, 0);
1031                 if (!cmd)
1032                         return 0;
1033
1034                 ddp_data_len = offset - cmd->write_data_done;
1035                 atomic_long_add(ddp_data_len, &conn->sess->rx_data_octets);
1036
1037                 cmd->write_data_done = offset;
1038                 cmd->next_burst_len = ddp_data_len;
1039                 cmd->data_sn = be32_to_cpu(hdr->datasn);
1040
1041                 rc = __iscsit_check_dataout_hdr(conn, (unsigned char *)hdr,
1042                                                 cmd, payload_length, &success);
1043                 if (rc < 0)
1044                         return rc;
1045                 else if (!success)
1046                         return 0;
1047         } else {
1048                 rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd);
1049                 if (rc < 0)
1050                         return rc;
1051                 else if (!cmd)
1052                         return 0;
1053         }
1054
1055         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1056                 pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1057                        " DataSN: 0x%08x\n",
1058                        hdr->itt, hdr->offset, data_len,
1059                        hdr->datasn);
1060
1061                 dcrc_err = true;
1062                 goto check_payload;
1063         }
1064
1065         pr_debug("DataOut data_len: %u, "
1066                 "write_data_done: %u, data_length: %u\n",
1067                   data_len,  cmd->write_data_done,
1068                   cmd->se_cmd.data_length);
1069
1070         if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) {
1071                 u32 skip = data_offset % PAGE_SIZE;
1072
1073                 sg_off = data_offset / PAGE_SIZE;
1074                 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1075                 sg_nents = max(1UL, DIV_ROUND_UP(skip + data_len, PAGE_SIZE));
1076
1077                 cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents, skip);
1078         }
1079
1080 check_payload:
1081
1082         rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err);
1083         if (rc < 0)
1084                 return rc;
1085
1086         return 0;
1087 }
1088
1089 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1090 {
1091         struct iscsi_conn *conn = csk->conn;
1092         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1093         struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr;
1094         unsigned char *ping_data = NULL;
1095         u32 payload_length = pdu_cb->dlen;
1096         int ret;
1097
1098         ret = iscsit_setup_nop_out(conn, cmd, hdr);
1099         if (ret < 0)
1100                 return 0;
1101
1102         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1103                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1104                         pr_err("Unable to recover from"
1105                                " NOPOUT Ping DataCRC failure while in"
1106                                " ERL=0.\n");
1107                         ret = -1;
1108                         goto out;
1109                 } else {
1110                         /*
1111                          * drop this PDU and let the
1112                          * initiator plug the CmdSN gap.
1113                          */
1114                         pr_info("Dropping NOPOUT"
1115                                 " Command CmdSN: 0x%08x due to"
1116                                 " DataCRC error.\n", hdr->cmdsn);
1117                         ret = 0;
1118                         goto out;
1119                 }
1120         }
1121
1122         /*
1123          * Handle NOP-OUT payload for traditional iSCSI sockets
1124          */
1125         if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1126                 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1127                 if (!ping_data) {
1128                         pr_err("Unable to allocate memory for"
1129                                 " NOPOUT ping data.\n");
1130                         ret = -1;
1131                         goto out;
1132                 }
1133
1134                 skb_copy_bits(csk->skb, pdu_cb->doffset,
1135                               ping_data, payload_length);
1136
1137                 ping_data[payload_length] = '\0';
1138                 /*
1139                  * Attach ping data to struct iscsi_cmd->buf_ptr.
1140                  */
1141                 cmd->buf_ptr = ping_data;
1142                 cmd->buf_ptr_size = payload_length;
1143
1144                 pr_debug("Got %u bytes of NOPOUT ping"
1145                         " data.\n", payload_length);
1146                 pr_debug("Ping Data: \"%s\"\n", ping_data);
1147         }
1148
1149         return iscsit_process_nop_out(conn, cmd, hdr);
1150 out:
1151         if (cmd)
1152                 iscsit_free_cmd(cmd, false);
1153         return ret;
1154 }
1155
1156 static int
1157 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1158 {
1159         struct iscsi_conn *conn = csk->conn;
1160         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1161         struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr;
1162         u32 payload_length = pdu_cb->dlen;
1163         int rc;
1164         unsigned char *text_in = NULL;
1165
1166         rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1167         if (rc < 0)
1168                 return rc;
1169
1170         if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1171                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1172                         pr_err("Unable to recover from"
1173                                " Text Data digest failure while in"
1174                                " ERL=0.\n");
1175                         goto reject;
1176                 } else {
1177                         /*
1178                          * drop this PDU and let the
1179                          * initiator plug the CmdSN gap.
1180                          */
1181                         pr_info("Dropping Text"
1182                                 " Command CmdSN: 0x%08x due to"
1183                                 " DataCRC error.\n", hdr->cmdsn);
1184                         return 0;
1185                 }
1186         }
1187
1188         if (payload_length) {
1189                 text_in = kzalloc(payload_length, GFP_KERNEL);
1190                 if (!text_in) {
1191                         pr_err("Unable to allocate text_in of payload_length: %u\n",
1192                                payload_length);
1193                         return -ENOMEM;
1194                 }
1195                 skb_copy_bits(csk->skb, pdu_cb->doffset,
1196                               text_in, payload_length);
1197
1198                 text_in[payload_length - 1] = '\0';
1199
1200                 cmd->text_in_ptr = text_in;
1201         }
1202
1203         return iscsit_process_text_cmd(conn, cmd, hdr);
1204
1205 reject:
1206         return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1207                                  pdu_cb->hdr);
1208 }
1209
1210 static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk)
1211 {
1212         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1213         struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr;
1214         struct iscsi_conn *conn = csk->conn;
1215         struct iscsi_cmd *cmd = NULL;
1216         u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1217         int ret = -EINVAL;
1218
1219         switch (opcode) {
1220         case ISCSI_OP_SCSI_CMD:
1221                 cmd = cxgbit_allocate_cmd(csk);
1222                 if (!cmd)
1223                         goto reject;
1224
1225                 ret = cxgbit_handle_scsi_cmd(csk, cmd);
1226                 break;
1227         case ISCSI_OP_SCSI_DATA_OUT:
1228                 ret = cxgbit_handle_iscsi_dataout(csk);
1229                 break;
1230         case ISCSI_OP_NOOP_OUT:
1231                 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1232                         cmd = cxgbit_allocate_cmd(csk);
1233                         if (!cmd)
1234                                 goto reject;
1235                 }
1236
1237                 ret = cxgbit_handle_nop_out(csk, cmd);
1238                 break;
1239         case ISCSI_OP_SCSI_TMFUNC:
1240                 cmd = cxgbit_allocate_cmd(csk);
1241                 if (!cmd)
1242                         goto reject;
1243
1244                 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1245                                                  (unsigned char *)hdr);
1246                 break;
1247         case ISCSI_OP_TEXT:
1248                 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
1249                         cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
1250                         if (!cmd)
1251                                 goto reject;
1252                 } else {
1253                         cmd = cxgbit_allocate_cmd(csk);
1254                         if (!cmd)
1255                                 goto reject;
1256                 }
1257
1258                 ret = cxgbit_handle_text_cmd(csk, cmd);
1259                 break;
1260         case ISCSI_OP_LOGOUT:
1261                 cmd = cxgbit_allocate_cmd(csk);
1262                 if (!cmd)
1263                         goto reject;
1264
1265                 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1266                 if (ret > 0)
1267                         wait_for_completion_timeout(&conn->conn_logout_comp,
1268                                                     SECONDS_FOR_LOGOUT_COMP
1269                                                     * HZ);
1270                 break;
1271         case ISCSI_OP_SNACK:
1272                 ret = iscsit_handle_snack(conn, (unsigned char *)hdr);
1273                 break;
1274         default:
1275                 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1276                 dump_stack();
1277                 break;
1278         }
1279
1280         return ret;
1281
1282 reject:
1283         return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1284                                  (unsigned char *)hdr);
1285         return ret;
1286 }
1287
1288 static int cxgbit_rx_opcode(struct cxgbit_sock *csk)
1289 {
1290         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1291         struct iscsi_conn *conn = csk->conn;
1292         struct iscsi_hdr *hdr = pdu_cb->hdr;
1293         u8 opcode;
1294
1295         if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) {
1296                 atomic_long_inc(&conn->sess->conn_digest_errors);
1297                 goto transport_err;
1298         }
1299
1300         if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
1301                 goto transport_err;
1302
1303         opcode = hdr->opcode & ISCSI_OPCODE_MASK;
1304
1305         if (conn->sess->sess_ops->SessionType &&
1306             ((!(opcode & ISCSI_OP_TEXT)) ||
1307              (!(opcode & ISCSI_OP_LOGOUT)))) {
1308                 pr_err("Received illegal iSCSI Opcode: 0x%02x"
1309                         " while in Discovery Session, rejecting.\n", opcode);
1310                 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1311                                   (unsigned char *)hdr);
1312                 goto transport_err;
1313         }
1314
1315         if (cxgbit_target_rx_opcode(csk) < 0)
1316                 goto transport_err;
1317
1318         return 0;
1319
1320 transport_err:
1321         return -1;
1322 }
1323
1324 static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk)
1325 {
1326         struct iscsi_conn *conn = csk->conn;
1327         struct iscsi_login *login = conn->login;
1328         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1329         struct iscsi_login_req *login_req;
1330
1331         login_req = (struct iscsi_login_req *)login->req;
1332         memcpy(login_req, pdu_cb->hdr, sizeof(*login_req));
1333
1334         pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x,"
1335                 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n",
1336                 login_req->flags, login_req->itt, login_req->cmdsn,
1337                 login_req->exp_statsn, login_req->cid, pdu_cb->dlen);
1338         /*
1339          * Setup the initial iscsi_login values from the leading
1340          * login request PDU.
1341          */
1342         if (login->first_request) {
1343                 login_req = (struct iscsi_login_req *)login->req;
1344                 login->leading_connection = (!login_req->tsih) ? 1 : 0;
1345                 login->current_stage    = ISCSI_LOGIN_CURRENT_STAGE(
1346                                 login_req->flags);
1347                 login->version_min      = login_req->min_version;
1348                 login->version_max      = login_req->max_version;
1349                 memcpy(login->isid, login_req->isid, 6);
1350                 login->cmd_sn           = be32_to_cpu(login_req->cmdsn);
1351                 login->init_task_tag    = login_req->itt;
1352                 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1353                 login->cid              = be16_to_cpu(login_req->cid);
1354                 login->tsih             = be16_to_cpu(login_req->tsih);
1355         }
1356
1357         if (iscsi_target_check_login_request(conn, login) < 0)
1358                 return -1;
1359
1360         memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS);
1361         skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen);
1362
1363         return 0;
1364 }
1365
1366 static int
1367 cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx)
1368 {
1369         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx);
1370         int ret;
1371
1372         cxgbit_rx_pdu_cb(skb) = pdu_cb;
1373
1374         csk->skb = skb;
1375
1376         if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) {
1377                 ret = cxgbit_rx_login_pdu(csk);
1378                 set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1379         } else {
1380                 ret = cxgbit_rx_opcode(csk);
1381         }
1382
1383         return ret;
1384 }
1385
1386 static void cxgbit_lro_skb_dump(struct sk_buff *skb)
1387 {
1388         struct skb_shared_info *ssi = skb_shinfo(skb);
1389         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1390         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1391         u8 i;
1392
1393         pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n",
1394                 skb, skb->head, skb->data, skb->len, skb->data_len,
1395                 ssi->nr_frags);
1396         pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n",
1397                 skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen);
1398
1399         for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++)
1400                 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, "
1401                         "frags %u.\n",
1402                         skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq,
1403                         pdu_cb->ddigest, pdu_cb->frags);
1404         for (i = 0; i < ssi->nr_frags; i++)
1405                 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n",
1406                         skb, i, ssi->frags[i].page_offset, ssi->frags[i].size);
1407 }
1408
1409 static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk)
1410 {
1411         struct sk_buff *skb = csk->lro_hskb;
1412         struct skb_shared_info *ssi = skb_shinfo(skb);
1413         u8 i;
1414
1415         memset(skb->data, 0, LRO_SKB_MIN_HEADROOM);
1416         for (i = 0; i < ssi->nr_frags; i++)
1417                 put_page(skb_frag_page(&ssi->frags[i]));
1418         ssi->nr_frags = 0;
1419         skb->data_len = 0;
1420         skb->truesize -= skb->len;
1421         skb->len = 0;
1422 }
1423
1424 static void
1425 cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx)
1426 {
1427         struct sk_buff *hskb = csk->lro_hskb;
1428         struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0);
1429         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx);
1430         struct skb_shared_info *hssi = skb_shinfo(hskb);
1431         struct skb_shared_info *ssi = skb_shinfo(skb);
1432         unsigned int len = 0;
1433
1434         if (pdu_cb->flags & PDUCBF_RX_HDR) {
1435                 u8 hfrag_idx = hssi->nr_frags;
1436
1437                 hpdu_cb->flags |= pdu_cb->flags;
1438                 hpdu_cb->seq = pdu_cb->seq;
1439                 hpdu_cb->hdr = pdu_cb->hdr;
1440                 hpdu_cb->hlen = pdu_cb->hlen;
1441
1442                 memcpy(&hssi->frags[hfrag_idx], &ssi->frags[pdu_cb->hfrag_idx],
1443                        sizeof(skb_frag_t));
1444
1445                 get_page(skb_frag_page(&hssi->frags[hfrag_idx]));
1446                 hssi->nr_frags++;
1447                 hpdu_cb->frags++;
1448                 hpdu_cb->hfrag_idx = hfrag_idx;
1449
1450                 len = hssi->frags[hfrag_idx].size;
1451                 hskb->len += len;
1452                 hskb->data_len += len;
1453                 hskb->truesize += len;
1454         }
1455
1456         if (pdu_cb->flags & PDUCBF_RX_DATA) {
1457                 u8 dfrag_idx = hssi->nr_frags, i;
1458
1459                 hpdu_cb->flags |= pdu_cb->flags;
1460                 hpdu_cb->dfrag_idx = dfrag_idx;
1461
1462                 len = 0;
1463                 for (i = 0; i < pdu_cb->nr_dfrags; dfrag_idx++, i++) {
1464                         memcpy(&hssi->frags[dfrag_idx],
1465                                &ssi->frags[pdu_cb->dfrag_idx + i],
1466                                sizeof(skb_frag_t));
1467
1468                         get_page(skb_frag_page(&hssi->frags[dfrag_idx]));
1469
1470                         len += hssi->frags[dfrag_idx].size;
1471
1472                         hssi->nr_frags++;
1473                         hpdu_cb->frags++;
1474                 }
1475
1476                 hpdu_cb->dlen = pdu_cb->dlen;
1477                 hpdu_cb->doffset = hpdu_cb->hlen;
1478                 hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags;
1479                 hskb->len += len;
1480                 hskb->data_len += len;
1481                 hskb->truesize += len;
1482         }
1483
1484         if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1485                 hpdu_cb->flags |= pdu_cb->flags;
1486
1487                 if (hpdu_cb->flags & PDUCBF_RX_DATA)
1488                         hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD;
1489
1490                 hpdu_cb->ddigest = pdu_cb->ddigest;
1491                 hpdu_cb->pdulen = pdu_cb->pdulen;
1492         }
1493 }
1494
1495 static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1496 {
1497         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1498         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1499         u8 pdu_idx = 0, last_idx = 0;
1500         int ret = 0;
1501
1502         if (!pdu_cb->complete) {
1503                 cxgbit_lro_skb_merge(csk, skb, 0);
1504
1505                 if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1506                         struct sk_buff *hskb = csk->lro_hskb;
1507
1508                         ret = cxgbit_process_iscsi_pdu(csk, hskb, 0);
1509
1510                         cxgbit_lro_hskb_reset(csk);
1511
1512                         if (ret < 0)
1513                                 goto out;
1514                 }
1515
1516                 pdu_idx = 1;
1517         }
1518
1519         if (lro_cb->pdu_idx)
1520                 last_idx = lro_cb->pdu_idx - 1;
1521
1522         for (; pdu_idx <= last_idx; pdu_idx++) {
1523                 ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx);
1524                 if (ret < 0)
1525                         goto out;
1526         }
1527
1528         if ((!lro_cb->complete) && lro_cb->pdu_idx)
1529                 cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx);
1530
1531 out:
1532         return ret;
1533 }
1534
1535 static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1536 {
1537         struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1538         struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1539         int ret = -1;
1540
1541         if ((pdu_cb->flags & PDUCBF_RX_HDR) &&
1542             (pdu_cb->seq != csk->rcv_nxt)) {
1543                 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n",
1544                         csk, csk->tid, pdu_cb->seq, csk->rcv_nxt);
1545                 cxgbit_lro_skb_dump(skb);
1546                 return ret;
1547         }
1548
1549         csk->rcv_nxt += lro_cb->pdu_totallen;
1550
1551         ret = cxgbit_process_lro_skb(csk, skb);
1552
1553         csk->rx_credits += lro_cb->pdu_totallen;
1554
1555         if (csk->rx_credits >= (csk->rcv_win / 4))
1556                 cxgbit_rx_data_ack(csk);
1557
1558         return ret;
1559 }
1560
1561 static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1562 {
1563         struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
1564         int ret = -1;
1565
1566         if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) {
1567                 if (is_t5(lldi->adapter_type))
1568                         ret = cxgbit_rx_lro_skb(csk, skb);
1569                 else
1570                         ret = cxgbit_process_lro_skb(csk, skb);
1571         }
1572
1573         __kfree_skb(skb);
1574         return ret;
1575 }
1576
1577 static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq)
1578 {
1579         spin_lock_bh(&csk->rxq.lock);
1580         if (skb_queue_len(&csk->rxq)) {
1581                 skb_queue_splice_init(&csk->rxq, rxq);
1582                 spin_unlock_bh(&csk->rxq.lock);
1583                 return true;
1584         }
1585         spin_unlock_bh(&csk->rxq.lock);
1586         return false;
1587 }
1588
1589 static int cxgbit_wait_rxq(struct cxgbit_sock *csk)
1590 {
1591         struct sk_buff *skb;
1592         struct sk_buff_head rxq;
1593
1594         skb_queue_head_init(&rxq);
1595
1596         wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq));
1597
1598         if (signal_pending(current))
1599                 goto out;
1600
1601         while ((skb = __skb_dequeue(&rxq))) {
1602                 if (cxgbit_rx_skb(csk, skb))
1603                         goto out;
1604         }
1605
1606         return 0;
1607 out:
1608         __skb_queue_purge(&rxq);
1609         return -1;
1610 }
1611
1612 int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
1613 {
1614         struct cxgbit_sock *csk = conn->context;
1615         int ret = -1;
1616
1617         while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) {
1618                 ret = cxgbit_wait_rxq(csk);
1619                 if (ret) {
1620                         clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1621                         break;
1622                 }
1623         }
1624
1625         return ret;
1626 }
1627
1628 void cxgbit_get_rx_pdu(struct iscsi_conn *conn)
1629 {
1630         struct cxgbit_sock *csk = conn->context;
1631
1632         while (!kthread_should_stop()) {
1633                 iscsit_thread_check_cpumask(conn, current, 0);
1634                 if (cxgbit_wait_rxq(csk))
1635                         return;
1636         }
1637 }