drivers/net/ethernet/qlogic/qed/qed_iwarp.c

   1 /* QLogic qed NIC Driver
   2  * Copyright (c) 2015-2017  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/if_ether.h>
  33 #include <linux/if_vlan.h>
  34 #include <linux/ip.h>
  35 #include <linux/ipv6.h>
  36 #include <linux/spinlock.h>
  37 #include <linux/tcp.h>
  38 #include "qed_cxt.h"
  39 #include "qed_hw.h"
  40 #include "qed_ll2.h"
  41 #include "qed_rdma.h"
  42 #include "qed_reg_addr.h"
  43 #include "qed_sp.h"
  44
  45 #define QED_IWARP_ORD_DEFAULT           32
  46 #define QED_IWARP_IRD_DEFAULT           32
  47 #define QED_IWARP_MAX_FW_MSS            4120
  48
  49 #define QED_EP_SIG 0xecabcdef
  50
  51 struct mpa_v2_hdr {
  52         __be16 ird;
  53         __be16 ord;
  54 };
  55
  56 #define MPA_V2_PEER2PEER_MODEL  0x8000
  57 #define MPA_V2_SEND_RTR         0x4000  /* on ird */
  58 #define MPA_V2_READ_RTR         0x4000  /* on ord */
  59 #define MPA_V2_WRITE_RTR        0x8000
  60 #define MPA_V2_IRD_ORD_MASK     0x3FFF
  61
  62 #define MPA_REV2(_mpa_rev) ((_mpa_rev) == MPA_NEGOTIATION_TYPE_ENHANCED)
  63
  64 #define QED_IWARP_INVALID_TCP_CID       0xffffffff
  65 #define QED_IWARP_RCV_WND_SIZE_DEF      (256 * 1024)
  66 #define QED_IWARP_RCV_WND_SIZE_MIN      (64 * 1024)
  67 #define TIMESTAMP_HEADER_SIZE           (12)
  68
  69 #define QED_IWARP_TS_EN                 BIT(0)
  70 #define QED_IWARP_DA_EN                 BIT(1)
  71 #define QED_IWARP_PARAM_CRC_NEEDED      (1)
  72 #define QED_IWARP_PARAM_P2P             (1)
  73
  74 static int qed_iwarp_async_event(struct qed_hwfn *p_hwfn,
  75                                  u8 fw_event_code, u16 echo,
  76                                  union event_ring_data *data,
  77                                  u8 fw_return_code);
  78
  79 /* Override devinfo with iWARP specific values */
  80 void qed_iwarp_init_devinfo(struct qed_hwfn *p_hwfn)
  81 {
  82         struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev;
  83
  84         dev->max_inline = IWARP_REQ_MAX_INLINE_DATA_SIZE;
  85         dev->max_qp = min_t(u32,
  86                             IWARP_MAX_QPS,
  87                             p_hwfn->p_rdma_info->num_qps) -
  88                       QED_IWARP_PREALLOC_CNT;
  89
  90         dev->max_cq = dev->max_qp;
  91
  92         dev->max_qp_resp_rd_atomic_resc = QED_IWARP_IRD_DEFAULT;
  93         dev->max_qp_req_rd_atomic_resc = QED_IWARP_ORD_DEFAULT;
  94 }
  95
  96 void qed_iwarp_init_hw(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
  97 {
  98         p_hwfn->rdma_prs_search_reg = PRS_REG_SEARCH_TCP;
  99         qed_wr(p_hwfn, p_ptt, p_hwfn->rdma_prs_search_reg, 1);
 100         p_hwfn->b_rdma_enabled_in_prs = true;
 101 }
 102
 103 /* We have two cid maps, one for tcp which should be used only from passive
 104  * syn processing and replacing a pre-allocated ep in the list. The second
 105  * for active tcp and for QPs.
 106  */
 107 static void qed_iwarp_cid_cleaned(struct qed_hwfn *p_hwfn, u32 cid)
 108 {
 109         cid -= qed_cxt_get_proto_cid_start(p_hwfn, p_hwfn->p_rdma_info->proto);
 110
 111         spin_lock_bh(&p_hwfn->p_rdma_info->lock);
 112
 113         if (cid < QED_IWARP_PREALLOC_CNT)
 114                 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map,
 115                                     cid);
 116         else
 117                 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->cid_map, cid);
 118
 119         spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
 120 }
 121
 122 static int qed_iwarp_alloc_cid(struct qed_hwfn *p_hwfn, u32 *cid)
 123 {
 124         int rc;
 125
 126         spin_lock_bh(&p_hwfn->p_rdma_info->lock);
 127         rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_hwfn->p_rdma_info->cid_map, cid);
 128         spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
 129         if (rc) {
 130                 DP_NOTICE(p_hwfn, "Failed in allocating iwarp cid\n");
 131                 return rc;
 132         }
 133         *cid += qed_cxt_get_proto_cid_start(p_hwfn, p_hwfn->p_rdma_info->proto);
 134
 135         rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, *cid);
 136         if (rc)
 137                 qed_iwarp_cid_cleaned(p_hwfn, *cid);
 138
 139         return rc;
 140 }
 141
 142 static void qed_iwarp_set_tcp_cid(struct qed_hwfn *p_hwfn, u32 cid)
 143 {
 144         cid -= qed_cxt_get_proto_cid_start(p_hwfn, p_hwfn->p_rdma_info->proto);
 145
 146         spin_lock_bh(&p_hwfn->p_rdma_info->lock);
 147         qed_bmap_set_id(p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map, cid);
 148         spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
 149 }
 150
 151 /* This function allocates a cid for passive tcp (called from syn receive)
 152  * the reason it's separate from the regular cid allocation is because it
 153  * is assured that these cids already have ilt allocated. They are preallocated
 154  * to ensure that we won't need to allocate memory during syn processing
 155  */
 156 static int qed_iwarp_alloc_tcp_cid(struct qed_hwfn *p_hwfn, u32 *cid)
 157 {
 158         int rc;
 159
 160         spin_lock_bh(&p_hwfn->p_rdma_info->lock);
 161
 162         rc = qed_rdma_bmap_alloc_id(p_hwfn,
 163                                     &p_hwfn->p_rdma_info->tcp_cid_map, cid);
 164
 165         spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
 166
 167         if (rc) {
 168                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
 169                            "can't allocate iwarp tcp cid max-count=%d\n",
 170                            p_hwfn->p_rdma_info->tcp_cid_map.max_count);
 171
 172                 *cid = QED_IWARP_INVALID_TCP_CID;
 173                 return rc;
 174         }
 175
 176         *cid += qed_cxt_get_proto_cid_start(p_hwfn,
 177                                             p_hwfn->p_rdma_info->proto);
 178         return 0;
 179 }
 180
 181 int qed_iwarp_create_qp(struct qed_hwfn *p_hwfn,
 182                         struct qed_rdma_qp *qp,
 183                         struct qed_rdma_create_qp_out_params *out_params)
 184 {
 185         struct iwarp_create_qp_ramrod_data *p_ramrod;
 186         struct qed_sp_init_data init_data;
 187         struct qed_spq_entry *p_ent;
 188         u16 physical_queue;
 189         u32 cid;
 190         int rc;
 191
 192         qp->shared_queue = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
 193                                               IWARP_SHARED_QUEUE_PAGE_SIZE,
 194                                               &qp->shared_queue_phys_addr,
 195                                               GFP_KERNEL);
 196         if (!qp->shared_queue)
 197                 return -ENOMEM;
 198
 199         out_params->sq_pbl_virt = (u8 *)qp->shared_queue +
 200             IWARP_SHARED_QUEUE_PAGE_SQ_PBL_OFFSET;
 201         out_params->sq_pbl_phys = qp->shared_queue_phys_addr +
 202             IWARP_SHARED_QUEUE_PAGE_SQ_PBL_OFFSET;
 203         out_params->rq_pbl_virt = (u8 *)qp->shared_queue +
 204             IWARP_SHARED_QUEUE_PAGE_RQ_PBL_OFFSET;
 205         out_params->rq_pbl_phys = qp->shared_queue_phys_addr +
 206             IWARP_SHARED_QUEUE_PAGE_RQ_PBL_OFFSET;
 207
 208         rc = qed_iwarp_alloc_cid(p_hwfn, &cid);
 209         if (rc)
 210                 goto err1;
 211
 212         qp->icid = (u16)cid;
 213
 214         memset(&init_data, 0, sizeof(init_data));
 215         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
 216         init_data.cid = qp->icid;
 217         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
 218
 219         rc = qed_sp_init_request(p_hwfn, &p_ent,
 220                                  IWARP_RAMROD_CMD_ID_CREATE_QP,
 221                                  PROTOCOLID_IWARP, &init_data);
 222         if (rc)
 223                 goto err2;
 224
 225         p_ramrod = &p_ent->ramrod.iwarp_create_qp;
 226
 227         SET_FIELD(p_ramrod->flags,
 228                   IWARP_CREATE_QP_RAMROD_DATA_FMR_AND_RESERVED_EN,
 229                   qp->fmr_and_reserved_lkey);
 230
 231         SET_FIELD(p_ramrod->flags,
 232                   IWARP_CREATE_QP_RAMROD_DATA_SIGNALED_COMP, qp->signal_all);
 233
 234         SET_FIELD(p_ramrod->flags,
 235                   IWARP_CREATE_QP_RAMROD_DATA_RDMA_RD_EN,
 236                   qp->incoming_rdma_read_en);
 237
 238         SET_FIELD(p_ramrod->flags,
 239                   IWARP_CREATE_QP_RAMROD_DATA_RDMA_WR_EN,
 240                   qp->incoming_rdma_write_en);
 241
 242         SET_FIELD(p_ramrod->flags,
 243                   IWARP_CREATE_QP_RAMROD_DATA_ATOMIC_EN,
 244                   qp->incoming_atomic_en);
 245
 246         SET_FIELD(p_ramrod->flags,
 247                   IWARP_CREATE_QP_RAMROD_DATA_SRQ_FLG, qp->use_srq);
 248
 249         p_ramrod->pd = qp->pd;
 250         p_ramrod->sq_num_pages = qp->sq_num_pages;
 251         p_ramrod->rq_num_pages = qp->rq_num_pages;
 252
 253         p_ramrod->qp_handle_for_cqe.hi = cpu_to_le32(qp->qp_handle.hi);
 254         p_ramrod->qp_handle_for_cqe.lo = cpu_to_le32(qp->qp_handle.lo);
 255
 256         p_ramrod->cq_cid_for_sq =
 257             cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) | qp->sq_cq_id);
 258         p_ramrod->cq_cid_for_rq =
 259             cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) | qp->rq_cq_id);
 260
 261         p_ramrod->dpi = cpu_to_le16(qp->dpi);
 262
 263         physical_queue = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD);
 264         p_ramrod->physical_q0 = cpu_to_le16(physical_queue);
 265         physical_queue = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_ACK);
 266         p_ramrod->physical_q1 = cpu_to_le16(physical_queue);
 267
 268         rc = qed_spq_post(p_hwfn, p_ent, NULL);
 269         if (rc)
 270                 goto err2;
 271
 272         return rc;
 273
 274 err2:
 275         qed_iwarp_cid_cleaned(p_hwfn, cid);
 276 err1:
 277         dma_free_coherent(&p_hwfn->cdev->pdev->dev,
 278                           IWARP_SHARED_QUEUE_PAGE_SIZE,
 279                           qp->shared_queue, qp->shared_queue_phys_addr);
 280
 281         return rc;
 282 }
 283
 284 static int qed_iwarp_modify_fw(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp)
 285 {
 286         struct iwarp_modify_qp_ramrod_data *p_ramrod;
 287         struct qed_sp_init_data init_data;
 288         struct qed_spq_entry *p_ent;
 289         int rc;
 290
 291         /* Get SPQ entry */
 292         memset(&init_data, 0, sizeof(init_data));
 293         init_data.cid = qp->icid;
 294         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
 295         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
 296
 297         rc = qed_sp_init_request(p_hwfn, &p_ent,
 298                                  IWARP_RAMROD_CMD_ID_MODIFY_QP,
 299                                  p_hwfn->p_rdma_info->proto, &init_data);
 300         if (rc)
 301                 return rc;
 302
 303         p_ramrod = &p_ent->ramrod.iwarp_modify_qp;
 304         SET_FIELD(p_ramrod->flags, IWARP_MODIFY_QP_RAMROD_DATA_STATE_TRANS_EN,
 305                   0x1);
 306         if (qp->iwarp_state == QED_IWARP_QP_STATE_CLOSING)
 307                 p_ramrod->transition_to_state = IWARP_MODIFY_QP_STATE_CLOSING;
 308         else
 309                 p_ramrod->transition_to_state = IWARP_MODIFY_QP_STATE_ERROR;
 310
 311         rc = qed_spq_post(p_hwfn, p_ent, NULL);
 312
 313         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x)rc=%d\n", qp->icid, rc);
 314
 315         return rc;
 316 }
 317
 318 enum qed_iwarp_qp_state qed_roce2iwarp_state(enum qed_roce_qp_state state)
 319 {
 320         switch (state) {
 321         case QED_ROCE_QP_STATE_RESET:
 322         case QED_ROCE_QP_STATE_INIT:
 323         case QED_ROCE_QP_STATE_RTR:
 324                 return QED_IWARP_QP_STATE_IDLE;
 325         case QED_ROCE_QP_STATE_RTS:
 326                 return QED_IWARP_QP_STATE_RTS;
 327         case QED_ROCE_QP_STATE_SQD:
 328                 return QED_IWARP_QP_STATE_CLOSING;
 329         case QED_ROCE_QP_STATE_ERR:
 330                 return QED_IWARP_QP_STATE_ERROR;
 331         case QED_ROCE_QP_STATE_SQE:
 332                 return QED_IWARP_QP_STATE_TERMINATE;
 333         default:
 334                 return QED_IWARP_QP_STATE_ERROR;
 335         }
 336 }
 337
 338 static enum qed_roce_qp_state
 339 qed_iwarp2roce_state(enum qed_iwarp_qp_state state)
 340 {
 341         switch (state) {
 342         case QED_IWARP_QP_STATE_IDLE:
 343                 return QED_ROCE_QP_STATE_INIT;
 344         case QED_IWARP_QP_STATE_RTS:
 345                 return QED_ROCE_QP_STATE_RTS;
 346         case QED_IWARP_QP_STATE_TERMINATE:
 347                 return QED_ROCE_QP_STATE_SQE;
 348         case QED_IWARP_QP_STATE_CLOSING:
 349                 return QED_ROCE_QP_STATE_SQD;
 350         case QED_IWARP_QP_STATE_ERROR:
 351                 return QED_ROCE_QP_STATE_ERR;
 352         default:
 353                 return QED_ROCE_QP_STATE_ERR;
 354         }
 355 }
 356
 357 const char *iwarp_state_names[] = {
 358         "IDLE",
 359         "RTS",
 360         "TERMINATE",
 361         "CLOSING",
 362         "ERROR",
 363 };
 364
 365 int
 366 qed_iwarp_modify_qp(struct qed_hwfn *p_hwfn,
 367                     struct qed_rdma_qp *qp,
 368                     enum qed_iwarp_qp_state new_state, bool internal)
 369 {
 370         enum qed_iwarp_qp_state prev_iw_state;
 371         bool modify_fw = false;
 372         int rc = 0;
 373
 374         /* modify QP can be called from upper-layer or as a result of async
 375          * RST/FIN... therefore need to protect
 376          */
 377         spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.qp_lock);
 378         prev_iw_state = qp->iwarp_state;
 379
 380         if (prev_iw_state == new_state) {
 381                 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.qp_lock);
 382                 return 0;
 383         }
 384
 385         switch (prev_iw_state) {
 386         case QED_IWARP_QP_STATE_IDLE:
 387                 switch (new_state) {
 388                 case QED_IWARP_QP_STATE_RTS:
 389                         qp->iwarp_state = QED_IWARP_QP_STATE_RTS;
 390                         break;
 391                 case QED_IWARP_QP_STATE_ERROR:
 392                         qp->iwarp_state = QED_IWARP_QP_STATE_ERROR;
 393                         if (!internal)
 394                                 modify_fw = true;
 395                         break;
 396                 default:
 397                         break;
 398                 }
 399                 break;
 400         case QED_IWARP_QP_STATE_RTS:
 401                 switch (new_state) {
 402                 case QED_IWARP_QP_STATE_CLOSING:
 403                         if (!internal)
 404                                 modify_fw = true;
 405
 406                         qp->iwarp_state = QED_IWARP_QP_STATE_CLOSING;
 407                         break;
 408                 case QED_IWARP_QP_STATE_ERROR:
 409                         if (!internal)
 410                                 modify_fw = true;
 411                         qp->iwarp_state = QED_IWARP_QP_STATE_ERROR;
 412                         break;
 413                 default:
 414                         break;
 415                 }
 416                 break;
 417         case QED_IWARP_QP_STATE_ERROR:
 418                 switch (new_state) {
 419                 case QED_IWARP_QP_STATE_IDLE:
 420
 421                         qp->iwarp_state = new_state;
 422                         break;
 423                 case QED_IWARP_QP_STATE_CLOSING:
 424                         /* could happen due to race... do nothing.... */
 425                         break;
 426                 default:
 427                         rc = -EINVAL;
 428                 }
 429                 break;
 430         case QED_IWARP_QP_STATE_TERMINATE:
 431         case QED_IWARP_QP_STATE_CLOSING:
 432                 qp->iwarp_state = new_state;
 433                 break;
 434         default:
 435                 break;
 436         }
 437
 438         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x) %s --> %s%s\n",
 439                    qp->icid,
 440                    iwarp_state_names[prev_iw_state],
 441                    iwarp_state_names[qp->iwarp_state],
 442                    internal ? "internal" : "");
 443
 444         spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.qp_lock);
 445
 446         if (modify_fw)
 447                 rc = qed_iwarp_modify_fw(p_hwfn, qp);
 448
 449         return rc;
 450 }
 451
 452 int qed_iwarp_fw_destroy(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp)
 453 {
 454         struct qed_sp_init_data init_data;
 455         struct qed_spq_entry *p_ent;
 456         int rc;
 457
 458         /* Get SPQ entry */
 459         memset(&init_data, 0, sizeof(init_data));
 460         init_data.cid = qp->icid;
 461         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
 462         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
 463
 464         rc = qed_sp_init_request(p_hwfn, &p_ent,
 465                                  IWARP_RAMROD_CMD_ID_DESTROY_QP,
 466                                  p_hwfn->p_rdma_info->proto, &init_data);
 467         if (rc)
 468                 return rc;
 469
 470         rc = qed_spq_post(p_hwfn, p_ent, NULL);
 471
 472         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x) rc = %d\n", qp->icid, rc);
 473
 474         return rc;
 475 }
 476
 477 static void qed_iwarp_destroy_ep(struct qed_hwfn *p_hwfn,
 478                                  struct qed_iwarp_ep *ep,
 479                                  bool remove_from_active_list)
 480 {
 481         dma_free_coherent(&p_hwfn->cdev->pdev->dev,
 482                           sizeof(*ep->ep_buffer_virt),
 483                           ep->ep_buffer_virt, ep->ep_buffer_phys);
 484
 485         if (remove_from_active_list) {
 486                 spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
 487                 list_del(&ep->list_entry);
 488                 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
 489         }
 490
 491         if (ep->qp)
 492                 ep->qp->ep = NULL;
 493
 494         kfree(ep);
 495 }
 496
 497 int qed_iwarp_destroy_qp(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp)
 498 {
 499         struct qed_iwarp_ep *ep = qp->ep;
 500         int wait_count = 0;
 501         int rc = 0;
 502
 503         if (qp->iwarp_state != QED_IWARP_QP_STATE_ERROR) {
 504                 rc = qed_iwarp_modify_qp(p_hwfn, qp,
 505                                          QED_IWARP_QP_STATE_ERROR, false);
 506                 if (rc)
 507                         return rc;
 508         }
 509
 510         /* Make sure ep is closed before returning and freeing memory. */
 511         if (ep) {
 512                 while (ep->state != QED_IWARP_EP_CLOSED && wait_count++ < 200)
 513                         msleep(100);
 514
 515                 if (ep->state != QED_IWARP_EP_CLOSED)
 516                         DP_NOTICE(p_hwfn, "ep state close timeout state=%x\n",
 517                                   ep->state);
 518
 519                 qed_iwarp_destroy_ep(p_hwfn, ep, false);
 520         }
 521
 522         rc = qed_iwarp_fw_destroy(p_hwfn, qp);
 523
 524         if (qp->shared_queue)
 525                 dma_free_coherent(&p_hwfn->cdev->pdev->dev,
 526                                   IWARP_SHARED_QUEUE_PAGE_SIZE,
 527                                   qp->shared_queue, qp->shared_queue_phys_addr);
 528
 529         return rc;
 530 }
 531
 532 static int
 533 qed_iwarp_create_ep(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep **ep_out)
 534 {
 535         struct qed_iwarp_ep *ep;
 536         int rc;
 537
 538         ep = kzalloc(sizeof(*ep), GFP_KERNEL);
 539         if (!ep)
 540                 return -ENOMEM;
 541
 542         ep->state = QED_IWARP_EP_INIT;
 543
 544         ep->ep_buffer_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
 545                                                 sizeof(*ep->ep_buffer_virt),
 546                                                 &ep->ep_buffer_phys,
 547                                                 GFP_KERNEL);
 548         if (!ep->ep_buffer_virt) {
 549                 rc = -ENOMEM;
 550                 goto err;
 551         }
 552
 553         ep->sig = QED_EP_SIG;
 554
 555         *ep_out = ep;
 556
 557         return 0;
 558
 559 err:
 560         kfree(ep);
 561         return rc;
 562 }
 563
 564 static void
 565 qed_iwarp_print_tcp_ramrod(struct qed_hwfn *p_hwfn,
 566                            struct iwarp_tcp_offload_ramrod_data *p_tcp_ramrod)
 567 {
 568         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "local_mac=%x %x %x, remote_mac=%x %x %x\n",
 569                    p_tcp_ramrod->tcp.local_mac_addr_lo,
 570                    p_tcp_ramrod->tcp.local_mac_addr_mid,
 571                    p_tcp_ramrod->tcp.local_mac_addr_hi,
 572                    p_tcp_ramrod->tcp.remote_mac_addr_lo,
 573                    p_tcp_ramrod->tcp.remote_mac_addr_mid,
 574                    p_tcp_ramrod->tcp.remote_mac_addr_hi);
 575
 576         if (p_tcp_ramrod->tcp.ip_version == TCP_IPV4) {
 577                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
 578                            "local_ip=%pI4h:%x, remote_ip=%pI4h:%x, vlan=%x\n",
 579                            p_tcp_ramrod->tcp.local_ip,
 580                            p_tcp_ramrod->tcp.local_port,
 581                            p_tcp_ramrod->tcp.remote_ip,
 582                            p_tcp_ramrod->tcp.remote_port,
 583                            p_tcp_ramrod->tcp.vlan_id);
 584         } else {
 585                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
 586                            "local_ip=%pI6:%x, remote_ip=%pI6:%x, vlan=%x\n",
 587                            p_tcp_ramrod->tcp.local_ip,
 588                            p_tcp_ramrod->tcp.local_port,
 589                            p_tcp_ramrod->tcp.remote_ip,
 590                            p_tcp_ramrod->tcp.remote_port,
 591                            p_tcp_ramrod->tcp.vlan_id);
 592         }
 593
 594         DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
 595                    "flow_label=%x, ttl=%x, tos_or_tc=%x, mss=%x, rcv_wnd_scale=%x, connect_mode=%x, flags=%x\n",
 596                    p_tcp_ramrod->tcp.flow_label,
 597                    p_tcp_ramrod->tcp.ttl,
 598                    p_tcp_ramrod->tcp.tos_or_tc,
 599                    p_tcp_ramrod->tcp.mss,
 600                    p_tcp_ramrod->tcp.rcv_wnd_scale,
 601                    p_tcp_ramrod->tcp.connect_mode,
 602                    p_tcp_ramrod->tcp.flags);
 603
 604         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "syn_ip_payload_length=%x, lo=%x, hi=%x\n",
 605                    p_tcp_ramrod->tcp.syn_ip_payload_length,
 606                    p_tcp_ramrod->tcp.syn_phy_addr_lo,
 607                    p_tcp_ramrod->tcp.syn_phy_addr_hi);
 608 }
 609
 610 static int
 611 qed_iwarp_tcp_offload(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep *ep)
 612 {
 613         struct qed_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp;
 614         struct iwarp_tcp_offload_ramrod_data *p_tcp_ramrod;
 615         struct tcp_offload_params_opt2 *tcp;
 616         struct qed_sp_init_data init_data;
 617         struct qed_spq_entry *p_ent;
 618         dma_addr_t async_output_phys;
 619         dma_addr_t in_pdata_phys;
 620         u16 physical_q;
 621         u8 tcp_flags;
 622         int rc;
 623         int i;
 624
 625         memset(&init_data, 0, sizeof(init_data));
 626         init_data.cid = ep->tcp_cid;
 627         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
 628         if (ep->connect_mode == TCP_CONNECT_PASSIVE)
 629                 init_data.comp_mode = QED_SPQ_MODE_CB;
 630         else
 631                 init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
 632
 633         rc = qed_sp_init_request(p_hwfn, &p_ent,
 634                                  IWARP_RAMROD_CMD_ID_TCP_OFFLOAD,
 635                                  PROTOCOLID_IWARP, &init_data);
 636         if (rc)
 637                 return rc;
 638
 639         p_tcp_ramrod = &p_ent->ramrod.iwarp_tcp_offload;
 640
 641         in_pdata_phys = ep->ep_buffer_phys +
 642                         offsetof(struct qed_iwarp_ep_memory, in_pdata);
 643         DMA_REGPAIR_LE(p_tcp_ramrod->iwarp.incoming_ulp_buffer.addr,
 644                        in_pdata_phys);
 645
 646         p_tcp_ramrod->iwarp.incoming_ulp_buffer.len =
 647             cpu_to_le16(sizeof(ep->ep_buffer_virt->in_pdata));
 648
 649         async_output_phys = ep->ep_buffer_phys +
 650                             offsetof(struct qed_iwarp_ep_memory, async_output);
 651         DMA_REGPAIR_LE(p_tcp_ramrod->iwarp.async_eqe_output_buf,
 652                        async_output_phys);
 653
 654         p_tcp_ramrod->iwarp.handle_for_async.hi = cpu_to_le32(PTR_HI(ep));
 655         p_tcp_ramrod->iwarp.handle_for_async.lo = cpu_to_le32(PTR_LO(ep));
 656
 657         physical_q = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD);
 658         p_tcp_ramrod->iwarp.physical_q0 = cpu_to_le16(physical_q);
 659         physical_q = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_ACK);
 660         p_tcp_ramrod->iwarp.physical_q1 = cpu_to_le16(physical_q);
 661         p_tcp_ramrod->iwarp.mpa_mode = iwarp_info->mpa_rev;
 662
 663         tcp = &p_tcp_ramrod->tcp;
 664         qed_set_fw_mac_addr(&tcp->remote_mac_addr_hi,
 665                             &tcp->remote_mac_addr_mid,
 666                             &tcp->remote_mac_addr_lo, ep->remote_mac_addr);
 667         qed_set_fw_mac_addr(&tcp->local_mac_addr_hi, &tcp->local_mac_addr_mid,
 668                             &tcp->local_mac_addr_lo, ep->local_mac_addr);
 669
 670         tcp->vlan_id = cpu_to_le16(ep->cm_info.vlan);
 671
 672         tcp_flags = p_hwfn->p_rdma_info->iwarp.tcp_flags;
 673         tcp->flags = 0;
 674         SET_FIELD(tcp->flags, TCP_OFFLOAD_PARAMS_OPT2_TS_EN,
 675                   !!(tcp_flags & QED_IWARP_TS_EN));
 676
 677         SET_FIELD(tcp->flags, TCP_OFFLOAD_PARAMS_OPT2_DA_EN,
 678                   !!(tcp_flags & QED_IWARP_DA_EN));
 679
 680         tcp->ip_version = ep->cm_info.ip_version;
 681
 682         for (i = 0; i < 4; i++) {
 683                 tcp->remote_ip[i] = cpu_to_le32(ep->cm_info.remote_ip[i]);
 684                 tcp->local_ip[i] = cpu_to_le32(ep->cm_info.local_ip[i]);
 685         }
 686
 687         tcp->remote_port = cpu_to_le16(ep->cm_info.remote_port);
 688         tcp->local_port = cpu_to_le16(ep->cm_info.local_port);
 689         tcp->mss = cpu_to_le16(ep->mss);
 690         tcp->flow_label = 0;
 691         tcp->ttl = 0x40;
 692         tcp->tos_or_tc = 0;
 693
 694         tcp->rcv_wnd_scale = (u8)p_hwfn->p_rdma_info->iwarp.rcv_wnd_scale;
 695         tcp->connect_mode = ep->connect_mode;
 696
 697         if (ep->connect_mode == TCP_CONNECT_PASSIVE) {
 698                 tcp->syn_ip_payload_length =
 699                         cpu_to_le16(ep->syn_ip_payload_length);
 700                 tcp->syn_phy_addr_hi = DMA_HI_LE(ep->syn_phy_addr);
 701                 tcp->syn_phy_addr_lo = DMA_LO_LE(ep->syn_phy_addr);
 702         }
 703
 704         qed_iwarp_print_tcp_ramrod(p_hwfn, p_tcp_ramrod);
 705
 706         rc = qed_spq_post(p_hwfn, p_ent, NULL);
 707
 708         DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
 709                    "EP(0x%x) Offload completed rc=%d\n", ep->tcp_cid, rc);
 710
 711         return rc;
 712 }
 713
 714 static void
 715 qed_iwarp_mpa_received(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep *ep)
 716 {
 717         struct qed_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp;
 718         struct qed_iwarp_cm_event_params params;
 719         struct mpa_v2_hdr *mpa_v2;
 720         union async_output *async_data;
 721         u16 mpa_ord, mpa_ird;
 722         u8 mpa_hdr_size = 0;
 723         u8 mpa_rev;
 724
 725         async_data = &ep->ep_buffer_virt->async_output;
 726
 727         mpa_rev = async_data->mpa_request.mpa_handshake_mode;
 728         DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
 729                    "private_data_len=%x handshake_mode=%x private_data=(%x)\n",
 730                    async_data->mpa_request.ulp_data_len,
 731                    mpa_rev, *((u32 *)(ep->ep_buffer_virt->in_pdata)));
 732
 733         if (mpa_rev == MPA_NEGOTIATION_TYPE_ENHANCED) {
 734                 /* Read ord/ird values from private data buffer */
 735                 mpa_v2 = (struct mpa_v2_hdr *)ep->ep_buffer_virt->in_pdata;
 736                 mpa_hdr_size = sizeof(*mpa_v2);
 737
 738                 mpa_ord = ntohs(mpa_v2->ord);
 739                 mpa_ird = ntohs(mpa_v2->ird);
 740
 741                 /* Temprary store in cm_info incoming ord/ird requested, later
 742                  * replace with negotiated value during accept
 743                  */
 744                 ep->cm_info.ord = (u8)min_t(u16,
 745                                             (mpa_ord & MPA_V2_IRD_ORD_MASK),
 746                                             QED_IWARP_ORD_DEFAULT);
 747
 748                 ep->cm_info.ird = (u8)min_t(u16,
 749                                             (mpa_ird & MPA_V2_IRD_ORD_MASK),
 750                                             QED_IWARP_IRD_DEFAULT);
 751
 752                 /* Peer2Peer negotiation */
 753                 ep->rtr_type = MPA_RTR_TYPE_NONE;
 754                 if (mpa_ird & MPA_V2_PEER2PEER_MODEL) {
 755                         if (mpa_ord & MPA_V2_WRITE_RTR)
 756                                 ep->rtr_type |= MPA_RTR_TYPE_ZERO_WRITE;
 757
 758                         if (mpa_ord & MPA_V2_READ_RTR)
 759                                 ep->rtr_type |= MPA_RTR_TYPE_ZERO_READ;
 760
 761                         if (mpa_ird & MPA_V2_SEND_RTR)
 762                                 ep->rtr_type |= MPA_RTR_TYPE_ZERO_SEND;
 763
 764                         ep->rtr_type &= iwarp_info->rtr_type;
 765
 766                         /* if we're left with no match send our capabilities */
 767                         if (ep->rtr_type == MPA_RTR_TYPE_NONE)
 768                                 ep->rtr_type = iwarp_info->rtr_type;
 769                 }
 770
 771                 ep->mpa_rev = MPA_NEGOTIATION_TYPE_ENHANCED;
 772         } else {
 773                 ep->cm_info.ord = QED_IWARP_ORD_DEFAULT;
 774                 ep->cm_info.ird = QED_IWARP_IRD_DEFAULT;
 775                 ep->mpa_rev = MPA_NEGOTIATION_TYPE_BASIC;
 776         }
 777
 778         DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
 779                    "MPA_NEGOTIATE (v%d): ORD: 0x%x IRD: 0x%x rtr:0x%x ulp_data_len = %x mpa_hdr_size = %x\n",
 780                    mpa_rev, ep->cm_info.ord, ep->cm_info.ird, ep->rtr_type,
 781                    async_data->mpa_request.ulp_data_len, mpa_hdr_size);
 782
 783         /* Strip mpa v2 hdr from private data before sending to upper layer */
 784         ep->cm_info.private_data = ep->ep_buffer_virt->in_pdata + mpa_hdr_size;
 785
 786         ep->cm_info.private_data_len = async_data->mpa_request.ulp_data_len -
 787                                        mpa_hdr_size;
 788
 789         params.event = QED_IWARP_EVENT_MPA_REQUEST;
 790         params.cm_info = &ep->cm_info;
 791         params.ep_context = ep;
 792         params.status = 0;
 793
 794         ep->state = QED_IWARP_EP_MPA_REQ_RCVD;
 795         ep->event_cb(ep->cb_context, &params);
 796 }
 797
 798 static int
 799 qed_iwarp_mpa_offload(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep *ep)
 800 {
 801         struct iwarp_mpa_offload_ramrod_data *p_mpa_ramrod;
 802         struct qed_sp_init_data init_data;
 803         dma_addr_t async_output_phys;
 804         struct qed_spq_entry *p_ent;
 805         dma_addr_t out_pdata_phys;
 806         dma_addr_t in_pdata_phys;
 807         struct qed_rdma_qp *qp;
 808         bool reject;
 809         int rc;
 810
 811         if (!ep)
 812                 return -EINVAL;
 813
 814         qp = ep->qp;
 815         reject = !qp;
 816
 817         memset(&init_data, 0, sizeof(init_data));
 818         init_data.cid = reject ? ep->tcp_cid : qp->icid;
 819         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
 820
 821         if (ep->connect_mode == TCP_CONNECT_ACTIVE)
 822                 init_data.comp_mode = QED_SPQ_MODE_CB;
 823         else
 824                 init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
 825
 826         rc = qed_sp_init_request(p_hwfn, &p_ent,
 827                                  IWARP_RAMROD_CMD_ID_MPA_OFFLOAD,
 828                                  PROTOCOLID_IWARP, &init_data);
 829         if (rc)
 830                 return rc;
 831
 832         p_mpa_ramrod = &p_ent->ramrod.iwarp_mpa_offload;
 833         out_pdata_phys = ep->ep_buffer_phys +
 834                          offsetof(struct qed_iwarp_ep_memory, out_pdata);
 835         DMA_REGPAIR_LE(p_mpa_ramrod->common.outgoing_ulp_buffer.addr,
 836                        out_pdata_phys);
 837         p_mpa_ramrod->common.outgoing_ulp_buffer.len =
 838             ep->cm_info.private_data_len;
 839         p_mpa_ramrod->common.crc_needed = p_hwfn->p_rdma_info->iwarp.crc_needed;
 840
 841         p_mpa_ramrod->common.out_rq.ord = ep->cm_info.ord;
 842         p_mpa_ramrod->common.out_rq.ird = ep->cm_info.ird;
 843
 844         p_mpa_ramrod->tcp_cid = p_hwfn->hw_info.opaque_fid << 16 | ep->tcp_cid;
 845
 846         in_pdata_phys = ep->ep_buffer_phys +
 847                         offsetof(struct qed_iwarp_ep_memory, in_pdata);
 848         p_mpa_ramrod->tcp_connect_side = ep->connect_mode;
 849         DMA_REGPAIR_LE(p_mpa_ramrod->incoming_ulp_buffer.addr,
 850                        in_pdata_phys);
 851         p_mpa_ramrod->incoming_ulp_buffer.len =
 852             cpu_to_le16(sizeof(ep->ep_buffer_virt->in_pdata));
 853         async_output_phys = ep->ep_buffer_phys +
 854                             offsetof(struct qed_iwarp_ep_memory, async_output);
 855         DMA_REGPAIR_LE(p_mpa_ramrod->async_eqe_output_buf,
 856                        async_output_phys);
 857         p_mpa_ramrod->handle_for_async.hi = cpu_to_le32(PTR_HI(ep));
 858         p_mpa_ramrod->handle_for_async.lo = cpu_to_le32(PTR_LO(ep));
 859
 860         if (!reject) {
 861                 DMA_REGPAIR_LE(p_mpa_ramrod->shared_queue_addr,
 862                                qp->shared_queue_phys_addr);
 863                 p_mpa_ramrod->stats_counter_id =
 864                     RESC_START(p_hwfn, QED_RDMA_STATS_QUEUE) + qp->stats_queue;
 865         } else {
 866                 p_mpa_ramrod->common.reject = 1;
 867         }
 868
 869         p_mpa_ramrod->mode = ep->mpa_rev;
 870         SET_FIELD(p_mpa_ramrod->rtr_pref,
 871                   IWARP_MPA_OFFLOAD_RAMROD_DATA_RTR_SUPPORTED, ep->rtr_type);
 872
 873         ep->state = QED_IWARP_EP_MPA_OFFLOADED;
 874         rc = qed_spq_post(p_hwfn, p_ent, NULL);
 875         if (!reject)
 876                 ep->cid = qp->icid;     /* Now they're migrated. */
 877
 878         DP_VERBOSE(p_hwfn,
 879                    QED_MSG_RDMA,
 880                    "QP(0x%x) EP(0x%x) MPA Offload rc = %d IRD=0x%x ORD=0x%x rtr_type=%d mpa_rev=%d reject=%d\n",
 881                    reject ? 0xffff : qp->icid,
 882                    ep->tcp_cid,
 883                    rc,
 884                    ep->cm_info.ird,
 885                    ep->cm_info.ord, ep->rtr_type, ep->mpa_rev, reject);
 886         return rc;
 887 }
 888
 889 static void
 890 qed_iwarp_return_ep(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep *ep)
 891 {
 892         ep->state = QED_IWARP_EP_INIT;
 893         if (ep->qp)
 894                 ep->qp->ep = NULL;
 895         ep->qp = NULL;
 896         memset(&ep->cm_info, 0, sizeof(ep->cm_info));
 897
 898         if (ep->tcp_cid == QED_IWARP_INVALID_TCP_CID) {
 899                 /* We don't care about the return code, it's ok if tcp_cid
 900                  * remains invalid...in this case we'll defer allocation
 901                  */
 902                 qed_iwarp_alloc_tcp_cid(p_hwfn, &ep->tcp_cid);
 903         }
 904         spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
 905
 906         list_del(&ep->list_entry);
 907         list_add_tail(&ep->list_entry,
 908                       &p_hwfn->p_rdma_info->iwarp.ep_free_list);
 909
 910         spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
 911 }
 912
 913 void
 914 qed_iwarp_parse_private_data(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep *ep)
 915 {
 916         struct mpa_v2_hdr *mpa_v2_params;
 917         union async_output *async_data;
 918         u16 mpa_ird, mpa_ord;
 919         u8 mpa_data_size = 0;
 920
 921         if (MPA_REV2(p_hwfn->p_rdma_info->iwarp.mpa_rev)) {
 922                 mpa_v2_params =
 923                         (struct mpa_v2_hdr *)(ep->ep_buffer_virt->in_pdata);
 924                 mpa_data_size = sizeof(*mpa_v2_params);
 925                 mpa_ird = ntohs(mpa_v2_params->ird);
 926                 mpa_ord = ntohs(mpa_v2_params->ord);
 927
 928                 ep->cm_info.ird = (u8)(mpa_ord & MPA_V2_IRD_ORD_MASK);
 929                 ep->cm_info.ord = (u8)(mpa_ird & MPA_V2_IRD_ORD_MASK);
 930         }
 931         async_data = &ep->ep_buffer_virt->async_output;
 932
 933         ep->cm_info.private_data = ep->ep_buffer_virt->in_pdata + mpa_data_size;
 934         ep->cm_info.private_data_len = async_data->mpa_response.ulp_data_len -
 935                                        mpa_data_size;
 936 }
 937
 938 void
 939 qed_iwarp_mpa_reply_arrived(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep *ep)
 940 {
 941         struct qed_iwarp_cm_event_params params;
 942
 943         if (ep->connect_mode == TCP_CONNECT_PASSIVE) {
 944                 DP_NOTICE(p_hwfn,
 945                           "MPA reply event not expected on passive side!\n");
 946                 return;
 947         }
 948
 949         params.event = QED_IWARP_EVENT_ACTIVE_MPA_REPLY;
 950
 951         qed_iwarp_parse_private_data(p_hwfn, ep);
 952
 953         DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
 954                    "MPA_NEGOTIATE (v%d): ORD: 0x%x IRD: 0x%x\n",
 955                    ep->mpa_rev, ep->cm_info.ord, ep->cm_info.ird);
 956
 957         params.cm_info = &ep->cm_info;
 958         params.ep_context = ep;
 959         params.status = 0;
 960
 961         ep->mpa_reply_processed = true;
 962
 963         ep->event_cb(ep->cb_context, &params);
 964 }
 965
 966 #define QED_IWARP_CONNECT_MODE_STRING(ep) \
 967         ((ep)->connect_mode == TCP_CONNECT_PASSIVE) ? "Passive" : "Active"
 968
 969 /* Called as a result of the event:
 970  * IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_COMPLETE
 971  */
 972 static void
 973 qed_iwarp_mpa_complete(struct qed_hwfn *p_hwfn,
 974                        struct qed_iwarp_ep *ep, u8 fw_return_code)
 975 {
 976         struct qed_iwarp_cm_event_params params;
 977
 978         if (ep->connect_mode == TCP_CONNECT_ACTIVE)
 979                 params.event = QED_IWARP_EVENT_ACTIVE_COMPLETE;
 980         else
 981                 params.event = QED_IWARP_EVENT_PASSIVE_COMPLETE;
 982
 983         if (ep->connect_mode == TCP_CONNECT_ACTIVE && !ep->mpa_reply_processed)
 984                 qed_iwarp_parse_private_data(p_hwfn, ep);
 985
 986         DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
 987                    "MPA_NEGOTIATE (v%d): ORD: 0x%x IRD: 0x%x\n",
 988                    ep->mpa_rev, ep->cm_info.ord, ep->cm_info.ird);
 989
 990         params.cm_info = &ep->cm_info;
 991
 992         params.ep_context = ep;
 993
 994         ep->state = QED_IWARP_EP_CLOSED;
 995
 996         switch (fw_return_code) {
 997         case RDMA_RETURN_OK:
 998                 ep->qp->max_rd_atomic_req = ep->cm_info.ord;
 999                 ep->qp->max_rd_atomic_resp = ep->cm_info.ird;
1000                 qed_iwarp_modify_qp(p_hwfn, ep->qp, QED_IWARP_QP_STATE_RTS, 1);
1001                 ep->state = QED_IWARP_EP_ESTABLISHED;
1002                 params.status = 0;
1003                 break;
1004         case IWARP_CONN_ERROR_MPA_TIMEOUT:
1005                 DP_NOTICE(p_hwfn, "%s(0x%x) MPA timeout\n",
1006                           QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid);
1007                 params.status = -EBUSY;
1008                 break;
1009         case IWARP_CONN_ERROR_MPA_ERROR_REJECT:
1010                 DP_NOTICE(p_hwfn, "%s(0x%x) MPA Reject\n",
1011                           QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid);
1012                 params.status = -ECONNREFUSED;
1013                 break;
1014         case IWARP_CONN_ERROR_MPA_RST:
1015                 DP_NOTICE(p_hwfn, "%s(0x%x) MPA reset(tcp cid: 0x%x)\n",
1016                           QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid,
1017                           ep->tcp_cid);
1018                 params.status = -ECONNRESET;
1019                 break;
1020         case IWARP_CONN_ERROR_MPA_FIN:
1021                 DP_NOTICE(p_hwfn, "%s(0x%x) MPA received FIN\n",
1022                           QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid);
1023                 params.status = -ECONNREFUSED;
1024                 break;
1025         case IWARP_CONN_ERROR_MPA_INSUF_IRD:
1026                 DP_NOTICE(p_hwfn, "%s(0x%x) MPA insufficient ird\n",
1027                           QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid);
1028                 params.status = -ECONNREFUSED;
1029                 break;
1030         case IWARP_CONN_ERROR_MPA_RTR_MISMATCH:
1031                 DP_NOTICE(p_hwfn, "%s(0x%x) MPA RTR MISMATCH\n",
1032                           QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid);
1033                 params.status = -ECONNREFUSED;
1034                 break;
1035         case IWARP_CONN_ERROR_MPA_INVALID_PACKET:
1036                 DP_NOTICE(p_hwfn, "%s(0x%x) MPA Invalid Packet\n",
1037                           QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid);
1038                 params.status = -ECONNREFUSED;
1039                 break;
1040         case IWARP_CONN_ERROR_MPA_LOCAL_ERROR:
1041                 DP_NOTICE(p_hwfn, "%s(0x%x) MPA Local Error\n",
1042                           QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid);
1043                 params.status = -ECONNREFUSED;
1044                 break;
1045         case IWARP_CONN_ERROR_MPA_TERMINATE:
1046                 DP_NOTICE(p_hwfn, "%s(0x%x) MPA TERMINATE\n",
1047                           QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid);
1048                 params.status = -ECONNREFUSED;
1049                 break;
1050         default:
1051                 params.status = -ECONNRESET;
1052                 break;
1053         }
1054
1055         ep->event_cb(ep->cb_context, &params);
1056
1057         /* on passive side, if there is no associated QP (REJECT) we need to
1058          * return the ep to the pool, (in the regular case we add an element
1059          * in accept instead of this one.
1060          * In both cases we need to remove it from the ep_list.
1061          */
1062         if (fw_return_code != RDMA_RETURN_OK) {
1063                 ep->tcp_cid = QED_IWARP_INVALID_TCP_CID;
1064                 if ((ep->connect_mode == TCP_CONNECT_PASSIVE) &&
1065                     (!ep->qp)) {        /* Rejected */
1066                         qed_iwarp_return_ep(p_hwfn, ep);
1067                 } else {
1068                         spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
1069                         list_del(&ep->list_entry);
1070                         spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
1071                 }
1072         }
1073 }
1074
1075 static void
1076 qed_iwarp_mpa_v2_set_private(struct qed_hwfn *p_hwfn,
1077                              struct qed_iwarp_ep *ep, u8 *mpa_data_size)
1078 {
1079         struct mpa_v2_hdr *mpa_v2_params;
1080         u16 mpa_ird, mpa_ord;
1081
1082         *mpa_data_size = 0;
1083         if (MPA_REV2(ep->mpa_rev)) {
1084                 mpa_v2_params =
1085                     (struct mpa_v2_hdr *)ep->ep_buffer_virt->out_pdata;
1086                 *mpa_data_size = sizeof(*mpa_v2_params);
1087
1088                 mpa_ird = (u16)ep->cm_info.ird;
1089                 mpa_ord = (u16)ep->cm_info.ord;
1090
1091                 if (ep->rtr_type != MPA_RTR_TYPE_NONE) {
1092                         mpa_ird |= MPA_V2_PEER2PEER_MODEL;
1093
1094                         if (ep->rtr_type & MPA_RTR_TYPE_ZERO_SEND)
1095                                 mpa_ird |= MPA_V2_SEND_RTR;
1096
1097                         if (ep->rtr_type & MPA_RTR_TYPE_ZERO_WRITE)
1098                                 mpa_ord |= MPA_V2_WRITE_RTR;
1099
1100                         if (ep->rtr_type & MPA_RTR_TYPE_ZERO_READ)
1101                                 mpa_ord |= MPA_V2_READ_RTR;
1102                 }
1103
1104                 mpa_v2_params->ird = htons(mpa_ird);
1105                 mpa_v2_params->ord = htons(mpa_ord);
1106
1107                 DP_VERBOSE(p_hwfn,
1108                            QED_MSG_RDMA,
1109                            "MPA_NEGOTIATE Header: [%x ord:%x ird] %x ord:%x ird:%x peer2peer:%x rtr_send:%x rtr_write:%x rtr_read:%x\n",
1110                            mpa_v2_params->ird,
1111                            mpa_v2_params->ord,
1112                            *((u32 *)mpa_v2_params),
1113                            mpa_ord & MPA_V2_IRD_ORD_MASK,
1114                            mpa_ird & MPA_V2_IRD_ORD_MASK,
1115                            !!(mpa_ird & MPA_V2_PEER2PEER_MODEL),
1116                            !!(mpa_ird & MPA_V2_SEND_RTR),
1117                            !!(mpa_ord & MPA_V2_WRITE_RTR),
1118                            !!(mpa_ord & MPA_V2_READ_RTR));
1119         }
1120 }
1121
1122 int qed_iwarp_connect(void *rdma_cxt,
1123                       struct qed_iwarp_connect_in *iparams,
1124                       struct qed_iwarp_connect_out *oparams)
1125 {
1126         struct qed_hwfn *p_hwfn = rdma_cxt;
1127         struct qed_iwarp_info *iwarp_info;
1128         struct qed_iwarp_ep *ep;
1129         u8 mpa_data_size = 0;
1130         u8 ts_hdr_size = 0;
1131         u32 cid;
1132         int rc;
1133
1134         if ((iparams->cm_info.ord > QED_IWARP_ORD_DEFAULT) ||
1135             (iparams->cm_info.ird > QED_IWARP_IRD_DEFAULT)) {
1136                 DP_NOTICE(p_hwfn,
1137                           "QP(0x%x) ERROR: Invalid ord(0x%x)/ird(0x%x)\n",
1138                           iparams->qp->icid, iparams->cm_info.ord,
1139                           iparams->cm_info.ird);
1140
1141                 return -EINVAL;
1142         }
1143
1144         iwarp_info = &p_hwfn->p_rdma_info->iwarp;
1145
1146         /* Allocate ep object */
1147         rc = qed_iwarp_alloc_cid(p_hwfn, &cid);
1148         if (rc)
1149                 return rc;
1150
1151         rc = qed_iwarp_create_ep(p_hwfn, &ep);
1152         if (rc)
1153                 goto err;
1154
1155         ep->tcp_cid = cid;
1156
1157         spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
1158         list_add_tail(&ep->list_entry, &p_hwfn->p_rdma_info->iwarp.ep_list);
1159         spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
1160
1161         ep->qp = iparams->qp;
1162         ep->qp->ep = ep;
1163         ether_addr_copy(ep->remote_mac_addr, iparams->remote_mac_addr);
1164         ether_addr_copy(ep->local_mac_addr, iparams->local_mac_addr);
1165         memcpy(&ep->cm_info, &iparams->cm_info, sizeof(ep->cm_info));
1166
1167         ep->cm_info.ord = iparams->cm_info.ord;
1168         ep->cm_info.ird = iparams->cm_info.ird;
1169
1170         ep->rtr_type = iwarp_info->rtr_type;
1171         if (!iwarp_info->peer2peer)
1172                 ep->rtr_type = MPA_RTR_TYPE_NONE;
1173
1174         if ((ep->rtr_type & MPA_RTR_TYPE_ZERO_READ) && (ep->cm_info.ord == 0))
1175                 ep->cm_info.ord = 1;
1176
1177         ep->mpa_rev = iwarp_info->mpa_rev;
1178
1179         qed_iwarp_mpa_v2_set_private(p_hwfn, ep, &mpa_data_size);
1180
1181         ep->cm_info.private_data = ep->ep_buffer_virt->out_pdata;
1182         ep->cm_info.private_data_len = iparams->cm_info.private_data_len +
1183                                        mpa_data_size;
1184
1185         memcpy((u8 *)ep->ep_buffer_virt->out_pdata + mpa_data_size,
1186                iparams->cm_info.private_data,
1187                iparams->cm_info.private_data_len);
1188
1189         if (p_hwfn->p_rdma_info->iwarp.tcp_flags & QED_IWARP_TS_EN)
1190                 ts_hdr_size = TIMESTAMP_HEADER_SIZE;
1191
1192         ep->mss = iparams->mss - ts_hdr_size;
1193         ep->mss = min_t(u16, QED_IWARP_MAX_FW_MSS, ep->mss);
1194
1195         ep->event_cb = iparams->event_cb;
1196         ep->cb_context = iparams->cb_context;
1197         ep->connect_mode = TCP_CONNECT_ACTIVE;
1198
1199         oparams->ep_context = ep;
1200
1201         rc = qed_iwarp_tcp_offload(p_hwfn, ep);
1202
1203         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x) EP(0x%x) rc = %d\n",
1204                    iparams->qp->icid, ep->tcp_cid, rc);
1205
1206         if (rc) {
1207                 qed_iwarp_destroy_ep(p_hwfn, ep, true);
1208                 goto err;
1209         }
1210
1211         return rc;
1212 err:
1213         qed_iwarp_cid_cleaned(p_hwfn, cid);
1214
1215         return rc;
1216 }
1217
1218 static struct qed_iwarp_ep *qed_iwarp_get_free_ep(struct qed_hwfn *p_hwfn)
1219 {
1220         struct qed_iwarp_ep *ep = NULL;
1221         int rc;
1222
1223         spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
1224
1225         if (list_empty(&p_hwfn->p_rdma_info->iwarp.ep_free_list)) {
1226                 DP_ERR(p_hwfn, "Ep list is empty\n");
1227                 goto out;
1228         }
1229
1230         ep = list_first_entry(&p_hwfn->p_rdma_info->iwarp.ep_free_list,
1231                               struct qed_iwarp_ep, list_entry);
1232
1233         /* in some cases we could have failed allocating a tcp cid when added
1234          * from accept / failure... retry now..this is not the common case.
1235          */
1236         if (ep->tcp_cid == QED_IWARP_INVALID_TCP_CID) {
1237                 rc = qed_iwarp_alloc_tcp_cid(p_hwfn, &ep->tcp_cid);
1238
1239                 /* if we fail we could look for another entry with a valid
1240                  * tcp_cid, but since we don't expect to reach this anyway
1241                  * it's not worth the handling
1242                  */
1243                 if (rc) {
1244                         ep->tcp_cid = QED_IWARP_INVALID_TCP_CID;
1245                         ep = NULL;
1246                         goto out;
1247                 }
1248         }
1249
1250         list_del(&ep->list_entry);
1251
1252 out:
1253         spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
1254         return ep;
1255 }
1256
1257 #define QED_IWARP_MAX_CID_CLEAN_TIME  100
1258 #define QED_IWARP_MAX_NO_PROGRESS_CNT 5
1259
1260 /* This function waits for all the bits of a bmap to be cleared, as long as
1261  * there is progress ( i.e. the number of bits left to be cleared decreases )
1262  * the function continues.
1263  */
1264 static int
1265 qed_iwarp_wait_cid_map_cleared(struct qed_hwfn *p_hwfn, struct qed_bmap *bmap)
1266 {
1267         int prev_weight = 0;
1268         int wait_count = 0;
1269         int weight = 0;
1270
1271         weight = bitmap_weight(bmap->bitmap, bmap->max_count);
1272         prev_weight = weight;
1273
1274         while (weight) {
1275                 msleep(QED_IWARP_MAX_CID_CLEAN_TIME);
1276
1277                 weight = bitmap_weight(bmap->bitmap, bmap->max_count);
1278
1279                 if (prev_weight == weight) {
1280                         wait_count++;
1281                 } else {
1282                         prev_weight = weight;
1283                         wait_count = 0;
1284                 }
1285
1286                 if (wait_count > QED_IWARP_MAX_NO_PROGRESS_CNT) {
1287                         DP_NOTICE(p_hwfn,
1288                                   "%s bitmap wait timed out (%d cids pending)\n",
1289                                   bmap->name, weight);
1290                         return -EBUSY;
1291                 }
1292         }
1293         return 0;
1294 }
1295
1296 static int qed_iwarp_wait_for_all_cids(struct qed_hwfn *p_hwfn)
1297 {
1298         int rc;
1299         int i;
1300
1301         rc = qed_iwarp_wait_cid_map_cleared(p_hwfn,
1302                                             &p_hwfn->p_rdma_info->tcp_cid_map);
1303         if (rc)
1304                 return rc;
1305
1306         /* Now free the tcp cids from the main cid map */
1307         for (i = 0; i < QED_IWARP_PREALLOC_CNT; i++)
1308                 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->cid_map, i);
1309
1310         /* Now wait for all cids to be completed */
1311         return qed_iwarp_wait_cid_map_cleared(p_hwfn,
1312                                               &p_hwfn->p_rdma_info->cid_map);
1313 }
1314
1315 static void qed_iwarp_free_prealloc_ep(struct qed_hwfn *p_hwfn)
1316 {
1317         struct qed_iwarp_ep *ep;
1318
1319         while (!list_empty(&p_hwfn->p_rdma_info->iwarp.ep_free_list)) {
1320                 spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
1321
1322                 ep = list_first_entry(&p_hwfn->p_rdma_info->iwarp.ep_free_list,
1323                                       struct qed_iwarp_ep, list_entry);
1324
1325                 if (!ep) {
1326                         spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
1327                         break;
1328                 }
1329                 list_del(&ep->list_entry);
1330
1331                 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
1332
1333                 if (ep->tcp_cid != QED_IWARP_INVALID_TCP_CID)
1334                         qed_iwarp_cid_cleaned(p_hwfn, ep->tcp_cid);
1335
1336                 qed_iwarp_destroy_ep(p_hwfn, ep, false);
1337         }
1338 }
1339
1340 static int qed_iwarp_prealloc_ep(struct qed_hwfn *p_hwfn, bool init)
1341 {
1342         struct qed_iwarp_ep *ep;
1343         int rc = 0;
1344         int count;
1345         u32 cid;
1346         int i;
1347
1348         count = init ? QED_IWARP_PREALLOC_CNT : 1;
1349         for (i = 0; i < count; i++) {
1350                 rc = qed_iwarp_create_ep(p_hwfn, &ep);
1351                 if (rc)
1352                         return rc;
1353
1354                 /* During initialization we allocate from the main pool,
1355                  * afterwards we allocate only from the tcp_cid.
1356                  */
1357                 if (init) {
1358                         rc = qed_iwarp_alloc_cid(p_hwfn, &cid);
1359                         if (rc)
1360                                 goto err;
1361                         qed_iwarp_set_tcp_cid(p_hwfn, cid);
1362                 } else {
1363                         /* We don't care about the return code, it's ok if
1364                          * tcp_cid remains invalid...in this case we'll
1365                          * defer allocation
1366                          */
1367                         qed_iwarp_alloc_tcp_cid(p_hwfn, &cid);
1368                 }
1369
1370                 ep->tcp_cid = cid;
1371
1372                 spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
1373                 list_add_tail(&ep->list_entry,
1374                               &p_hwfn->p_rdma_info->iwarp.ep_free_list);
1375                 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
1376         }
1377
1378         return rc;
1379
1380 err:
1381         qed_iwarp_destroy_ep(p_hwfn, ep, false);
1382
1383         return rc;
1384 }
1385
1386 int qed_iwarp_alloc(struct qed_hwfn *p_hwfn)
1387 {
1388         int rc;
1389
1390         /* Allocate bitmap for tcp cid. These are used by passive side
1391          * to ensure it can allocate a tcp cid during dpc that was
1392          * pre-acquired and doesn't require dynamic allocation of ilt
1393          */
1394         rc = qed_rdma_bmap_alloc(p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map,
1395                                  QED_IWARP_PREALLOC_CNT, "TCP_CID");
1396         if (rc) {
1397                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
1398                            "Failed to allocate tcp cid, rc = %d\n", rc);
1399                 return rc;
1400         }
1401
1402         INIT_LIST_HEAD(&p_hwfn->p_rdma_info->iwarp.ep_free_list);
1403         spin_lock_init(&p_hwfn->p_rdma_info->iwarp.iw_lock);
1404
1405         return qed_iwarp_prealloc_ep(p_hwfn, true);
1406 }
1407
1408 void qed_iwarp_resc_free(struct qed_hwfn *p_hwfn)
1409 {
1410         qed_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map, 1);
1411 }
1412
1413 int qed_iwarp_accept(void *rdma_cxt, struct qed_iwarp_accept_in *iparams)
1414 {
1415         struct qed_hwfn *p_hwfn = rdma_cxt;
1416         struct qed_iwarp_ep *ep;
1417         u8 mpa_data_size = 0;
1418         int rc;
1419
1420         ep = iparams->ep_context;
1421         if (!ep) {
1422                 DP_ERR(p_hwfn, "Ep Context receive in accept is NULL\n");
1423                 return -EINVAL;
1424         }
1425
1426         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x) EP(0x%x)\n",
1427                    iparams->qp->icid, ep->tcp_cid);
1428
1429         if ((iparams->ord > QED_IWARP_ORD_DEFAULT) ||
1430             (iparams->ird > QED_IWARP_IRD_DEFAULT)) {
1431                 DP_VERBOSE(p_hwfn,
1432                            QED_MSG_RDMA,
1433                            "QP(0x%x) EP(0x%x) ERROR: Invalid ord(0x%x)/ird(0x%x)\n",
1434                            iparams->qp->icid,
1435                            ep->tcp_cid, iparams->ord, iparams->ord);
1436                 return -EINVAL;
1437         }
1438
1439         qed_iwarp_prealloc_ep(p_hwfn, false);
1440
1441         ep->cb_context = iparams->cb_context;
1442         ep->qp = iparams->qp;
1443         ep->qp->ep = ep;
1444
1445         if (ep->mpa_rev == MPA_NEGOTIATION_TYPE_ENHANCED) {
1446                 /* Negotiate ord/ird: if upperlayer requested ord larger than
1447                  * ird advertised by remote, we need to decrease our ord
1448                  */
1449                 if (iparams->ord > ep->cm_info.ird)
1450                         iparams->ord = ep->cm_info.ird;
1451
1452                 if ((ep->rtr_type & MPA_RTR_TYPE_ZERO_READ) &&
1453                     (iparams->ird == 0))
1454                         iparams->ird = 1;
1455         }
1456
1457         /* Update cm_info ord/ird to be negotiated values */
1458         ep->cm_info.ord = iparams->ord;
1459         ep->cm_info.ird = iparams->ird;
1460
1461         qed_iwarp_mpa_v2_set_private(p_hwfn, ep, &mpa_data_size);
1462
1463         ep->cm_info.private_data = ep->ep_buffer_virt->out_pdata;
1464         ep->cm_info.private_data_len = iparams->private_data_len +
1465                                        mpa_data_size;
1466
1467         memcpy((u8 *)ep->ep_buffer_virt->out_pdata + mpa_data_size,
1468                iparams->private_data, iparams->private_data_len);
1469
1470         rc = qed_iwarp_mpa_offload(p_hwfn, ep);
1471         if (rc)
1472                 qed_iwarp_modify_qp(p_hwfn,
1473                                     iparams->qp, QED_IWARP_QP_STATE_ERROR, 1);
1474
1475         return rc;
1476 }
1477
1478 int qed_iwarp_reject(void *rdma_cxt, struct qed_iwarp_reject_in *iparams)
1479 {
1480         struct qed_hwfn *p_hwfn = rdma_cxt;
1481         struct qed_iwarp_ep *ep;
1482         u8 mpa_data_size = 0;
1483
1484         ep = iparams->ep_context;
1485         if (!ep) {
1486                 DP_ERR(p_hwfn, "Ep Context receive in reject is NULL\n");
1487                 return -EINVAL;
1488         }
1489
1490         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "EP(0x%x)\n", ep->tcp_cid);
1491
1492         ep->cb_context = iparams->cb_context;
1493         ep->qp = NULL;
1494
1495         qed_iwarp_mpa_v2_set_private(p_hwfn, ep, &mpa_data_size);
1496
1497         ep->cm_info.private_data = ep->ep_buffer_virt->out_pdata;
1498         ep->cm_info.private_data_len = iparams->private_data_len +
1499                                        mpa_data_size;
1500
1501         memcpy((u8 *)ep->ep_buffer_virt->out_pdata + mpa_data_size,
1502                iparams->private_data, iparams->private_data_len);
1503
1504         return qed_iwarp_mpa_offload(p_hwfn, ep);
1505 }
1506
1507 static void
1508 qed_iwarp_print_cm_info(struct qed_hwfn *p_hwfn,
1509                         struct qed_iwarp_cm_info *cm_info)
1510 {
1511         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "ip_version = %d\n",
1512                    cm_info->ip_version);
1513
1514         if (cm_info->ip_version == QED_TCP_IPV4)
1515                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
1516                            "remote_ip %pI4h:%x, local_ip %pI4h:%x vlan=%x\n",
1517                            cm_info->remote_ip, cm_info->remote_port,
1518                            cm_info->local_ip, cm_info->local_port,
1519                            cm_info->vlan);
1520         else
1521                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
1522                            "remote_ip %pI6:%x, local_ip %pI6:%x vlan=%x\n",
1523                            cm_info->remote_ip, cm_info->remote_port,
1524                            cm_info->local_ip, cm_info->local_port,
1525                            cm_info->vlan);
1526
1527         DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
1528                    "private_data_len = %x ord = %d, ird = %d\n",
1529                    cm_info->private_data_len, cm_info->ord, cm_info->ird);
1530 }
1531
1532 static int
1533 qed_iwarp_ll2_post_rx(struct qed_hwfn *p_hwfn,
1534                       struct qed_iwarp_ll2_buff *buf, u8 handle)
1535 {
1536         int rc;
1537
1538         rc = qed_ll2_post_rx_buffer(p_hwfn, handle, buf->data_phys_addr,
1539                                     (u16)buf->buff_size, buf, 1);
1540         if (rc) {
1541                 DP_NOTICE(p_hwfn,
1542                           "Failed to repost rx buffer to ll2 rc = %d, handle=%d\n",
1543                           rc, handle);
1544                 dma_free_coherent(&p_hwfn->cdev->pdev->dev, buf->buff_size,
1545                                   buf->data, buf->data_phys_addr);
1546                 kfree(buf);
1547         }
1548
1549         return rc;
1550 }
1551
1552 static bool
1553 qed_iwarp_ep_exists(struct qed_hwfn *p_hwfn, struct qed_iwarp_cm_info *cm_info)
1554 {
1555         struct qed_iwarp_ep *ep = NULL;
1556         bool found = false;
1557
1558         list_for_each_entry(ep,
1559                             &p_hwfn->p_rdma_info->iwarp.ep_list,
1560                             list_entry) {
1561                 if ((ep->cm_info.local_port == cm_info->local_port) &&
1562                     (ep->cm_info.remote_port == cm_info->remote_port) &&
1563                     (ep->cm_info.vlan == cm_info->vlan) &&
1564                     !memcmp(&ep->cm_info.local_ip, cm_info->local_ip,
1565                             sizeof(cm_info->local_ip)) &&
1566                     !memcmp(&ep->cm_info.remote_ip, cm_info->remote_ip,
1567                             sizeof(cm_info->remote_ip))) {
1568                         found = true;
1569                         break;
1570                 }
1571         }
1572
1573         if (found) {
1574                 DP_NOTICE(p_hwfn,
1575                           "SYN received on active connection - dropping\n");
1576                 qed_iwarp_print_cm_info(p_hwfn, cm_info);
1577
1578                 return true;
1579         }
1580
1581         return false;
1582 }
1583
1584 static struct qed_iwarp_listener *
1585 qed_iwarp_get_listener(struct qed_hwfn *p_hwfn,
1586                        struct qed_iwarp_cm_info *cm_info)
1587 {
1588         struct qed_iwarp_listener *listener = NULL;
1589         static const u32 ip_zero[4] = { 0, 0, 0, 0 };
1590         bool found = false;
1591
1592         qed_iwarp_print_cm_info(p_hwfn, cm_info);
1593
1594         list_for_each_entry(listener,
1595                             &p_hwfn->p_rdma_info->iwarp.listen_list,
1596                             list_entry) {
1597                 if (listener->port == cm_info->local_port) {
1598                         if (!memcmp(listener->ip_addr,
1599                                     ip_zero, sizeof(ip_zero))) {
1600                                 found = true;
1601                                 break;
1602                         }
1603
1604                         if (!memcmp(listener->ip_addr,
1605                                     cm_info->local_ip,
1606                                     sizeof(cm_info->local_ip)) &&
1607                             (listener->vlan == cm_info->vlan)) {
1608                                 found = true;
1609                                 break;
1610                         }
1611                 }
1612         }
1613
1614         if (found) {
1615                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "listener found = %p\n",
1616                            listener);
1617                 return listener;
1618         }
1619
1620         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "listener not found\n");
1621         return NULL;
1622 }
1623
1624 static int
1625 qed_iwarp_parse_rx_pkt(struct qed_hwfn *p_hwfn,
1626                        struct qed_iwarp_cm_info *cm_info,
1627                        void *buf,
1628                        u8 *remote_mac_addr,
1629                        u8 *local_mac_addr,
1630                        int *payload_len, int *tcp_start_offset)
1631 {
1632         struct vlan_ethhdr *vethh;
1633         bool vlan_valid = false;
1634         struct ipv6hdr *ip6h;
1635         struct ethhdr *ethh;
1636         struct tcphdr *tcph;
1637         struct iphdr *iph;
1638         int eth_hlen;
1639         int ip_hlen;
1640         int eth_type;
1641         int i;
1642
1643         ethh = buf;
1644         eth_type = ntohs(ethh->h_proto);
1645         if (eth_type == ETH_P_8021Q) {
1646                 vlan_valid = true;
1647                 vethh = (struct vlan_ethhdr *)ethh;
1648                 cm_info->vlan = ntohs(vethh->h_vlan_TCI) & VLAN_VID_MASK;
1649                 eth_type = ntohs(vethh->h_vlan_encapsulated_proto);
1650         }
1651
1652         eth_hlen = ETH_HLEN + (vlan_valid ? sizeof(u32) : 0);
1653
1654         ether_addr_copy(remote_mac_addr, ethh->h_source);
1655         ether_addr_copy(local_mac_addr, ethh->h_dest);
1656
1657         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "eth_type =%d source mac: %pM\n",
1658                    eth_type, ethh->h_source);
1659
1660         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "eth_hlen=%d destination mac: %pM\n",
1661                    eth_hlen, ethh->h_dest);
1662
1663         iph = (struct iphdr *)((u8 *)(ethh) + eth_hlen);
1664
1665         if (eth_type == ETH_P_IP) {
1666                 cm_info->local_ip[0] = ntohl(iph->daddr);
1667                 cm_info->remote_ip[0] = ntohl(iph->saddr);
1668                 cm_info->ip_version = TCP_IPV4;
1669
1670                 ip_hlen = (iph->ihl) * sizeof(u32);
1671                 *payload_len = ntohs(iph->tot_len) - ip_hlen;
1672         } else if (eth_type == ETH_P_IPV6) {
1673                 ip6h = (struct ipv6hdr *)iph;
1674                 for (i = 0; i < 4; i++) {
1675                         cm_info->local_ip[i] =
1676                             ntohl(ip6h->daddr.in6_u.u6_addr32[i]);
1677                         cm_info->remote_ip[i] =
1678                             ntohl(ip6h->saddr.in6_u.u6_addr32[i]);
1679                 }
1680                 cm_info->ip_version = TCP_IPV6;
1681
1682                 ip_hlen = sizeof(*ip6h);
1683                 *payload_len = ntohs(ip6h->payload_len);
1684         } else {
1685                 DP_NOTICE(p_hwfn, "Unexpected ethertype on ll2 %x\n", eth_type);
1686                 return -EINVAL;
1687         }
1688
1689         tcph = (struct tcphdr *)((u8 *)iph + ip_hlen);
1690
1691         if (!tcph->syn) {
1692                 DP_NOTICE(p_hwfn,
1693                           "Only SYN type packet expected on this ll2 conn, iph->ihl=%d source=%d dest=%d\n",
1694                           iph->ihl, tcph->source, tcph->dest);
1695                 return -EINVAL;
1696         }
1697
1698         cm_info->local_port = ntohs(tcph->dest);
1699         cm_info->remote_port = ntohs(tcph->source);
1700
1701         qed_iwarp_print_cm_info(p_hwfn, cm_info);
1702
1703         *tcp_start_offset = eth_hlen + ip_hlen;
1704
1705         return 0;
1706 }
1707
1708 static void
1709 qed_iwarp_ll2_comp_syn_pkt(void *cxt, struct qed_ll2_comp_rx_data *data)
1710 {
1711         struct qed_iwarp_ll2_buff *buf = data->cookie;
1712         struct qed_iwarp_listener *listener;
1713         struct qed_ll2_tx_pkt_info tx_pkt;
1714         struct qed_iwarp_cm_info cm_info;
1715         struct qed_hwfn *p_hwfn = cxt;
1716         u8 remote_mac_addr[ETH_ALEN];
1717         u8 local_mac_addr[ETH_ALEN];
1718         struct qed_iwarp_ep *ep;
1719         int tcp_start_offset;
1720         u8 ts_hdr_size = 0;
1721         u8 ll2_syn_handle;
1722         int payload_len;
1723         u32 hdr_size;
1724         int rc;
1725
1726         memset(&cm_info, 0, sizeof(cm_info));
1727         ll2_syn_handle = p_hwfn->p_rdma_info->iwarp.ll2_syn_handle;
1728         if (GET_FIELD(data->parse_flags,
1729                       PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED) &&
1730             GET_FIELD(data->parse_flags, PARSING_AND_ERR_FLAGS_L4CHKSMERROR)) {
1731                 DP_NOTICE(p_hwfn, "Syn packet received with checksum error\n");
1732                 goto err;
1733         }
1734
1735         rc = qed_iwarp_parse_rx_pkt(p_hwfn, &cm_info, (u8 *)(buf->data) +
1736                                     data->u.placement_offset, remote_mac_addr,
1737                                     local_mac_addr, &payload_len,
1738                                     &tcp_start_offset);
1739         if (rc)
1740                 goto err;
1741
1742         /* Check if there is a listener for this 4-tuple+vlan */
1743         listener = qed_iwarp_get_listener(p_hwfn, &cm_info);
1744         if (!listener) {
1745                 DP_VERBOSE(p_hwfn,
1746                            QED_MSG_RDMA,
1747                            "SYN received on tuple not listened on parse_flags=%d packet len=%d\n",
1748                            data->parse_flags, data->length.packet_length);
1749
1750                 memset(&tx_pkt, 0, sizeof(tx_pkt));
1751                 tx_pkt.num_of_bds = 1;
1752                 tx_pkt.vlan = data->vlan;
1753
1754                 if (GET_FIELD(data->parse_flags,
1755                               PARSING_AND_ERR_FLAGS_TAG8021QEXIST))
1756                         SET_FIELD(tx_pkt.bd_flags,
1757                                   CORE_TX_BD_DATA_VLAN_INSERTION, 1);
1758
1759                 tx_pkt.l4_hdr_offset_w = (data->length.packet_length) >> 2;
1760                 tx_pkt.tx_dest = QED_LL2_TX_DEST_LB;
1761                 tx_pkt.first_frag = buf->data_phys_addr +
1762                                     data->u.placement_offset;
1763                 tx_pkt.first_frag_len = data->length.packet_length;
1764                 tx_pkt.cookie = buf;
1765
1766                 rc = qed_ll2_prepare_tx_packet(p_hwfn, ll2_syn_handle,
1767                                                &tx_pkt, true);
1768
1769                 if (rc) {
1770                         DP_NOTICE(p_hwfn,
1771                                   "Can't post SYN back to chip rc=%d\n", rc);
1772                         goto err;
1773                 }
1774                 return;
1775         }
1776
1777         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Received syn on listening port\n");
1778         /* There may be an open ep on this connection if this is a syn
1779          * retrasnmit... need to make sure there isn't...
1780          */
1781         if (qed_iwarp_ep_exists(p_hwfn, &cm_info))
1782                 goto err;
1783
1784         ep = qed_iwarp_get_free_ep(p_hwfn);
1785         if (!ep)
1786                 goto err;
1787
1788         spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
1789         list_add_tail(&ep->list_entry, &p_hwfn->p_rdma_info->iwarp.ep_list);
1790         spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
1791
1792         ether_addr_copy(ep->remote_mac_addr, remote_mac_addr);
1793         ether_addr_copy(ep->local_mac_addr, local_mac_addr);
1794
1795         memcpy(&ep->cm_info, &cm_info, sizeof(ep->cm_info));
1796
1797         if (p_hwfn->p_rdma_info->iwarp.tcp_flags & QED_IWARP_TS_EN)
1798                 ts_hdr_size = TIMESTAMP_HEADER_SIZE;
1799
1800         hdr_size = ((cm_info.ip_version == QED_TCP_IPV4) ? 40 : 60) +
1801                    ts_hdr_size;
1802         ep->mss = p_hwfn->p_rdma_info->iwarp.max_mtu - hdr_size;
1803         ep->mss = min_t(u16, QED_IWARP_MAX_FW_MSS, ep->mss);
1804
1805         ep->event_cb = listener->event_cb;
1806         ep->cb_context = listener->cb_context;
1807         ep->connect_mode = TCP_CONNECT_PASSIVE;
1808
1809         ep->syn = buf;
1810         ep->syn_ip_payload_length = (u16)payload_len;
1811         ep->syn_phy_addr = buf->data_phys_addr + data->u.placement_offset +
1812                            tcp_start_offset;
1813
1814         rc = qed_iwarp_tcp_offload(p_hwfn, ep);
1815         if (rc) {
1816                 qed_iwarp_return_ep(p_hwfn, ep);
1817                 goto err;
1818         }
1819
1820         return;
1821 err:
1822         qed_iwarp_ll2_post_rx(p_hwfn, buf, ll2_syn_handle);
1823 }
1824
1825 static void qed_iwarp_ll2_rel_rx_pkt(void *cxt, u8 connection_handle,
1826                                      void *cookie, dma_addr_t rx_buf_addr,
1827                                      bool b_last_packet)
1828 {
1829         struct qed_iwarp_ll2_buff *buffer = cookie;
1830         struct qed_hwfn *p_hwfn = cxt;
1831
1832         dma_free_coherent(&p_hwfn->cdev->pdev->dev, buffer->buff_size,
1833                           buffer->data, buffer->data_phys_addr);
1834         kfree(buffer);
1835 }
1836
1837 static void qed_iwarp_ll2_comp_tx_pkt(void *cxt, u8 connection_handle,
1838                                       void *cookie, dma_addr_t first_frag_addr,
1839                                       bool b_last_fragment, bool b_last_packet)
1840 {
1841         struct qed_iwarp_ll2_buff *buffer = cookie;
1842         struct qed_hwfn *p_hwfn = cxt;
1843
1844         /* this was originally an rx packet, post it back */
1845         qed_iwarp_ll2_post_rx(p_hwfn, buffer, connection_handle);
1846 }
1847
1848 static void qed_iwarp_ll2_rel_tx_pkt(void *cxt, u8 connection_handle,
1849                                      void *cookie, dma_addr_t first_frag_addr,
1850                                      bool b_last_fragment, bool b_last_packet)
1851 {
1852         struct qed_iwarp_ll2_buff *buffer = cookie;
1853         struct qed_hwfn *p_hwfn = cxt;
1854
1855         if (!buffer)
1856                 return;
1857
1858         dma_free_coherent(&p_hwfn->cdev->pdev->dev, buffer->buff_size,
1859                           buffer->data, buffer->data_phys_addr);
1860
1861         kfree(buffer);
1862 }
1863
1864 static int qed_iwarp_ll2_stop(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
1865 {
1866         struct qed_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp;
1867         int rc = 0;
1868
1869         if (iwarp_info->ll2_syn_handle != QED_IWARP_HANDLE_INVAL) {
1870                 rc = qed_ll2_terminate_connection(p_hwfn,
1871                                                   iwarp_info->ll2_syn_handle);
1872                 if (rc)
1873                         DP_INFO(p_hwfn, "Failed to terminate syn connection\n");
1874
1875                 qed_ll2_release_connection(p_hwfn, iwarp_info->ll2_syn_handle);
1876                 iwarp_info->ll2_syn_handle = QED_IWARP_HANDLE_INVAL;
1877         }
1878
1879         qed_llh_remove_mac_filter(p_hwfn,
1880                                   p_ptt, p_hwfn->p_rdma_info->iwarp.mac_addr);
1881         return rc;
1882 }
1883
1884 static int
1885 qed_iwarp_ll2_alloc_buffers(struct qed_hwfn *p_hwfn,
1886                             int num_rx_bufs, int buff_size, u8 ll2_handle)
1887 {
1888         struct qed_iwarp_ll2_buff *buffer;
1889         int rc = 0;
1890         int i;
1891
1892         for (i = 0; i < num_rx_bufs; i++) {
1893                 buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
1894                 if (!buffer) {
1895                         rc = -ENOMEM;
1896                         break;
1897                 }
1898
1899                 buffer->data = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
1900                                                   buff_size,
1901                                                   &buffer->data_phys_addr,
1902                                                   GFP_KERNEL);
1903                 if (!buffer->data) {
1904                         kfree(buffer);
1905                         rc = -ENOMEM;
1906                         break;
1907                 }
1908
1909                 buffer->buff_size = buff_size;
1910                 rc = qed_iwarp_ll2_post_rx(p_hwfn, buffer, ll2_handle);
1911                 if (rc)
1912                         /* buffers will be deallocated by qed_ll2 */
1913                         break;
1914         }
1915         return rc;
1916 }
1917
1918 #define QED_IWARP_MAX_BUF_SIZE(mtu)                                  \
1919         ALIGN((mtu) + ETH_HLEN + 2 * VLAN_HLEN + 2 + ETH_CACHE_LINE_SIZE, \
1920                 ETH_CACHE_LINE_SIZE)
1921
1922 static int
1923 qed_iwarp_ll2_start(struct qed_hwfn *p_hwfn,
1924                     struct qed_rdma_start_in_params *params,
1925                     struct qed_ptt *p_ptt)
1926 {
1927         struct qed_iwarp_info *iwarp_info;
1928         struct qed_ll2_acquire_data data;
1929         struct qed_ll2_cbs cbs;
1930         int rc = 0;
1931
1932         iwarp_info = &p_hwfn->p_rdma_info->iwarp;
1933         iwarp_info->ll2_syn_handle = QED_IWARP_HANDLE_INVAL;
1934
1935         iwarp_info->max_mtu = params->max_mtu;
1936
1937         ether_addr_copy(p_hwfn->p_rdma_info->iwarp.mac_addr, params->mac_addr);
1938
1939         rc = qed_llh_add_mac_filter(p_hwfn, p_ptt, params->mac_addr);
1940         if (rc)
1941                 return rc;
1942
1943         /* Start SYN connection */
1944         cbs.rx_comp_cb = qed_iwarp_ll2_comp_syn_pkt;
1945         cbs.rx_release_cb = qed_iwarp_ll2_rel_rx_pkt;
1946         cbs.tx_comp_cb = qed_iwarp_ll2_comp_tx_pkt;
1947         cbs.tx_release_cb = qed_iwarp_ll2_rel_tx_pkt;
1948         cbs.cookie = p_hwfn;
1949
1950         memset(&data, 0, sizeof(data));
1951         data.input.conn_type = QED_LL2_TYPE_IWARP;
1952         data.input.mtu = QED_IWARP_MAX_SYN_PKT_SIZE;
1953         data.input.rx_num_desc = QED_IWARP_LL2_SYN_RX_SIZE;
1954         data.input.tx_num_desc = QED_IWARP_LL2_SYN_TX_SIZE;
1955         data.input.tx_max_bds_per_packet = 1;   /* will never be fragmented */
1956         data.input.tx_tc = PKT_LB_TC;
1957         data.input.tx_dest = QED_LL2_TX_DEST_LB;
1958         data.p_connection_handle = &iwarp_info->ll2_syn_handle;
1959         data.cbs = &cbs;
1960
1961         rc = qed_ll2_acquire_connection(p_hwfn, &data);
1962         if (rc) {
1963                 DP_NOTICE(p_hwfn, "Failed to acquire LL2 connection\n");
1964                 qed_llh_remove_mac_filter(p_hwfn, p_ptt, params->mac_addr);
1965                 return rc;
1966         }
1967
1968         rc = qed_ll2_establish_connection(p_hwfn, iwarp_info->ll2_syn_handle);
1969         if (rc) {
1970                 DP_NOTICE(p_hwfn, "Failed to establish LL2 connection\n");
1971                 goto err;
1972         }
1973
1974         rc = qed_iwarp_ll2_alloc_buffers(p_hwfn,
1975                                          QED_IWARP_LL2_SYN_RX_SIZE,
1976                                          QED_IWARP_MAX_SYN_PKT_SIZE,
1977                                          iwarp_info->ll2_syn_handle);
1978         if (rc)
1979                 goto err;
1980
1981         return rc;
1982 err:
1983         qed_iwarp_ll2_stop(p_hwfn, p_ptt);
1984
1985         return rc;
1986 }
1987
1988 int qed_iwarp_setup(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
1989                     struct qed_rdma_start_in_params *params)
1990 {
1991         struct qed_iwarp_info *iwarp_info;
1992         u32 rcv_wnd_size;
1993
1994         iwarp_info = &p_hwfn->p_rdma_info->iwarp;
1995
1996         iwarp_info->tcp_flags = QED_IWARP_TS_EN;
1997         rcv_wnd_size = QED_IWARP_RCV_WND_SIZE_DEF;
1998
1999         /* value 0 is used for ilog2(QED_IWARP_RCV_WND_SIZE_MIN) */
2000         iwarp_info->rcv_wnd_scale = ilog2(rcv_wnd_size) -
2001             ilog2(QED_IWARP_RCV_WND_SIZE_MIN);
2002         iwarp_info->crc_needed = QED_IWARP_PARAM_CRC_NEEDED;
2003         iwarp_info->mpa_rev = MPA_NEGOTIATION_TYPE_ENHANCED;
2004
2005         iwarp_info->peer2peer = QED_IWARP_PARAM_P2P;
2006
2007         iwarp_info->rtr_type =  MPA_RTR_TYPE_ZERO_SEND |
2008                                 MPA_RTR_TYPE_ZERO_WRITE |
2009                                 MPA_RTR_TYPE_ZERO_READ;
2010
2011         spin_lock_init(&p_hwfn->p_rdma_info->iwarp.qp_lock);
2012         INIT_LIST_HEAD(&p_hwfn->p_rdma_info->iwarp.ep_list);
2013         INIT_LIST_HEAD(&p_hwfn->p_rdma_info->iwarp.listen_list);
2014
2015         qed_spq_register_async_cb(p_hwfn, PROTOCOLID_IWARP,
2016                                   qed_iwarp_async_event);
2017
2018         return qed_iwarp_ll2_start(p_hwfn, params, p_ptt);
2019 }
2020
2021 int qed_iwarp_stop(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
2022 {
2023         int rc;
2024
2025         qed_iwarp_free_prealloc_ep(p_hwfn);
2026         rc = qed_iwarp_wait_for_all_cids(p_hwfn);
2027         if (rc)
2028                 return rc;
2029
2030         qed_spq_unregister_async_cb(p_hwfn, PROTOCOLID_IWARP);
2031
2032         return qed_iwarp_ll2_stop(p_hwfn, p_ptt);
2033 }
2034
2035 void qed_iwarp_qp_in_error(struct qed_hwfn *p_hwfn,
2036                            struct qed_iwarp_ep *ep, u8 fw_return_code)
2037 {
2038         struct qed_iwarp_cm_event_params params;
2039
2040         qed_iwarp_modify_qp(p_hwfn, ep->qp, QED_IWARP_QP_STATE_ERROR, true);
2041
2042         params.event = QED_IWARP_EVENT_CLOSE;
2043         params.ep_context = ep;
2044         params.cm_info = &ep->cm_info;
2045         params.status = (fw_return_code == IWARP_QP_IN_ERROR_GOOD_CLOSE) ?
2046                          0 : -ECONNRESET;
2047
2048         ep->state = QED_IWARP_EP_CLOSED;
2049         spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
2050         list_del(&ep->list_entry);
2051         spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
2052
2053         ep->event_cb(ep->cb_context, &params);
2054 }
2055
2056 void qed_iwarp_exception_received(struct qed_hwfn *p_hwfn,
2057                                   struct qed_iwarp_ep *ep, int fw_ret_code)
2058 {
2059         struct qed_iwarp_cm_event_params params;
2060         bool event_cb = false;
2061
2062         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "EP(0x%x) fw_ret_code=%d\n",
2063                    ep->cid, fw_ret_code);
2064
2065         switch (fw_ret_code) {
2066         case IWARP_EXCEPTION_DETECTED_LLP_CLOSED:
2067                 params.status = 0;
2068                 params.event = QED_IWARP_EVENT_DISCONNECT;
2069                 event_cb = true;
2070                 break;
2071         case IWARP_EXCEPTION_DETECTED_LLP_RESET:
2072                 params.status = -ECONNRESET;
2073                 params.event = QED_IWARP_EVENT_DISCONNECT;
2074                 event_cb = true;
2075                 break;
2076         case IWARP_EXCEPTION_DETECTED_RQ_EMPTY:
2077                 params.event = QED_IWARP_EVENT_RQ_EMPTY;
2078                 event_cb = true;
2079                 break;
2080         case IWARP_EXCEPTION_DETECTED_IRQ_FULL:
2081                 params.event = QED_IWARP_EVENT_IRQ_FULL;
2082                 event_cb = true;
2083                 break;
2084         case IWARP_EXCEPTION_DETECTED_LLP_TIMEOUT:
2085                 params.event = QED_IWARP_EVENT_LLP_TIMEOUT;
2086                 event_cb = true;
2087                 break;
2088         case IWARP_EXCEPTION_DETECTED_REMOTE_PROTECTION_ERROR:
2089                 params.event = QED_IWARP_EVENT_REMOTE_PROTECTION_ERROR;
2090                 event_cb = true;
2091                 break;
2092         case IWARP_EXCEPTION_DETECTED_CQ_OVERFLOW:
2093                 params.event = QED_IWARP_EVENT_CQ_OVERFLOW;
2094                 event_cb = true;
2095                 break;
2096         case IWARP_EXCEPTION_DETECTED_LOCAL_CATASTROPHIC:
2097                 params.event = QED_IWARP_EVENT_QP_CATASTROPHIC;
2098                 event_cb = true;
2099                 break;
2100         case IWARP_EXCEPTION_DETECTED_LOCAL_ACCESS_ERROR:
2101                 params.event = QED_IWARP_EVENT_LOCAL_ACCESS_ERROR;
2102                 event_cb = true;
2103                 break;
2104         case IWARP_EXCEPTION_DETECTED_REMOTE_OPERATION_ERROR:
2105                 params.event = QED_IWARP_EVENT_REMOTE_OPERATION_ERROR;
2106                 event_cb = true;
2107                 break;
2108         case IWARP_EXCEPTION_DETECTED_TERMINATE_RECEIVED:
2109                 params.event = QED_IWARP_EVENT_TERMINATE_RECEIVED;
2110                 event_cb = true;
2111                 break;
2112         default:
2113                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
2114                            "Unhandled exception received...fw_ret_code=%d\n",
2115                            fw_ret_code);
2116                 break;
2117         }
2118
2119         if (event_cb) {
2120                 params.ep_context = ep;
2121                 params.cm_info = &ep->cm_info;
2122                 ep->event_cb(ep->cb_context, &params);
2123         }
2124 }
2125
2126 static void
2127 qed_iwarp_tcp_connect_unsuccessful(struct qed_hwfn *p_hwfn,
2128                                    struct qed_iwarp_ep *ep, u8 fw_return_code)
2129 {
2130         struct qed_iwarp_cm_event_params params;
2131
2132         memset(&params, 0, sizeof(params));
2133         params.event = QED_IWARP_EVENT_ACTIVE_COMPLETE;
2134         params.ep_context = ep;
2135         params.cm_info = &ep->cm_info;
2136         ep->state = QED_IWARP_EP_CLOSED;
2137
2138         switch (fw_return_code) {
2139         case IWARP_CONN_ERROR_TCP_CONNECT_INVALID_PACKET:
2140                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
2141                            "%s(0x%x) TCP connect got invalid packet\n",
2142                            QED_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid);
2143                 params.status = -ECONNRESET;
2144                 break;
2145         case IWARP_CONN_ERROR_TCP_CONNECTION_RST:
2146                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
2147                            "%s(0x%x) TCP Connection Reset\n",
2148                            QED_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid);
2149                 params.status = -ECONNRESET;
2150                 break;
2151         case IWARP_CONN_ERROR_TCP_CONNECT_TIMEOUT:
2152                 DP_NOTICE(p_hwfn, "%s(0x%x) TCP timeout\n",
2153                           QED_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid);
2154                 params.status = -EBUSY;
2155                 break;
2156         case IWARP_CONN_ERROR_MPA_NOT_SUPPORTED_VER:
2157                 DP_NOTICE(p_hwfn, "%s(0x%x) MPA not supported VER\n",
2158                           QED_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid);
2159                 params.status = -ECONNREFUSED;
2160                 break;
2161         case IWARP_CONN_ERROR_MPA_INVALID_PACKET:
2162                 DP_NOTICE(p_hwfn, "%s(0x%x) MPA Invalid Packet\n",
2163                           QED_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid);
2164                 params.status = -ECONNRESET;
2165                 break;
2166         default:
2167                 DP_ERR(p_hwfn,
2168                        "%s(0x%x) Unexpected return code tcp connect: %d\n",
2169                        QED_IWARP_CONNECT_MODE_STRING(ep),
2170                        ep->tcp_cid, fw_return_code);
2171                 params.status = -ECONNRESET;
2172                 break;
2173         }
2174
2175         if (ep->connect_mode == TCP_CONNECT_PASSIVE) {
2176                 ep->tcp_cid = QED_IWARP_INVALID_TCP_CID;
2177                 qed_iwarp_return_ep(p_hwfn, ep);
2178         } else {
2179                 ep->event_cb(ep->cb_context, &params);
2180                 spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
2181                 list_del(&ep->list_entry);
2182                 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
2183         }
2184 }
2185
2186 void
2187 qed_iwarp_connect_complete(struct qed_hwfn *p_hwfn,
2188                            struct qed_iwarp_ep *ep, u8 fw_return_code)
2189 {
2190         u8 ll2_syn_handle = p_hwfn->p_rdma_info->iwarp.ll2_syn_handle;
2191
2192         if (ep->connect_mode == TCP_CONNECT_PASSIVE) {
2193                 /* Done with the SYN packet, post back to ll2 rx */
2194                 qed_iwarp_ll2_post_rx(p_hwfn, ep->syn, ll2_syn_handle);
2195
2196                 ep->syn = NULL;
2197
2198                 /* If connect failed - upper layer doesn't know about it */
2199                 if (fw_return_code == RDMA_RETURN_OK)
2200                         qed_iwarp_mpa_received(p_hwfn, ep);
2201                 else
2202                         qed_iwarp_tcp_connect_unsuccessful(p_hwfn, ep,
2203                                                            fw_return_code);
2204         } else {
2205                 if (fw_return_code == RDMA_RETURN_OK)
2206                         qed_iwarp_mpa_offload(p_hwfn, ep);
2207                 else
2208                         qed_iwarp_tcp_connect_unsuccessful(p_hwfn, ep,
2209                                                            fw_return_code);
2210         }
2211 }
2212
2213 static inline bool
2214 qed_iwarp_check_ep_ok(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep *ep)
2215 {
2216         if (!ep || (ep->sig != QED_EP_SIG)) {
2217                 DP_ERR(p_hwfn, "ERROR ON ASYNC ep=%p\n", ep);
2218                 return false;
2219         }
2220
2221         return true;
2222 }
2223
2224 static int qed_iwarp_async_event(struct qed_hwfn *p_hwfn,
2225                                  u8 fw_event_code, u16 echo,
2226                                  union event_ring_data *data,
2227                                  u8 fw_return_code)
2228 {
2229         struct regpair *fw_handle = &data->rdma_data.async_handle;
2230         struct qed_iwarp_ep *ep = NULL;
2231         u16 cid;
2232
2233         ep = (struct qed_iwarp_ep *)(uintptr_t)HILO_64(fw_handle->hi,
2234                                                        fw_handle->lo);
2235
2236         switch (fw_event_code) {
2237         case IWARP_EVENT_TYPE_ASYNC_CONNECT_COMPLETE:
2238                 /* Async completion after TCP 3-way handshake */
2239                 if (!qed_iwarp_check_ep_ok(p_hwfn, ep))
2240                         return -EINVAL;
2241                 DP_VERBOSE(p_hwfn,
2242                            QED_MSG_RDMA,
2243                            "EP(0x%x) IWARP_EVENT_TYPE_ASYNC_CONNECT_COMPLETE fw_ret_code=%d\n",
2244                            ep->tcp_cid, fw_return_code);
2245                 qed_iwarp_connect_complete(p_hwfn, ep, fw_return_code);
2246                 break;
2247         case IWARP_EVENT_TYPE_ASYNC_EXCEPTION_DETECTED:
2248                 if (!qed_iwarp_check_ep_ok(p_hwfn, ep))
2249                         return -EINVAL;
2250                 DP_VERBOSE(p_hwfn,
2251                            QED_MSG_RDMA,
2252                            "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_EXCEPTION_DETECTED fw_ret_code=%d\n",
2253                            ep->cid, fw_return_code);
2254                 qed_iwarp_exception_received(p_hwfn, ep, fw_return_code);
2255                 break;
2256         case IWARP_EVENT_TYPE_ASYNC_QP_IN_ERROR_STATE:
2257                 /* Async completion for Close Connection ramrod */
2258                 if (!qed_iwarp_check_ep_ok(p_hwfn, ep))
2259                         return -EINVAL;
2260                 DP_VERBOSE(p_hwfn,
2261                            QED_MSG_RDMA,
2262                            "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_QP_IN_ERROR_STATE fw_ret_code=%d\n",
2263                            ep->cid, fw_return_code);
2264                 qed_iwarp_qp_in_error(p_hwfn, ep, fw_return_code);
2265                 break;
2266         case IWARP_EVENT_TYPE_ASYNC_ENHANCED_MPA_REPLY_ARRIVED:
2267                 /* Async event for active side only */
2268                 if (!qed_iwarp_check_ep_ok(p_hwfn, ep))
2269                         return -EINVAL;
2270                 DP_VERBOSE(p_hwfn,
2271                            QED_MSG_RDMA,
2272                            "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_MPA_REPLY_ARRIVED fw_ret_code=%d\n",
2273                            ep->cid, fw_return_code);
2274                 qed_iwarp_mpa_reply_arrived(p_hwfn, ep);
2275                 break;
2276         case IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_COMPLETE:
2277                 if (!qed_iwarp_check_ep_ok(p_hwfn, ep))
2278                         return -EINVAL;
2279                 DP_VERBOSE(p_hwfn,
2280                            QED_MSG_RDMA,
2281                            "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_COMPLETE fw_ret_code=%d\n",
2282                            ep->cid, fw_return_code);
2283                 qed_iwarp_mpa_complete(p_hwfn, ep, fw_return_code);
2284                 break;
2285         case IWARP_EVENT_TYPE_ASYNC_CID_CLEANED:
2286                 cid = (u16)le32_to_cpu(fw_handle->lo);
2287                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
2288                            "(0x%x)IWARP_EVENT_TYPE_ASYNC_CID_CLEANED\n", cid);
2289                 qed_iwarp_cid_cleaned(p_hwfn, cid);
2290
2291                 break;
2292         case IWARP_EVENT_TYPE_ASYNC_CQ_OVERFLOW:
2293                 DP_NOTICE(p_hwfn, "IWARP_EVENT_TYPE_ASYNC_CQ_OVERFLOW\n");
2294
2295                 p_hwfn->p_rdma_info->events.affiliated_event(
2296                         p_hwfn->p_rdma_info->events.context,
2297                         QED_IWARP_EVENT_CQ_OVERFLOW,
2298                         (void *)fw_handle);
2299                 break;
2300         default:
2301                 DP_ERR(p_hwfn, "Received unexpected async iwarp event %d\n",
2302                        fw_event_code);
2303                 return -EINVAL;
2304         }
2305         return 0;
2306 }
2307
2308 int
2309 qed_iwarp_create_listen(void *rdma_cxt,
2310                         struct qed_iwarp_listen_in *iparams,
2311                         struct qed_iwarp_listen_out *oparams)
2312 {
2313         struct qed_hwfn *p_hwfn = rdma_cxt;
2314         struct qed_iwarp_listener *listener;
2315
2316         listener = kzalloc(sizeof(*listener), GFP_KERNEL);
2317         if (!listener)
2318                 return -ENOMEM;
2319
2320         listener->ip_version = iparams->ip_version;
2321         memcpy(listener->ip_addr, iparams->ip_addr, sizeof(listener->ip_addr));
2322         listener->port = iparams->port;
2323         listener->vlan = iparams->vlan;
2324
2325         listener->event_cb = iparams->event_cb;
2326         listener->cb_context = iparams->cb_context;
2327         listener->max_backlog = iparams->max_backlog;
2328         oparams->handle = listener;
2329
2330         spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
2331         list_add_tail(&listener->list_entry,
2332                       &p_hwfn->p_rdma_info->iwarp.listen_list);
2333         spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
2334
2335         DP_VERBOSE(p_hwfn,
2336                    QED_MSG_RDMA,
2337                    "callback=%p handle=%p ip=%x:%x:%x:%x port=0x%x vlan=0x%x\n",
2338                    listener->event_cb,
2339                    listener,
2340                    listener->ip_addr[0],
2341                    listener->ip_addr[1],
2342                    listener->ip_addr[2],
2343                    listener->ip_addr[3], listener->port, listener->vlan);
2344
2345         return 0;
2346 }
2347
2348 int qed_iwarp_destroy_listen(void *rdma_cxt, void *handle)
2349 {
2350         struct qed_iwarp_listener *listener = handle;
2351         struct qed_hwfn *p_hwfn = rdma_cxt;
2352
2353         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "handle=%p\n", handle);
2354
2355         spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
2356         list_del(&listener->list_entry);
2357         spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock);
2358
2359         kfree(listener);
2360
2361         return 0;
2362 }
2363
2364 int qed_iwarp_send_rtr(void *rdma_cxt, struct qed_iwarp_send_rtr_in *iparams)
2365 {
2366         struct qed_hwfn *p_hwfn = rdma_cxt;
2367         struct qed_sp_init_data init_data;
2368         struct qed_spq_entry *p_ent;
2369         struct qed_iwarp_ep *ep;
2370         struct qed_rdma_qp *qp;
2371         int rc;
2372
2373         ep = iparams->ep_context;
2374         if (!ep) {
2375                 DP_ERR(p_hwfn, "Ep Context receive in send_rtr is NULL\n");
2376                 return -EINVAL;
2377         }
2378
2379         qp = ep->qp;
2380
2381         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x) EP(0x%x)\n",
2382                    qp->icid, ep->tcp_cid);
2383
2384         memset(&init_data, 0, sizeof(init_data));
2385         init_data.cid = qp->icid;
2386         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
2387         init_data.comp_mode = QED_SPQ_MODE_CB;
2388
2389         rc = qed_sp_init_request(p_hwfn, &p_ent,
2390                                  IWARP_RAMROD_CMD_ID_MPA_OFFLOAD_SEND_RTR,
2391                                  PROTOCOLID_IWARP, &init_data);
2392
2393         if (rc)
2394                 return rc;
2395
2396         rc = qed_spq_post(p_hwfn, p_ent, NULL);
2397
2398         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = 0x%x\n", rc);
2399
2400         return rc;
2401 }
2402
2403 void
2404 qed_iwarp_query_qp(struct qed_rdma_qp *qp,
2405                    struct qed_rdma_query_qp_out_params *out_params)
2406 {
2407         out_params->state = qed_iwarp2roce_state(qp->iwarp_state);
2408 }