drivers/infiniband/hw/qedr/verbs.c

   1 /* QLogic qedr NIC Driver
   2  * Copyright (c) 2015-2016  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/dma-mapping.h>
  33 #include <linux/crc32.h>
  34 #include <net/ip.h>
  35 #include <net/ipv6.h>
  36 #include <net/udp.h>
  37 #include <linux/iommu.h>
  38
  39 #include <rdma/ib_verbs.h>
  40 #include <rdma/ib_user_verbs.h>
  41 #include <rdma/iw_cm.h>
  42 #include <rdma/ib_umem.h>
  43 #include <rdma/ib_addr.h>
  44 #include <rdma/ib_cache.h>
  45 #include <rdma/uverbs_ioctl.h>
  46
  47 #include <linux/qed/common_hsi.h>
  48 #include "qedr_hsi_rdma.h"
  49 #include <linux/qed/qed_if.h>
  50 #include "qedr.h"
  51 #include "verbs.h"
  52 #include <rdma/qedr-abi.h>
  53 #include "qedr_roce_cm.h"
  54 #include "qedr_iw_cm.h"
  55
  56 #define QEDR_SRQ_WQE_ELEM_SIZE  sizeof(union rdma_srq_elm)
  57 #define RDMA_MAX_SGE_PER_SRQ    (4)
  58 #define RDMA_MAX_SRQ_WQE_SIZE   (RDMA_MAX_SGE_PER_SRQ + 1)
  59
  60 #define DB_ADDR_SHIFT(addr)             ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
  61
  62 enum {
  63         QEDR_USER_MMAP_IO_WC = 0,
  64         QEDR_USER_MMAP_PHYS_PAGE,
  65 };
  66
  67 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
  68                                         size_t len)
  69 {
  70         size_t min_len = min_t(size_t, len, udata->outlen);
  71
  72         return ib_copy_to_udata(udata, src, min_len);
  73 }
  74
  75 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
  76 {
  77         if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
  78                 return -EINVAL;
  79
  80         *pkey = QEDR_ROCE_PKEY_DEFAULT;
  81         return 0;
  82 }
  83
  84 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
  85                       int index, union ib_gid *sgid)
  86 {
  87         struct qedr_dev *dev = get_qedr_dev(ibdev);
  88
  89         memset(sgid->raw, 0, sizeof(sgid->raw));
  90         ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
  91
  92         DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
  93                  sgid->global.interface_id, sgid->global.subnet_prefix);
  94
  95         return 0;
  96 }
  97
  98 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
  99 {
 100         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
 101         struct qedr_device_attr *qattr = &dev->attr;
 102         struct qedr_srq *srq = get_qedr_srq(ibsrq);
 103
 104         srq_attr->srq_limit = srq->srq_limit;
 105         srq_attr->max_wr = qattr->max_srq_wr;
 106         srq_attr->max_sge = qattr->max_sge;
 107
 108         return 0;
 109 }
 110
 111 int qedr_query_device(struct ib_device *ibdev,
 112                       struct ib_device_attr *attr, struct ib_udata *udata)
 113 {
 114         struct qedr_dev *dev = get_qedr_dev(ibdev);
 115         struct qedr_device_attr *qattr = &dev->attr;
 116
 117         if (!dev->rdma_ctx) {
 118                 DP_ERR(dev,
 119                        "qedr_query_device called with invalid params rdma_ctx=%p\n",
 120                        dev->rdma_ctx);
 121                 return -EINVAL;
 122         }
 123
 124         memset(attr, 0, sizeof(*attr));
 125
 126         attr->fw_ver = qattr->fw_ver;
 127         attr->sys_image_guid = qattr->sys_image_guid;
 128         attr->max_mr_size = qattr->max_mr_size;
 129         attr->page_size_cap = qattr->page_size_caps;
 130         attr->vendor_id = qattr->vendor_id;
 131         attr->vendor_part_id = qattr->vendor_part_id;
 132         attr->hw_ver = qattr->hw_ver;
 133         attr->max_qp = qattr->max_qp;
 134         attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
 135         attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
 136             IB_DEVICE_RC_RNR_NAK_GEN |
 137             IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
 138
 139         if (!rdma_protocol_iwarp(&dev->ibdev, 1))
 140                 attr->device_cap_flags |= IB_DEVICE_XRC;
 141         attr->max_send_sge = qattr->max_sge;
 142         attr->max_recv_sge = qattr->max_sge;
 143         attr->max_sge_rd = qattr->max_sge;
 144         attr->max_cq = qattr->max_cq;
 145         attr->max_cqe = qattr->max_cqe;
 146         attr->max_mr = qattr->max_mr;
 147         attr->max_mw = qattr->max_mw;
 148         attr->max_pd = qattr->max_pd;
 149         attr->atomic_cap = dev->atomic_cap;
 150         attr->max_qp_init_rd_atom =
 151             1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
 152         attr->max_qp_rd_atom =
 153             min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
 154                 attr->max_qp_init_rd_atom);
 155
 156         attr->max_srq = qattr->max_srq;
 157         attr->max_srq_sge = qattr->max_srq_sge;
 158         attr->max_srq_wr = qattr->max_srq_wr;
 159
 160         attr->local_ca_ack_delay = qattr->dev_ack_delay;
 161         attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
 162         attr->max_pkeys = qattr->max_pkey;
 163         attr->max_ah = qattr->max_ah;
 164
 165         return 0;
 166 }
 167
 168 static inline void get_link_speed_and_width(int speed, u16 *ib_speed,
 169                                             u8 *ib_width)
 170 {
 171         switch (speed) {
 172         case 1000:
 173                 *ib_speed = IB_SPEED_SDR;
 174                 *ib_width = IB_WIDTH_1X;
 175                 break;
 176         case 10000:
 177                 *ib_speed = IB_SPEED_QDR;
 178                 *ib_width = IB_WIDTH_1X;
 179                 break;
 180
 181         case 20000:
 182                 *ib_speed = IB_SPEED_DDR;
 183                 *ib_width = IB_WIDTH_4X;
 184                 break;
 185
 186         case 25000:
 187                 *ib_speed = IB_SPEED_EDR;
 188                 *ib_width = IB_WIDTH_1X;
 189                 break;
 190
 191         case 40000:
 192                 *ib_speed = IB_SPEED_QDR;
 193                 *ib_width = IB_WIDTH_4X;
 194                 break;
 195
 196         case 50000:
 197                 *ib_speed = IB_SPEED_HDR;
 198                 *ib_width = IB_WIDTH_1X;
 199                 break;
 200
 201         case 100000:
 202                 *ib_speed = IB_SPEED_EDR;
 203                 *ib_width = IB_WIDTH_4X;
 204                 break;
 205
 206         default:
 207                 /* Unsupported */
 208                 *ib_speed = IB_SPEED_SDR;
 209                 *ib_width = IB_WIDTH_1X;
 210         }
 211 }
 212
 213 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
 214 {
 215         struct qedr_dev *dev;
 216         struct qed_rdma_port *rdma_port;
 217
 218         dev = get_qedr_dev(ibdev);
 219
 220         if (!dev->rdma_ctx) {
 221                 DP_ERR(dev, "rdma_ctx is NULL\n");
 222                 return -EINVAL;
 223         }
 224
 225         rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
 226
 227         /* *attr being zeroed by the caller, avoid zeroing it here */
 228         if (rdma_port->port_state == QED_RDMA_PORT_UP) {
 229                 attr->state = IB_PORT_ACTIVE;
 230                 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 231         } else {
 232                 attr->state = IB_PORT_DOWN;
 233                 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 234         }
 235         attr->max_mtu = IB_MTU_4096;
 236         attr->lid = 0;
 237         attr->lmc = 0;
 238         attr->sm_lid = 0;
 239         attr->sm_sl = 0;
 240         attr->ip_gids = true;
 241         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
 242                 attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu);
 243                 attr->gid_tbl_len = 1;
 244         } else {
 245                 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
 246                 attr->gid_tbl_len = QEDR_MAX_SGID;
 247                 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
 248         }
 249         attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
 250         attr->qkey_viol_cntr = 0;
 251         get_link_speed_and_width(rdma_port->link_speed,
 252                                  &attr->active_speed, &attr->active_width);
 253         attr->max_msg_sz = rdma_port->max_msg_size;
 254         attr->max_vl_num = 4;
 255
 256         return 0;
 257 }
 258
 259 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
 260 {
 261         struct ib_device *ibdev = uctx->device;
 262         int rc;
 263         struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
 264         struct qedr_alloc_ucontext_resp uresp = {};
 265         struct qedr_alloc_ucontext_req ureq = {};
 266         struct qedr_dev *dev = get_qedr_dev(ibdev);
 267         struct qed_rdma_add_user_out_params oparams;
 268         struct qedr_user_mmap_entry *entry;
 269
 270         if (!udata)
 271                 return -EFAULT;
 272
 273         if (udata->inlen) {
 274                 rc = ib_copy_from_udata(&ureq, udata,
 275                                         min(sizeof(ureq), udata->inlen));
 276                 if (rc) {
 277                         DP_ERR(dev, "Problem copying data from user space\n");
 278                         return -EFAULT;
 279                 }
 280                 ctx->edpm_mode = !!(ureq.context_flags &
 281                                     QEDR_ALLOC_UCTX_EDPM_MODE);
 282                 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
 283         }
 284
 285         rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
 286         if (rc) {
 287                 DP_ERR(dev,
 288                        "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
 289                        rc);
 290                 return rc;
 291         }
 292
 293         ctx->dpi = oparams.dpi;
 294         ctx->dpi_addr = oparams.dpi_addr;
 295         ctx->dpi_phys_addr = oparams.dpi_phys_addr;
 296         ctx->dpi_size = oparams.dpi_size;
 297         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 298         if (!entry) {
 299                 rc = -ENOMEM;
 300                 goto err;
 301         }
 302
 303         entry->io_address = ctx->dpi_phys_addr;
 304         entry->length = ctx->dpi_size;
 305         entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
 306         entry->dpi = ctx->dpi;
 307         entry->dev = dev;
 308         rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,
 309                                          ctx->dpi_size);
 310         if (rc) {
 311                 kfree(entry);
 312                 goto err;
 313         }
 314         ctx->db_mmap_entry = &entry->rdma_entry;
 315
 316         if (!dev->user_dpm_enabled)
 317                 uresp.dpm_flags = 0;
 318         else if (rdma_protocol_iwarp(&dev->ibdev, 1))
 319                 uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
 320         else
 321                 uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
 322                                   QEDR_DPM_TYPE_ROCE_LEGACY |
 323                                   QEDR_DPM_TYPE_ROCE_EDPM_MODE;
 324
 325         if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) {
 326                 uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
 327                 uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
 328                 uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
 329                 uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE;
 330         }
 331
 332         uresp.wids_enabled = 1;
 333         uresp.wid_count = oparams.wid_count;
 334         uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);
 335         uresp.db_size = ctx->dpi_size;
 336         uresp.max_send_wr = dev->attr.max_sqe;
 337         uresp.max_recv_wr = dev->attr.max_rqe;
 338         uresp.max_srq_wr = dev->attr.max_srq_wr;
 339         uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
 340         uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
 341         uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
 342         uresp.max_cqes = QEDR_MAX_CQES;
 343
 344         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 345         if (rc)
 346                 goto err;
 347
 348         ctx->dev = dev;
 349
 350         DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
 351                  &ctx->ibucontext);
 352         return 0;
 353
 354 err:
 355         if (!ctx->db_mmap_entry)
 356                 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
 357         else
 358                 rdma_user_mmap_entry_remove(ctx->db_mmap_entry);
 359
 360         return rc;
 361 }
 362
 363 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
 364 {
 365         struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
 366
 367         DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
 368                  uctx);
 369
 370         rdma_user_mmap_entry_remove(uctx->db_mmap_entry);
 371 }
 372
 373 void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
 374 {
 375         struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
 376         struct qedr_dev *dev = entry->dev;
 377
 378         if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
 379                 free_page((unsigned long)entry->address);
 380         else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
 381                 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
 382
 383         kfree(entry);
 384 }
 385
 386 int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
 387 {
 388         struct ib_device *dev = ucontext->device;
 389         size_t length = vma->vm_end - vma->vm_start;
 390         struct rdma_user_mmap_entry *rdma_entry;
 391         struct qedr_user_mmap_entry *entry;
 392         int rc = 0;
 393         u64 pfn;
 394
 395         ibdev_dbg(dev,
 396                   "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
 397                   vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
 398
 399         rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
 400         if (!rdma_entry) {
 401                 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
 402                           vma->vm_pgoff);
 403                 return -EINVAL;
 404         }
 405         entry = get_qedr_mmap_entry(rdma_entry);
 406         ibdev_dbg(dev,
 407                   "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
 408                   entry->io_address, length, entry->mmap_flag);
 409
 410         switch (entry->mmap_flag) {
 411         case QEDR_USER_MMAP_IO_WC:
 412                 pfn = entry->io_address >> PAGE_SHIFT;
 413                 rc = rdma_user_mmap_io(ucontext, vma, pfn, length,
 414                                        pgprot_writecombine(vma->vm_page_prot),
 415                                        rdma_entry);
 416                 break;
 417         case QEDR_USER_MMAP_PHYS_PAGE:
 418                 rc = vm_insert_page(vma, vma->vm_start,
 419                                     virt_to_page(entry->address));
 420                 break;
 421         default:
 422                 rc = -EINVAL;
 423         }
 424
 425         if (rc)
 426                 ibdev_dbg(dev,
 427                           "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
 428                           entry->io_address, length, entry->mmap_flag, rc);
 429
 430         rdma_user_mmap_entry_put(rdma_entry);
 431         return rc;
 432 }
 433
 434 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 435 {
 436         struct ib_device *ibdev = ibpd->device;
 437         struct qedr_dev *dev = get_qedr_dev(ibdev);
 438         struct qedr_pd *pd = get_qedr_pd(ibpd);
 439         u16 pd_id;
 440         int rc;
 441
 442         DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
 443                  udata ? "User Lib" : "Kernel");
 444
 445         if (!dev->rdma_ctx) {
 446                 DP_ERR(dev, "invalid RDMA context\n");
 447                 return -EINVAL;
 448         }
 449
 450         rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
 451         if (rc)
 452                 return rc;
 453
 454         pd->pd_id = pd_id;
 455
 456         if (udata) {
 457                 struct qedr_alloc_pd_uresp uresp = {
 458                         .pd_id = pd_id,
 459                 };
 460                 struct qedr_ucontext *context = rdma_udata_to_drv_context(
 461                         udata, struct qedr_ucontext, ibucontext);
 462
 463                 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 464                 if (rc) {
 465                         DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
 466                         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
 467                         return rc;
 468                 }
 469
 470                 pd->uctx = context;
 471                 pd->uctx->pd = pd;
 472         }
 473
 474         return 0;
 475 }
 476
 477 int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 478 {
 479         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
 480         struct qedr_pd *pd = get_qedr_pd(ibpd);
 481
 482         DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
 483         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
 484         return 0;
 485 }
 486
 487
 488 int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
 489 {
 490         struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
 491         struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd);
 492
 493         return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id);
 494 }
 495
 496 int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
 497 {
 498         struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
 499         u16 xrcd_id = get_qedr_xrcd(ibxrcd)->xrcd_id;
 500
 501         dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id);
 502         return 0;
 503 }
 504 static void qedr_free_pbl(struct qedr_dev *dev,
 505                           struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
 506 {
 507         struct pci_dev *pdev = dev->pdev;
 508         int i;
 509
 510         for (i = 0; i < pbl_info->num_pbls; i++) {
 511                 if (!pbl[i].va)
 512                         continue;
 513                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 514                                   pbl[i].va, pbl[i].pa);
 515         }
 516
 517         kfree(pbl);
 518 }
 519
 520 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
 521 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
 522
 523 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
 524 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
 525 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
 526
 527 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
 528                                            struct qedr_pbl_info *pbl_info,
 529                                            gfp_t flags)
 530 {
 531         struct pci_dev *pdev = dev->pdev;
 532         struct qedr_pbl *pbl_table;
 533         dma_addr_t *pbl_main_tbl;
 534         dma_addr_t pa;
 535         void *va;
 536         int i;
 537
 538         pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
 539         if (!pbl_table)
 540                 return ERR_PTR(-ENOMEM);
 541
 542         for (i = 0; i < pbl_info->num_pbls; i++) {
 543                 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
 544                                         flags);
 545                 if (!va)
 546                         goto err;
 547
 548                 pbl_table[i].va = va;
 549                 pbl_table[i].pa = pa;
 550         }
 551
 552         /* Two-Layer PBLs, if we have more than one pbl we need to initialize
 553          * the first one with physical pointers to all of the rest
 554          */
 555         pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
 556         for (i = 0; i < pbl_info->num_pbls - 1; i++)
 557                 pbl_main_tbl[i] = pbl_table[i + 1].pa;
 558
 559         return pbl_table;
 560
 561 err:
 562         for (i--; i >= 0; i--)
 563                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 564                                   pbl_table[i].va, pbl_table[i].pa);
 565
 566         qedr_free_pbl(dev, pbl_info, pbl_table);
 567
 568         return ERR_PTR(-ENOMEM);
 569 }
 570
 571 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
 572                                 struct qedr_pbl_info *pbl_info,
 573                                 u32 num_pbes, int two_layer_capable)
 574 {
 575         u32 pbl_capacity;
 576         u32 pbl_size;
 577         u32 num_pbls;
 578
 579         if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
 580                 if (num_pbes > MAX_PBES_TWO_LAYER) {
 581                         DP_ERR(dev, "prepare pbl table: too many pages %d\n",
 582                                num_pbes);
 583                         return -EINVAL;
 584                 }
 585
 586                 /* calculate required pbl page size */
 587                 pbl_size = MIN_FW_PBL_PAGE_SIZE;
 588                 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
 589                                NUM_PBES_ON_PAGE(pbl_size);
 590
 591                 while (pbl_capacity < num_pbes) {
 592                         pbl_size *= 2;
 593                         pbl_capacity = pbl_size / sizeof(u64);
 594                         pbl_capacity = pbl_capacity * pbl_capacity;
 595                 }
 596
 597                 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
 598                 num_pbls++;     /* One for the layer0 ( points to the pbls) */
 599                 pbl_info->two_layered = true;
 600         } else {
 601                 /* One layered PBL */
 602                 num_pbls = 1;
 603                 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
 604                                  roundup_pow_of_two((num_pbes * sizeof(u64))));
 605                 pbl_info->two_layered = false;
 606         }
 607
 608         pbl_info->num_pbls = num_pbls;
 609         pbl_info->pbl_size = pbl_size;
 610         pbl_info->num_pbes = num_pbes;
 611
 612         DP_DEBUG(dev, QEDR_MSG_MR,
 613                  "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
 614                  pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
 615
 616         return 0;
 617 }
 618
 619 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
 620                                struct qedr_pbl *pbl,
 621                                struct qedr_pbl_info *pbl_info, u32 pg_shift)
 622 {
 623         int pbe_cnt, total_num_pbes = 0;
 624         struct qedr_pbl *pbl_tbl;
 625         struct ib_block_iter biter;
 626         struct regpair *pbe;
 627
 628         if (!pbl_info->num_pbes)
 629                 return;
 630
 631         /* If we have a two layered pbl, the first pbl points to the rest
 632          * of the pbls and the first entry lays on the second pbl in the table
 633          */
 634         if (pbl_info->two_layered)
 635                 pbl_tbl = &pbl[1];
 636         else
 637                 pbl_tbl = pbl;
 638
 639         pbe = (struct regpair *)pbl_tbl->va;
 640         if (!pbe) {
 641                 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
 642                 return;
 643         }
 644
 645         pbe_cnt = 0;
 646
 647         rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) {
 648                 u64 pg_addr = rdma_block_iter_dma_address(&biter);
 649
 650                 pbe->lo = cpu_to_le32(pg_addr);
 651                 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
 652
 653                 pbe_cnt++;
 654                 total_num_pbes++;
 655                 pbe++;
 656
 657                 if (total_num_pbes == pbl_info->num_pbes)
 658                         return;
 659
 660                 /* If the given pbl is full storing the pbes, move to next pbl.
 661                  */
 662                 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
 663                         pbl_tbl++;
 664                         pbe = (struct regpair *)pbl_tbl->va;
 665                         pbe_cnt = 0;
 666                 }
 667         }
 668 }
 669
 670 static int qedr_db_recovery_add(struct qedr_dev *dev,
 671                                 void __iomem *db_addr,
 672                                 void *db_data,
 673                                 enum qed_db_rec_width db_width,
 674                                 enum qed_db_rec_space db_space)
 675 {
 676         if (!db_data) {
 677                 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
 678                 return 0;
 679         }
 680
 681         return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
 682                                                  db_width, db_space);
 683 }
 684
 685 static void qedr_db_recovery_del(struct qedr_dev *dev,
 686                                  void __iomem *db_addr,
 687                                  void *db_data)
 688 {
 689         if (!db_data) {
 690                 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
 691                 return;
 692         }
 693
 694         /* Ignore return code as there is not much we can do about it. Error
 695          * log will be printed inside.
 696          */
 697         dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
 698 }
 699
 700 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
 701                               struct qedr_cq *cq, struct ib_udata *udata,
 702                               u32 db_offset)
 703 {
 704         struct qedr_create_cq_uresp uresp;
 705         int rc;
 706
 707         memset(&uresp, 0, sizeof(uresp));
 708
 709         uresp.db_offset = db_offset;
 710         uresp.icid = cq->icid;
 711         if (cq->q.db_mmap_entry)
 712                 uresp.db_rec_addr =
 713                         rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
 714
 715         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 716         if (rc)
 717                 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
 718
 719         return rc;
 720 }
 721
 722 static void consume_cqe(struct qedr_cq *cq)
 723 {
 724         if (cq->latest_cqe == cq->toggle_cqe)
 725                 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 726
 727         cq->latest_cqe = qed_chain_consume(&cq->pbl);
 728 }
 729
 730 static inline int qedr_align_cq_entries(int entries)
 731 {
 732         u64 size, aligned_size;
 733
 734         /* We allocate an extra entry that we don't report to the FW. */
 735         size = (entries + 1) * QEDR_CQE_SIZE;
 736         aligned_size = ALIGN(size, PAGE_SIZE);
 737
 738         return aligned_size / QEDR_CQE_SIZE;
 739 }
 740
 741 static int qedr_init_user_db_rec(struct ib_udata *udata,
 742                                  struct qedr_dev *dev, struct qedr_userq *q,
 743                                  bool requires_db_rec)
 744 {
 745         struct qedr_ucontext *uctx =
 746                 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
 747                                           ibucontext);
 748         struct qedr_user_mmap_entry *entry;
 749         int rc;
 750
 751         /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
 752         if (requires_db_rec == 0 || !uctx->db_rec)
 753                 return 0;
 754
 755         /* Allocate a page for doorbell recovery, add to mmap */
 756         q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
 757         if (!q->db_rec_data) {
 758                 DP_ERR(dev, "get_zeroed_page failed\n");
 759                 return -ENOMEM;
 760         }
 761
 762         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 763         if (!entry)
 764                 goto err_free_db_data;
 765
 766         entry->address = q->db_rec_data;
 767         entry->length = PAGE_SIZE;
 768         entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
 769         rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,
 770                                          &entry->rdma_entry,
 771                                          PAGE_SIZE);
 772         if (rc)
 773                 goto err_free_entry;
 774
 775         q->db_mmap_entry = &entry->rdma_entry;
 776
 777         return 0;
 778
 779 err_free_entry:
 780         kfree(entry);
 781
 782 err_free_db_data:
 783         free_page((unsigned long)q->db_rec_data);
 784         q->db_rec_data = NULL;
 785         return -ENOMEM;
 786 }
 787
 788 static inline int qedr_init_user_queue(struct ib_udata *udata,
 789                                        struct qedr_dev *dev,
 790                                        struct qedr_userq *q, u64 buf_addr,
 791                                        size_t buf_len, bool requires_db_rec,
 792                                        int access,
 793                                        int alloc_and_init)
 794 {
 795         u32 fw_pages;
 796         int rc;
 797
 798         q->buf_addr = buf_addr;
 799         q->buf_len = buf_len;
 800         q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
 801         if (IS_ERR(q->umem)) {
 802                 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
 803                        PTR_ERR(q->umem));
 804                 return PTR_ERR(q->umem);
 805         }
 806
 807         fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT);
 808         rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
 809         if (rc)
 810                 goto err0;
 811
 812         if (alloc_and_init) {
 813                 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
 814                 if (IS_ERR(q->pbl_tbl)) {
 815                         rc = PTR_ERR(q->pbl_tbl);
 816                         goto err0;
 817                 }
 818                 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
 819                                    FW_PAGE_SHIFT);
 820         } else {
 821                 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
 822                 if (!q->pbl_tbl) {
 823                         rc = -ENOMEM;
 824                         goto err0;
 825                 }
 826         }
 827
 828         /* mmap the user address used to store doorbell data for recovery */
 829         return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
 830
 831 err0:
 832         ib_umem_release(q->umem);
 833         q->umem = NULL;
 834
 835         return rc;
 836 }
 837
 838 static inline void qedr_init_cq_params(struct qedr_cq *cq,
 839                                        struct qedr_ucontext *ctx,
 840                                        struct qedr_dev *dev, int vector,
 841                                        int chain_entries, int page_cnt,
 842                                        u64 pbl_ptr,
 843                                        struct qed_rdma_create_cq_in_params
 844                                        *params)
 845 {
 846         memset(params, 0, sizeof(*params));
 847         params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
 848         params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
 849         params->cnq_id = vector;
 850         params->cq_size = chain_entries - 1;
 851         params->dpi = (ctx) ? ctx->dpi : dev->dpi;
 852         params->pbl_num_pages = page_cnt;
 853         params->pbl_ptr = pbl_ptr;
 854         params->pbl_two_level = 0;
 855 }
 856
 857 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
 858 {
 859         cq->db.data.agg_flags = flags;
 860         cq->db.data.value = cpu_to_le32(cons);
 861         writeq(cq->db.raw, cq->db_addr);
 862 }
 863
 864 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 865 {
 866         struct qedr_cq *cq = get_qedr_cq(ibcq);
 867         unsigned long sflags;
 868         struct qedr_dev *dev;
 869
 870         dev = get_qedr_dev(ibcq->device);
 871
 872         if (cq->destroyed) {
 873                 DP_ERR(dev,
 874                        "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
 875                        cq, cq->icid);
 876                 return -EINVAL;
 877         }
 878
 879
 880         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
 881                 return 0;
 882
 883         spin_lock_irqsave(&cq->cq_lock, sflags);
 884
 885         cq->arm_flags = 0;
 886
 887         if (flags & IB_CQ_SOLICITED)
 888                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
 889
 890         if (flags & IB_CQ_NEXT_COMP)
 891                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
 892
 893         doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
 894
 895         spin_unlock_irqrestore(&cq->cq_lock, sflags);
 896
 897         return 0;
 898 }
 899
 900 int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 901                    struct ib_udata *udata)
 902 {
 903         struct ib_device *ibdev = ibcq->device;
 904         struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
 905                 udata, struct qedr_ucontext, ibucontext);
 906         struct qed_rdma_destroy_cq_out_params destroy_oparams;
 907         struct qed_rdma_destroy_cq_in_params destroy_iparams;
 908         struct qed_chain_init_params chain_params = {
 909                 .mode           = QED_CHAIN_MODE_PBL,
 910                 .intended_use   = QED_CHAIN_USE_TO_CONSUME,
 911                 .cnt_type       = QED_CHAIN_CNT_TYPE_U32,
 912                 .elem_size      = sizeof(union rdma_cqe),
 913         };
 914         struct qedr_dev *dev = get_qedr_dev(ibdev);
 915         struct qed_rdma_create_cq_in_params params;
 916         struct qedr_create_cq_ureq ureq = {};
 917         int vector = attr->comp_vector;
 918         int entries = attr->cqe;
 919         struct qedr_cq *cq = get_qedr_cq(ibcq);
 920         int chain_entries;
 921         u32 db_offset;
 922         int page_cnt;
 923         u64 pbl_ptr;
 924         u16 icid;
 925         int rc;
 926
 927         DP_DEBUG(dev, QEDR_MSG_INIT,
 928                  "create_cq: called from %s. entries=%d, vector=%d\n",
 929                  udata ? "User Lib" : "Kernel", entries, vector);
 930
 931         if (attr->flags)
 932                 return -EOPNOTSUPP;
 933
 934         if (entries > QEDR_MAX_CQES) {
 935                 DP_ERR(dev,
 936                        "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
 937                        entries, QEDR_MAX_CQES);
 938                 return -EINVAL;
 939         }
 940
 941         chain_entries = qedr_align_cq_entries(entries);
 942         chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
 943         chain_params.num_elems = chain_entries;
 944
 945         /* calc db offset. user will add DPI base, kernel will add db addr */
 946         db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 947
 948         if (udata) {
 949                 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
 950                                                          udata->inlen))) {
 951                         DP_ERR(dev,
 952                                "create cq: problem copying data from user space\n");
 953                         goto err0;
 954                 }
 955
 956                 if (!ureq.len) {
 957                         DP_ERR(dev,
 958                                "create cq: cannot create a cq with 0 entries\n");
 959                         goto err0;
 960                 }
 961
 962                 cq->cq_type = QEDR_CQ_TYPE_USER;
 963
 964                 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
 965                                           ureq.len, true, IB_ACCESS_LOCAL_WRITE,
 966                                           1);
 967                 if (rc)
 968                         goto err0;
 969
 970                 pbl_ptr = cq->q.pbl_tbl->pa;
 971                 page_cnt = cq->q.pbl_info.num_pbes;
 972
 973                 cq->ibcq.cqe = chain_entries;
 974                 cq->q.db_addr = ctx->dpi_addr + db_offset;
 975         } else {
 976                 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
 977
 978                 rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl,
 979                                                    &chain_params);
 980                 if (rc)
 981                         goto err0;
 982
 983                 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
 984                 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
 985                 cq->ibcq.cqe = cq->pbl.capacity;
 986         }
 987
 988         qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
 989                             pbl_ptr, &params);
 990
 991         rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
 992         if (rc)
 993                 goto err1;
 994
 995         cq->icid = icid;
 996         cq->sig = QEDR_CQ_MAGIC_NUMBER;
 997         spin_lock_init(&cq->cq_lock);
 998
 999         if (udata) {
1000                 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
1001                 if (rc)
1002                         goto err2;
1003
1004                 rc = qedr_db_recovery_add(dev, cq->q.db_addr,
1005                                           &cq->q.db_rec_data->db_data,
1006                                           DB_REC_WIDTH_64B,
1007                                           DB_REC_USER);
1008                 if (rc)
1009                         goto err2;
1010
1011         } else {
1012                 /* Generate doorbell address. */
1013                 cq->db.data.icid = cq->icid;
1014                 cq->db_addr = dev->db_addr + db_offset;
1015                 cq->db.data.params = DB_AGG_CMD_MAX <<
1016                     RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1017
1018                 /* point to the very last element, passing it we will toggle */
1019                 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
1020                 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1021                 cq->latest_cqe = NULL;
1022                 consume_cqe(cq);
1023                 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
1024
1025                 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
1026                                           DB_REC_WIDTH_64B, DB_REC_KERNEL);
1027                 if (rc)
1028                         goto err2;
1029         }
1030
1031         DP_DEBUG(dev, QEDR_MSG_CQ,
1032                  "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1033                  cq->icid, cq, params.cq_size);
1034
1035         return 0;
1036
1037 err2:
1038         destroy_iparams.icid = cq->icid;
1039         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1040                                   &destroy_oparams);
1041 err1:
1042         if (udata) {
1043                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1044                 ib_umem_release(cq->q.umem);
1045                 if (cq->q.db_mmap_entry)
1046                         rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1047         } else {
1048                 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1049         }
1050 err0:
1051         return -EINVAL;
1052 }
1053
1054 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
1055 {
1056         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1057         struct qedr_cq *cq = get_qedr_cq(ibcq);
1058
1059         DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1060
1061         return 0;
1062 }
1063
1064 #define QEDR_DESTROY_CQ_MAX_ITERATIONS          (10)
1065 #define QEDR_DESTROY_CQ_ITER_DURATION           (10)
1066
1067 int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1068 {
1069         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1070         struct qed_rdma_destroy_cq_out_params oparams;
1071         struct qed_rdma_destroy_cq_in_params iparams;
1072         struct qedr_cq *cq = get_qedr_cq(ibcq);
1073         int iter;
1074
1075         DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1076
1077         cq->destroyed = 1;
1078
1079         /* GSIs CQs are handled by driver, so they don't exist in the FW */
1080         if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1081                 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1082                 return 0;
1083         }
1084
1085         iparams.icid = cq->icid;
1086         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1087         dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1088
1089         if (udata) {
1090                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1091                 ib_umem_release(cq->q.umem);
1092
1093                 if (cq->q.db_rec_data) {
1094                         qedr_db_recovery_del(dev, cq->q.db_addr,
1095                                              &cq->q.db_rec_data->db_data);
1096                         rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1097                 }
1098         } else {
1099                 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1100         }
1101
1102         /* We don't want the IRQ handler to handle a non-existing CQ so we
1103          * wait until all CNQ interrupts, if any, are received. This will always
1104          * happen and will always happen very fast. If not, then a serious error
1105          * has occured. That is why we can use a long delay.
1106          * We spin for a short time so we don’t lose time on context switching
1107          * in case all the completions are handled in that span. Otherwise
1108          * we sleep for a while and check again. Since the CNQ may be
1109          * associated with (only) the current CPU we use msleep to allow the
1110          * current CPU to be freed.
1111          * The CNQ notification is increased in qedr_irq_handler().
1112          */
1113         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1114         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1115                 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1116                 iter--;
1117         }
1118
1119         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1120         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1121                 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1122                 iter--;
1123         }
1124
1125         /* Note that we don't need to have explicit code to wait for the
1126          * completion of the event handler because it is invoked from the EQ.
1127          * Since the destroy CQ ramrod has also been received on the EQ we can
1128          * be certain that there's no event handler in process.
1129          */
1130         return 0;
1131 }
1132
1133 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1134                                           struct ib_qp_attr *attr,
1135                                           int attr_mask,
1136                                           struct qed_rdma_modify_qp_in_params
1137                                           *qp_params)
1138 {
1139         const struct ib_gid_attr *gid_attr;
1140         enum rdma_network_type nw_type;
1141         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1142         u32 ipv4_addr;
1143         int ret;
1144         int i;
1145
1146         gid_attr = grh->sgid_attr;
1147         ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1148         if (ret)
1149                 return ret;
1150
1151         nw_type = rdma_gid_attr_network_type(gid_attr);
1152         switch (nw_type) {
1153         case RDMA_NETWORK_IPV6:
1154                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1155                        sizeof(qp_params->sgid));
1156                 memcpy(&qp_params->dgid.bytes[0],
1157                        &grh->dgid,
1158                        sizeof(qp_params->dgid));
1159                 qp_params->roce_mode = ROCE_V2_IPV6;
1160                 SET_FIELD(qp_params->modify_flags,
1161                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1162                 break;
1163         case RDMA_NETWORK_ROCE_V1:
1164                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1165                        sizeof(qp_params->sgid));
1166                 memcpy(&qp_params->dgid.bytes[0],
1167                        &grh->dgid,
1168                        sizeof(qp_params->dgid));
1169                 qp_params->roce_mode = ROCE_V1;
1170                 break;
1171         case RDMA_NETWORK_IPV4:
1172                 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1173                 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1174                 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1175                 qp_params->sgid.ipv4_addr = ipv4_addr;
1176                 ipv4_addr =
1177                     qedr_get_ipv4_from_gid(grh->dgid.raw);
1178                 qp_params->dgid.ipv4_addr = ipv4_addr;
1179                 SET_FIELD(qp_params->modify_flags,
1180                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1181                 qp_params->roce_mode = ROCE_V2_IPV4;
1182                 break;
1183         default:
1184                 return -EINVAL;
1185         }
1186
1187         for (i = 0; i < 4; i++) {
1188                 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1189                 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1190         }
1191
1192         if (qp_params->vlan_id >= VLAN_CFI_MASK)
1193                 qp_params->vlan_id = 0;
1194
1195         return 0;
1196 }
1197
1198 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1199                                struct ib_qp_init_attr *attrs,
1200                                struct ib_udata *udata)
1201 {
1202         struct qedr_device_attr *qattr = &dev->attr;
1203
1204         /* QP0... attrs->qp_type == IB_QPT_GSI */
1205         if (attrs->qp_type != IB_QPT_RC &&
1206             attrs->qp_type != IB_QPT_GSI &&
1207             attrs->qp_type != IB_QPT_XRC_INI &&
1208             attrs->qp_type != IB_QPT_XRC_TGT) {
1209                 DP_DEBUG(dev, QEDR_MSG_QP,
1210                          "create qp: unsupported qp type=0x%x requested\n",
1211                          attrs->qp_type);
1212                 return -EOPNOTSUPP;
1213         }
1214
1215         if (attrs->cap.max_send_wr > qattr->max_sqe) {
1216                 DP_ERR(dev,
1217                        "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1218                        attrs->cap.max_send_wr, qattr->max_sqe);
1219                 return -EINVAL;
1220         }
1221
1222         if (attrs->cap.max_inline_data > qattr->max_inline) {
1223                 DP_ERR(dev,
1224                        "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1225                        attrs->cap.max_inline_data, qattr->max_inline);
1226                 return -EINVAL;
1227         }
1228
1229         if (attrs->cap.max_send_sge > qattr->max_sge) {
1230                 DP_ERR(dev,
1231                        "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1232                        attrs->cap.max_send_sge, qattr->max_sge);
1233                 return -EINVAL;
1234         }
1235
1236         if (attrs->cap.max_recv_sge > qattr->max_sge) {
1237                 DP_ERR(dev,
1238                        "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1239                        attrs->cap.max_recv_sge, qattr->max_sge);
1240                 return -EINVAL;
1241         }
1242
1243         /* verify consumer QPs are not trying to use GSI QP's CQ.
1244          * TGT QP isn't associated with RQ/SQ
1245          */
1246         if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) &&
1247             (attrs->qp_type != IB_QPT_XRC_TGT) &&
1248             (attrs->qp_type != IB_QPT_XRC_INI)) {
1249                 struct qedr_cq *send_cq = get_qedr_cq(attrs->send_cq);
1250                 struct qedr_cq *recv_cq = get_qedr_cq(attrs->recv_cq);
1251
1252                 if ((send_cq->cq_type == QEDR_CQ_TYPE_GSI) ||
1253                     (recv_cq->cq_type == QEDR_CQ_TYPE_GSI)) {
1254                         DP_ERR(dev,
1255                                "create qp: consumer QP cannot use GSI CQs.\n");
1256                         return -EINVAL;
1257                 }
1258         }
1259
1260         return 0;
1261 }
1262
1263 static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1264                                struct qedr_srq *srq, struct ib_udata *udata)
1265 {
1266         struct qedr_create_srq_uresp uresp = {};
1267         int rc;
1268
1269         uresp.srq_id = srq->srq_id;
1270
1271         rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1272         if (rc)
1273                 DP_ERR(dev, "create srq: problem copying data to user space\n");
1274
1275         return rc;
1276 }
1277
1278 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1279                                struct qedr_create_qp_uresp *uresp,
1280                                struct qedr_qp *qp)
1281 {
1282         /* iWARP requires two doorbells per RQ. */
1283         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1284                 uresp->rq_db_offset =
1285                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1286                 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1287         } else {
1288                 uresp->rq_db_offset =
1289                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1290         }
1291
1292         uresp->rq_icid = qp->icid;
1293         if (qp->urq.db_mmap_entry)
1294                 uresp->rq_db_rec_addr =
1295                         rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
1296 }
1297
1298 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1299                                struct qedr_create_qp_uresp *uresp,
1300                                struct qedr_qp *qp)
1301 {
1302         uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1303
1304         /* iWARP uses the same cid for rq and sq */
1305         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1306                 uresp->sq_icid = qp->icid;
1307         else
1308                 uresp->sq_icid = qp->icid + 1;
1309
1310         if (qp->usq.db_mmap_entry)
1311                 uresp->sq_db_rec_addr =
1312                         rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
1313 }
1314
1315 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1316                               struct qedr_qp *qp, struct ib_udata *udata,
1317                               struct qedr_create_qp_uresp *uresp)
1318 {
1319         int rc;
1320
1321         memset(uresp, 0, sizeof(*uresp));
1322
1323         if (qedr_qp_has_sq(qp))
1324                 qedr_copy_sq_uresp(dev, uresp, qp);
1325
1326         if (qedr_qp_has_rq(qp))
1327                 qedr_copy_rq_uresp(dev, uresp, qp);
1328
1329         uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1330         uresp->qp_id = qp->qp_id;
1331
1332         rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
1333         if (rc)
1334                 DP_ERR(dev,
1335                        "create qp: failed a copy to user space with qp icid=0x%x.\n",
1336                        qp->icid);
1337
1338         return rc;
1339 }
1340
1341 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1342                                       struct qedr_qp *qp,
1343                                       struct qedr_pd *pd,
1344                                       struct ib_qp_init_attr *attrs)
1345 {
1346         spin_lock_init(&qp->q_lock);
1347         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1348                 kref_init(&qp->refcnt);
1349                 init_completion(&qp->iwarp_cm_comp);
1350         }
1351
1352         qp->pd = pd;
1353         qp->qp_type = attrs->qp_type;
1354         qp->max_inline_data = attrs->cap.max_inline_data;
1355         qp->state = QED_ROCE_QP_STATE_RESET;
1356         qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1357         qp->dev = dev;
1358         if (qedr_qp_has_sq(qp)) {
1359                 qp->sq.max_sges = attrs->cap.max_send_sge;
1360                 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1361                 DP_DEBUG(dev, QEDR_MSG_QP,
1362                          "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1363                          qp->sq.max_sges, qp->sq_cq->icid);
1364         }
1365
1366         if (attrs->srq)
1367                 qp->srq = get_qedr_srq(attrs->srq);
1368
1369         if (qedr_qp_has_rq(qp)) {
1370                 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1371                 qp->rq.max_sges = attrs->cap.max_recv_sge;
1372                 DP_DEBUG(dev, QEDR_MSG_QP,
1373                          "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1374                          qp->rq.max_sges, qp->rq_cq->icid);
1375         }
1376
1377         DP_DEBUG(dev, QEDR_MSG_QP,
1378                  "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1379                  pd->pd_id, qp->qp_type, qp->max_inline_data,
1380                  qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1381         DP_DEBUG(dev, QEDR_MSG_QP,
1382                  "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1383                  qp->sq.max_sges, qp->sq_cq->icid);
1384 }
1385
1386 static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1387 {
1388         int rc = 0;
1389
1390         if (qedr_qp_has_sq(qp)) {
1391                 qp->sq.db = dev->db_addr +
1392                             DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1393                 qp->sq.db_data.data.icid = qp->icid + 1;
1394                 rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data,
1395                                           DB_REC_WIDTH_32B, DB_REC_KERNEL);
1396                 if (rc)
1397                         return rc;
1398         }
1399
1400         if (qedr_qp_has_rq(qp)) {
1401                 qp->rq.db = dev->db_addr +
1402                             DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1403                 qp->rq.db_data.data.icid = qp->icid;
1404                 rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data,
1405                                           DB_REC_WIDTH_32B, DB_REC_KERNEL);
1406                 if (rc && qedr_qp_has_sq(qp))
1407                         qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
1408         }
1409
1410         return rc;
1411 }
1412
1413 static int qedr_check_srq_params(struct qedr_dev *dev,
1414                                  struct ib_srq_init_attr *attrs,
1415                                  struct ib_udata *udata)
1416 {
1417         struct qedr_device_attr *qattr = &dev->attr;
1418
1419         if (attrs->attr.max_wr > qattr->max_srq_wr) {
1420                 DP_ERR(dev,
1421                        "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1422                        attrs->attr.max_wr, qattr->max_srq_wr);
1423                 return -EINVAL;
1424         }
1425
1426         if (attrs->attr.max_sge > qattr->max_sge) {
1427                 DP_ERR(dev,
1428                        "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1429                        attrs->attr.max_sge, qattr->max_sge);
1430         }
1431
1432         if (!udata && attrs->srq_type == IB_SRQT_XRC) {
1433                 DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n");
1434                 return -EINVAL;
1435         }
1436
1437         return 0;
1438 }
1439
1440 static void qedr_free_srq_user_params(struct qedr_srq *srq)
1441 {
1442         qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1443         ib_umem_release(srq->usrq.umem);
1444         ib_umem_release(srq->prod_umem);
1445 }
1446
1447 static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1448 {
1449         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1450         struct qedr_dev *dev = srq->dev;
1451
1452         dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1453
1454         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1455                           hw_srq->virt_prod_pair_addr,
1456                           hw_srq->phy_prod_pair_addr);
1457 }
1458
1459 static int qedr_init_srq_user_params(struct ib_udata *udata,
1460                                      struct qedr_srq *srq,
1461                                      struct qedr_create_srq_ureq *ureq,
1462                                      int access)
1463 {
1464         struct scatterlist *sg;
1465         int rc;
1466
1467         rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1468                                   ureq->srq_len, false, access, 1);
1469         if (rc)
1470                 return rc;
1471
1472         srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,
1473                                      sizeof(struct rdma_srq_producers), access);
1474         if (IS_ERR(srq->prod_umem)) {
1475                 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1476                 ib_umem_release(srq->usrq.umem);
1477                 DP_ERR(srq->dev,
1478                        "create srq: failed ib_umem_get for producer, got %ld\n",
1479                        PTR_ERR(srq->prod_umem));
1480                 return PTR_ERR(srq->prod_umem);
1481         }
1482
1483         sg = srq->prod_umem->sg_head.sgl;
1484         srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1485
1486         return 0;
1487 }
1488
1489 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1490                                         struct qedr_dev *dev,
1491                                         struct ib_srq_init_attr *init_attr)
1492 {
1493         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1494         struct qed_chain_init_params params = {
1495                 .mode           = QED_CHAIN_MODE_PBL,
1496                 .intended_use   = QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1497                 .cnt_type       = QED_CHAIN_CNT_TYPE_U32,
1498                 .elem_size      = QEDR_SRQ_WQE_ELEM_SIZE,
1499         };
1500         dma_addr_t phy_prod_pair_addr;
1501         u32 num_elems;
1502         void *va;
1503         int rc;
1504
1505         va = dma_alloc_coherent(&dev->pdev->dev,
1506                                 sizeof(struct rdma_srq_producers),
1507                                 &phy_prod_pair_addr, GFP_KERNEL);
1508         if (!va) {
1509                 DP_ERR(dev,
1510                        "create srq: failed to allocate dma memory for producer\n");
1511                 return -ENOMEM;
1512         }
1513
1514         hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1515         hw_srq->virt_prod_pair_addr = va;
1516
1517         num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1518         params.num_elems = num_elems;
1519
1520         rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, &params);
1521         if (rc)
1522                 goto err0;
1523
1524         hw_srq->num_elems = num_elems;
1525
1526         return 0;
1527
1528 err0:
1529         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1530                           va, phy_prod_pair_addr);
1531         return rc;
1532 }
1533
1534 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1535                     struct ib_udata *udata)
1536 {
1537         struct qed_rdma_destroy_srq_in_params destroy_in_params;
1538         struct qed_rdma_create_srq_in_params in_params = {};
1539         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1540         struct qed_rdma_create_srq_out_params out_params;
1541         struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
1542         struct qedr_create_srq_ureq ureq = {};
1543         u64 pbl_base_addr, phy_prod_pair_addr;
1544         struct qedr_srq_hwq_info *hw_srq;
1545         u32 page_cnt, page_size;
1546         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1547         int rc = 0;
1548
1549         DP_DEBUG(dev, QEDR_MSG_QP,
1550                  "create SRQ called from %s (pd %p)\n",
1551                  (udata) ? "User lib" : "kernel", pd);
1552
1553         if (init_attr->srq_type != IB_SRQT_BASIC &&
1554             init_attr->srq_type != IB_SRQT_XRC)
1555                 return -EOPNOTSUPP;
1556
1557         rc = qedr_check_srq_params(dev, init_attr, udata);
1558         if (rc)
1559                 return -EINVAL;
1560
1561         srq->dev = dev;
1562         srq->is_xrc = (init_attr->srq_type == IB_SRQT_XRC);
1563         hw_srq = &srq->hw_srq;
1564         spin_lock_init(&srq->lock);
1565
1566         hw_srq->max_wr = init_attr->attr.max_wr;
1567         hw_srq->max_sges = init_attr->attr.max_sge;
1568
1569         if (udata) {
1570                 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1571                                                          udata->inlen))) {
1572                         DP_ERR(dev,
1573                                "create srq: problem copying data from user space\n");
1574                         goto err0;
1575                 }
1576
1577                 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0);
1578                 if (rc)
1579                         goto err0;
1580
1581                 page_cnt = srq->usrq.pbl_info.num_pbes;
1582                 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1583                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1584                 page_size = PAGE_SIZE;
1585         } else {
1586                 struct qed_chain *pbl;
1587
1588                 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1589                 if (rc)
1590                         goto err0;
1591
1592                 pbl = &hw_srq->pbl;
1593                 page_cnt = qed_chain_get_page_cnt(pbl);
1594                 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1595                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1596                 page_size = QED_CHAIN_PAGE_SIZE;
1597         }
1598
1599         in_params.pd_id = pd->pd_id;
1600         in_params.pbl_base_addr = pbl_base_addr;
1601         in_params.prod_pair_addr = phy_prod_pair_addr;
1602         in_params.num_pages = page_cnt;
1603         in_params.page_size = page_size;
1604         if (srq->is_xrc) {
1605                 struct qedr_xrcd *xrcd = get_qedr_xrcd(init_attr->ext.xrc.xrcd);
1606                 struct qedr_cq *cq = get_qedr_cq(init_attr->ext.cq);
1607
1608                 in_params.is_xrc = 1;
1609                 in_params.xrcd_id = xrcd->xrcd_id;
1610                 in_params.cq_cid = cq->icid;
1611         }
1612
1613         rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1614         if (rc)
1615                 goto err1;
1616
1617         srq->srq_id = out_params.srq_id;
1618
1619         if (udata) {
1620                 rc = qedr_copy_srq_uresp(dev, srq, udata);
1621                 if (rc)
1622                         goto err2;
1623         }
1624
1625         rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
1626         if (rc)
1627                 goto err2;
1628
1629         DP_DEBUG(dev, QEDR_MSG_SRQ,
1630                  "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1631         return 0;
1632
1633 err2:
1634         destroy_in_params.srq_id = srq->srq_id;
1635
1636         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1637 err1:
1638         if (udata)
1639                 qedr_free_srq_user_params(srq);
1640         else
1641                 qedr_free_srq_kernel_params(srq);
1642 err0:
1643         return -EFAULT;
1644 }
1645
1646 int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1647 {
1648         struct qed_rdma_destroy_srq_in_params in_params = {};
1649         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1650         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1651
1652         xa_erase_irq(&dev->srqs, srq->srq_id);
1653         in_params.srq_id = srq->srq_id;
1654         in_params.is_xrc = srq->is_xrc;
1655         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1656
1657         if (ibsrq->uobject)
1658                 qedr_free_srq_user_params(srq);
1659         else
1660                 qedr_free_srq_kernel_params(srq);
1661
1662         DP_DEBUG(dev, QEDR_MSG_SRQ,
1663                  "destroy srq: destroyed srq with srq_id=0x%0x\n",
1664                  srq->srq_id);
1665         return 0;
1666 }
1667
1668 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1669                     enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1670 {
1671         struct qed_rdma_modify_srq_in_params in_params = {};
1672         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1673         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1674         int rc;
1675
1676         if (attr_mask & IB_SRQ_MAX_WR) {
1677                 DP_ERR(dev,
1678                        "modify srq: invalid attribute mask=0x%x specified for %p\n",
1679                        attr_mask, srq);
1680                 return -EINVAL;
1681         }
1682
1683         if (attr_mask & IB_SRQ_LIMIT) {
1684                 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1685                         DP_ERR(dev,
1686                                "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1687                                attr->srq_limit, srq->hw_srq.max_wr);
1688                         return -EINVAL;
1689                 }
1690
1691                 in_params.srq_id = srq->srq_id;
1692                 in_params.wqe_limit = attr->srq_limit;
1693                 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1694                 if (rc)
1695                         return rc;
1696         }
1697
1698         srq->srq_limit = attr->srq_limit;
1699
1700         DP_DEBUG(dev, QEDR_MSG_SRQ,
1701                  "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1702
1703         return 0;
1704 }
1705
1706 static enum qed_rdma_qp_type qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type)
1707 {
1708         switch (ib_qp_type) {
1709         case IB_QPT_RC:
1710                 return QED_RDMA_QP_TYPE_RC;
1711         case IB_QPT_XRC_INI:
1712                 return QED_RDMA_QP_TYPE_XRC_INI;
1713         case IB_QPT_XRC_TGT:
1714                 return QED_RDMA_QP_TYPE_XRC_TGT;
1715         default:
1716                 return QED_RDMA_QP_TYPE_INVAL;
1717         }
1718 }
1719
1720 static inline void
1721 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1722                               struct qedr_pd *pd,
1723                               struct qedr_qp *qp,
1724                               struct ib_qp_init_attr *attrs,
1725                               bool fmr_and_reserved_lkey,
1726                               struct qed_rdma_create_qp_in_params *params)
1727 {
1728         /* QP handle to be written in an async event */
1729         params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1730         params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1731
1732         params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1733         params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1734         params->qp_type = qedr_ib_to_qed_qp_type(attrs->qp_type);
1735         params->stats_queue = 0;
1736
1737         if (pd) {
1738                 params->pd = pd->pd_id;
1739                 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1740         }
1741
1742         if (qedr_qp_has_sq(qp))
1743                 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1744
1745         if (qedr_qp_has_rq(qp))
1746                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1747
1748         if (qedr_qp_has_srq(qp)) {
1749                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1750                 params->srq_id = qp->srq->srq_id;
1751                 params->use_srq = true;
1752         } else {
1753                 params->srq_id = 0;
1754                 params->use_srq = false;
1755         }
1756 }
1757
1758 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1759 {
1760         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1761                  "qp=%p. "
1762                  "sq_addr=0x%llx, "
1763                  "sq_len=%zd, "
1764                  "rq_addr=0x%llx, "
1765                  "rq_len=%zd"
1766                  "\n",
1767                  qp,
1768                  qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0,
1769                  qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0,
1770                  qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0,
1771                  qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0);
1772 }
1773
1774 static inline void
1775 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1776                             struct qedr_qp *qp,
1777                             struct qed_rdma_create_qp_out_params *out_params)
1778 {
1779         qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1780         qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1781
1782         qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1783                            &qp->usq.pbl_info, FW_PAGE_SHIFT);
1784         if (!qp->srq) {
1785                 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1786                 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1787         }
1788
1789         qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1790                            &qp->urq.pbl_info, FW_PAGE_SHIFT);
1791 }
1792
1793 static void qedr_cleanup_user(struct qedr_dev *dev,
1794                               struct qedr_ucontext *ctx,
1795                               struct qedr_qp *qp)
1796 {
1797         if (qedr_qp_has_sq(qp)) {
1798                 ib_umem_release(qp->usq.umem);
1799                 qp->usq.umem = NULL;
1800         }
1801
1802         if (qedr_qp_has_rq(qp)) {
1803                 ib_umem_release(qp->urq.umem);
1804                 qp->urq.umem = NULL;
1805         }
1806
1807         if (rdma_protocol_roce(&dev->ibdev, 1)) {
1808                 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1809                 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1810         } else {
1811                 kfree(qp->usq.pbl_tbl);
1812                 kfree(qp->urq.pbl_tbl);
1813         }
1814
1815         if (qp->usq.db_rec_data) {
1816                 qedr_db_recovery_del(dev, qp->usq.db_addr,
1817                                      &qp->usq.db_rec_data->db_data);
1818                 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1819         }
1820
1821         if (qp->urq.db_rec_data) {
1822                 qedr_db_recovery_del(dev, qp->urq.db_addr,
1823                                      &qp->urq.db_rec_data->db_data);
1824                 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1825         }
1826
1827         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1828                 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1829                                      &qp->urq.db_rec_db2_data);
1830 }
1831
1832 static int qedr_create_user_qp(struct qedr_dev *dev,
1833                                struct qedr_qp *qp,
1834                                struct ib_pd *ibpd,
1835                                struct ib_udata *udata,
1836                                struct ib_qp_init_attr *attrs)
1837 {
1838         struct qed_rdma_create_qp_in_params in_params;
1839         struct qed_rdma_create_qp_out_params out_params;
1840         struct qedr_create_qp_uresp uresp = {};
1841         struct qedr_create_qp_ureq ureq = {};
1842         int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1843         struct qedr_ucontext *ctx = NULL;
1844         struct qedr_pd *pd = NULL;
1845         int rc = 0;
1846
1847         qp->create_type = QEDR_QP_CREATE_USER;
1848
1849         if (ibpd) {
1850                 pd = get_qedr_pd(ibpd);
1851                 ctx = pd->uctx;
1852         }
1853
1854         if (udata) {
1855                 rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1856                                         udata->inlen));
1857                 if (rc) {
1858                         DP_ERR(dev, "Problem copying data from user space\n");
1859                         return rc;
1860                 }
1861         }
1862
1863         if (qedr_qp_has_sq(qp)) {
1864                 /* SQ - read access only (0) */
1865                 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1866                                           ureq.sq_len, true, 0, alloc_and_init);
1867                 if (rc)
1868                         return rc;
1869         }
1870
1871         if (qedr_qp_has_rq(qp)) {
1872                 /* RQ - read access only (0) */
1873                 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1874                                           ureq.rq_len, true, 0, alloc_and_init);
1875                 if (rc)
1876                         return rc;
1877         }
1878
1879         memset(&in_params, 0, sizeof(in_params));
1880         qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1881         in_params.qp_handle_lo = ureq.qp_handle_lo;
1882         in_params.qp_handle_hi = ureq.qp_handle_hi;
1883
1884         if (qp->qp_type == IB_QPT_XRC_TGT) {
1885                 struct qedr_xrcd *xrcd = get_qedr_xrcd(attrs->xrcd);
1886
1887                 in_params.xrcd_id = xrcd->xrcd_id;
1888                 in_params.qp_handle_lo = qp->qp_id;
1889                 in_params.use_srq = 1;
1890         }
1891
1892         if (qedr_qp_has_sq(qp)) {
1893                 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1894                 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1895         }
1896
1897         if (qedr_qp_has_rq(qp)) {
1898                 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1899                 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1900         }
1901
1902         if (ctx)
1903                 SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
1904
1905         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1906                                               &in_params, &out_params);
1907
1908         if (!qp->qed_qp) {
1909                 rc = -ENOMEM;
1910                 goto err1;
1911         }
1912
1913         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1914                 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1915
1916         qp->qp_id = out_params.qp_id;
1917         qp->icid = out_params.icid;
1918
1919         if (udata) {
1920                 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1921                 if (rc)
1922                         goto err;
1923         }
1924
1925         /* db offset was calculated in copy_qp_uresp, now set in the user q */
1926         if (qedr_qp_has_sq(qp)) {
1927                 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1928                 rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1929                                           &qp->usq.db_rec_data->db_data,
1930                                           DB_REC_WIDTH_32B,
1931                                           DB_REC_USER);
1932                 if (rc)
1933                         goto err;
1934         }
1935
1936         if (qedr_qp_has_rq(qp)) {
1937                 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1938                 rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1939                                           &qp->urq.db_rec_data->db_data,
1940                                           DB_REC_WIDTH_32B,
1941                                           DB_REC_USER);
1942                 if (rc)
1943                         goto err;
1944         }
1945
1946         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1947                 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1948
1949                 /* calculate the db_rec_db2 data since it is constant so no
1950                  * need to reflect from user
1951                  */
1952                 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1953                 qp->urq.db_rec_db2_data.data.value =
1954                         cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1955
1956                 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1957                                           &qp->urq.db_rec_db2_data,
1958                                           DB_REC_WIDTH_32B,
1959                                           DB_REC_USER);
1960                 if (rc)
1961                         goto err;
1962         }
1963         qedr_qp_user_print(dev, qp);
1964         return rc;
1965 err:
1966         rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1967         if (rc)
1968                 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1969
1970 err1:
1971         qedr_cleanup_user(dev, ctx, qp);
1972         return rc;
1973 }
1974
1975 static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1976 {
1977         int rc;
1978
1979         qp->sq.db = dev->db_addr +
1980             DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1981         qp->sq.db_data.data.icid = qp->icid;
1982
1983         rc = qedr_db_recovery_add(dev, qp->sq.db,
1984                                   &qp->sq.db_data,
1985                                   DB_REC_WIDTH_32B,
1986                                   DB_REC_KERNEL);
1987         if (rc)
1988                 return rc;
1989
1990         qp->rq.db = dev->db_addr +
1991                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1992         qp->rq.db_data.data.icid = qp->icid;
1993         qp->rq.iwarp_db2 = dev->db_addr +
1994                            DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1995         qp->rq.iwarp_db2_data.data.icid = qp->icid;
1996         qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1997
1998         rc = qedr_db_recovery_add(dev, qp->rq.db,
1999                                   &qp->rq.db_data,
2000                                   DB_REC_WIDTH_32B,
2001                                   DB_REC_KERNEL);
2002         if (rc)
2003                 return rc;
2004
2005         rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
2006                                   &qp->rq.iwarp_db2_data,
2007                                   DB_REC_WIDTH_32B,
2008                                   DB_REC_KERNEL);
2009         return rc;
2010 }
2011
2012 static int
2013 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
2014                            struct qedr_qp *qp,
2015                            struct qed_rdma_create_qp_in_params *in_params,
2016                            u32 n_sq_elems, u32 n_rq_elems)
2017 {
2018         struct qed_rdma_create_qp_out_params out_params;
2019         struct qed_chain_init_params params = {
2020                 .mode           = QED_CHAIN_MODE_PBL,
2021                 .cnt_type       = QED_CHAIN_CNT_TYPE_U32,
2022         };
2023         int rc;
2024
2025         params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2026         params.num_elems = n_sq_elems;
2027         params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2028
2029         rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2030         if (rc)
2031                 return rc;
2032
2033         in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
2034         in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
2035
2036         params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2037         params.num_elems = n_rq_elems;
2038         params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2039
2040         rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2041         if (rc)
2042                 return rc;
2043
2044         in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
2045         in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
2046
2047         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2048                                               in_params, &out_params);
2049
2050         if (!qp->qed_qp)
2051                 return -EINVAL;
2052
2053         qp->qp_id = out_params.qp_id;
2054         qp->icid = out_params.icid;
2055
2056         return qedr_set_roce_db_info(dev, qp);
2057 }
2058
2059 static int
2060 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
2061                             struct qedr_qp *qp,
2062                             struct qed_rdma_create_qp_in_params *in_params,
2063                             u32 n_sq_elems, u32 n_rq_elems)
2064 {
2065         struct qed_rdma_create_qp_out_params out_params;
2066         struct qed_chain_init_params params = {
2067                 .mode           = QED_CHAIN_MODE_PBL,
2068                 .cnt_type       = QED_CHAIN_CNT_TYPE_U32,
2069         };
2070         int rc;
2071
2072         in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
2073                                                      QEDR_SQE_ELEMENT_SIZE,
2074                                                      QED_CHAIN_PAGE_SIZE,
2075                                                      QED_CHAIN_MODE_PBL);
2076         in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
2077                                                      QEDR_RQE_ELEMENT_SIZE,
2078                                                      QED_CHAIN_PAGE_SIZE,
2079                                                      QED_CHAIN_MODE_PBL);
2080
2081         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2082                                               in_params, &out_params);
2083
2084         if (!qp->qed_qp)
2085                 return -EINVAL;
2086
2087         /* Now we allocate the chain */
2088
2089         params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2090         params.num_elems = n_sq_elems;
2091         params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2092         params.ext_pbl_virt = out_params.sq_pbl_virt;
2093         params.ext_pbl_phys = out_params.sq_pbl_phys;
2094
2095         rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2096         if (rc)
2097                 goto err;
2098
2099         params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2100         params.num_elems = n_rq_elems;
2101         params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2102         params.ext_pbl_virt = out_params.rq_pbl_virt;
2103         params.ext_pbl_phys = out_params.rq_pbl_phys;
2104
2105         rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2106         if (rc)
2107                 goto err;
2108
2109         qp->qp_id = out_params.qp_id;
2110         qp->icid = out_params.icid;
2111
2112         return qedr_set_iwarp_db_info(dev, qp);
2113
2114 err:
2115         dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2116
2117         return rc;
2118 }
2119
2120 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
2121 {
2122         dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2123         kfree(qp->wqe_wr_id);
2124
2125         dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2126         kfree(qp->rqe_wr_id);
2127
2128         /* GSI qp is not registered to db mechanism so no need to delete */
2129         if (qp->qp_type == IB_QPT_GSI)
2130                 return;
2131
2132         qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2133
2134         if (!qp->srq) {
2135                 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
2136
2137                 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2138                         qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2139                                              &qp->rq.iwarp_db2_data);
2140         }
2141 }
2142
2143 static int qedr_create_kernel_qp(struct qedr_dev *dev,
2144                                  struct qedr_qp *qp,
2145                                  struct ib_pd *ibpd,
2146                                  struct ib_qp_init_attr *attrs)
2147 {
2148         struct qed_rdma_create_qp_in_params in_params;
2149         struct qedr_pd *pd = get_qedr_pd(ibpd);
2150         int rc = -EINVAL;
2151         u32 n_rq_elems;
2152         u32 n_sq_elems;
2153         u32 n_sq_entries;
2154
2155         memset(&in_params, 0, sizeof(in_params));
2156         qp->create_type = QEDR_QP_CREATE_KERNEL;
2157
2158         /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2159          * the ring. The ring should allow at least a single WR, even if the
2160          * user requested none, due to allocation issues.
2161          * We should add an extra WR since the prod and cons indices of
2162          * wqe_wr_id are managed in such a way that the WQ is considered full
2163          * when (prod+1)%max_wr==cons. We currently don't do that because we
2164          * double the number of entries due an iSER issue that pushes far more
2165          * WRs than indicated. If we decline its ib_post_send() then we get
2166          * error prints in the dmesg we'd like to avoid.
2167          */
2168         qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2169                               dev->attr.max_sqe);
2170
2171         qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
2172                                 GFP_KERNEL);
2173         if (!qp->wqe_wr_id) {
2174                 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2175                 return -ENOMEM;
2176         }
2177
2178         /* QP handle to be written in CQE */
2179         in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2180         in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
2181
2182         /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2183          * the ring. There ring should allow at least a single WR, even if the
2184          * user requested none, due to allocation issues.
2185          */
2186         qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
2187
2188         /* Allocate driver internal RQ array */
2189         qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
2190                                 GFP_KERNEL);
2191         if (!qp->rqe_wr_id) {
2192                 DP_ERR(dev,
2193                        "create qp: failed RQ shadow memory allocation\n");
2194                 kfree(qp->wqe_wr_id);
2195                 return -ENOMEM;
2196         }
2197
2198         qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
2199
2200         n_sq_entries = attrs->cap.max_send_wr;
2201         n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2202         n_sq_entries = max_t(u32, n_sq_entries, 1);
2203         n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2204
2205         n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2206
2207         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2208                 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2209                                                  n_sq_elems, n_rq_elems);
2210         else
2211                 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2212                                                 n_sq_elems, n_rq_elems);
2213         if (rc)
2214                 qedr_cleanup_kernel(dev, qp);
2215
2216         return rc;
2217 }
2218
2219 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2220                                   struct ib_udata *udata)
2221 {
2222         struct qedr_ucontext *ctx =
2223                 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2224                                           ibucontext);
2225         int rc;
2226
2227         if (qp->qp_type != IB_QPT_GSI) {
2228                 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2229                 if (rc)
2230                         return rc;
2231         }
2232
2233         if (qp->create_type == QEDR_QP_CREATE_USER)
2234                 qedr_cleanup_user(dev, ctx, qp);
2235         else
2236                 qedr_cleanup_kernel(dev, qp);
2237
2238         return 0;
2239 }
2240
2241 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
2242                              struct ib_qp_init_attr *attrs,
2243                              struct ib_udata *udata)
2244 {
2245         struct qedr_xrcd *xrcd = NULL;
2246         struct qedr_pd *pd = NULL;
2247         struct qedr_dev *dev;
2248         struct qedr_qp *qp;
2249         struct ib_qp *ibqp;
2250         int rc = 0;
2251
2252         if (attrs->create_flags)
2253                 return ERR_PTR(-EOPNOTSUPP);
2254
2255         if (attrs->qp_type == IB_QPT_XRC_TGT) {
2256                 xrcd = get_qedr_xrcd(attrs->xrcd);
2257                 dev = get_qedr_dev(xrcd->ibxrcd.device);
2258         } else {
2259                 pd = get_qedr_pd(ibpd);
2260                 dev = get_qedr_dev(ibpd->device);
2261         }
2262
2263         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2264                  udata ? "user library" : "kernel", pd);
2265
2266         rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
2267         if (rc)
2268                 return ERR_PTR(rc);
2269
2270         DP_DEBUG(dev, QEDR_MSG_QP,
2271                  "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2272                  udata ? "user library" : "kernel", attrs->event_handler, pd,
2273                  get_qedr_cq(attrs->send_cq),
2274                  get_qedr_cq(attrs->send_cq)->icid,
2275                  get_qedr_cq(attrs->recv_cq),
2276                  attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
2277
2278         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2279         if (!qp) {
2280                 DP_ERR(dev, "create qp: failed allocating memory\n");
2281                 return ERR_PTR(-ENOMEM);
2282         }
2283
2284         qedr_set_common_qp_params(dev, qp, pd, attrs);
2285
2286         if (attrs->qp_type == IB_QPT_GSI) {
2287                 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
2288                 if (IS_ERR(ibqp))
2289                         kfree(qp);
2290                 return ibqp;
2291         }
2292
2293         if (udata || xrcd)
2294                 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2295         else
2296                 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2297
2298         if (rc)
2299                 goto out_free_qp;
2300
2301         qp->ibqp.qp_num = qp->qp_id;
2302
2303         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2304                 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
2305                 if (rc)
2306                         goto out_free_qp_resources;
2307         }
2308
2309         return &qp->ibqp;
2310
2311 out_free_qp_resources:
2312         qedr_free_qp_resources(dev, qp, udata);
2313 out_free_qp:
2314         kfree(qp);
2315
2316         return ERR_PTR(-EFAULT);
2317 }
2318
2319 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
2320 {
2321         switch (qp_state) {
2322         case QED_ROCE_QP_STATE_RESET:
2323                 return IB_QPS_RESET;
2324         case QED_ROCE_QP_STATE_INIT:
2325                 return IB_QPS_INIT;
2326         case QED_ROCE_QP_STATE_RTR:
2327                 return IB_QPS_RTR;
2328         case QED_ROCE_QP_STATE_RTS:
2329                 return IB_QPS_RTS;
2330         case QED_ROCE_QP_STATE_SQD:
2331                 return IB_QPS_SQD;
2332         case QED_ROCE_QP_STATE_ERR:
2333                 return IB_QPS_ERR;
2334         case QED_ROCE_QP_STATE_SQE:
2335                 return IB_QPS_SQE;
2336         }
2337         return IB_QPS_ERR;
2338 }
2339
2340 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2341                                         enum ib_qp_state qp_state)
2342 {
2343         switch (qp_state) {
2344         case IB_QPS_RESET:
2345                 return QED_ROCE_QP_STATE_RESET;
2346         case IB_QPS_INIT:
2347                 return QED_ROCE_QP_STATE_INIT;
2348         case IB_QPS_RTR:
2349                 return QED_ROCE_QP_STATE_RTR;
2350         case IB_QPS_RTS:
2351                 return QED_ROCE_QP_STATE_RTS;
2352         case IB_QPS_SQD:
2353                 return QED_ROCE_QP_STATE_SQD;
2354         case IB_QPS_ERR:
2355                 return QED_ROCE_QP_STATE_ERR;
2356         default:
2357                 return QED_ROCE_QP_STATE_ERR;
2358         }
2359 }
2360
2361 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
2362 {
2363         qed_chain_reset(&qph->pbl);
2364         qph->prod = 0;
2365         qph->cons = 0;
2366         qph->wqe_cons = 0;
2367         qph->db_data.data.value = cpu_to_le16(0);
2368 }
2369
2370 static int qedr_update_qp_state(struct qedr_dev *dev,
2371                                 struct qedr_qp *qp,
2372                                 enum qed_roce_qp_state cur_state,
2373                                 enum qed_roce_qp_state new_state)
2374 {
2375         int status = 0;
2376
2377         if (new_state == cur_state)
2378                 return 0;
2379
2380         switch (cur_state) {
2381         case QED_ROCE_QP_STATE_RESET:
2382                 switch (new_state) {
2383                 case QED_ROCE_QP_STATE_INIT:
2384                         qp->prev_wqe_size = 0;
2385                         qedr_reset_qp_hwq_info(&qp->sq);
2386                         qedr_reset_qp_hwq_info(&qp->rq);
2387                         break;
2388                 default:
2389                         status = -EINVAL;
2390                         break;
2391                 }
2392                 break;
2393         case QED_ROCE_QP_STATE_INIT:
2394                 switch (new_state) {
2395                 case QED_ROCE_QP_STATE_RTR:
2396                         /* Update doorbell (in case post_recv was
2397                          * done before move to RTR)
2398                          */
2399
2400                         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2401                                 writel(qp->rq.db_data.raw, qp->rq.db);
2402                         }
2403                         break;
2404                 case QED_ROCE_QP_STATE_ERR:
2405                         break;
2406                 default:
2407                         /* Invalid state change. */
2408                         status = -EINVAL;
2409                         break;
2410                 }
2411                 break;
2412         case QED_ROCE_QP_STATE_RTR:
2413                 /* RTR->XXX */
2414                 switch (new_state) {
2415                 case QED_ROCE_QP_STATE_RTS:
2416                         break;
2417                 case QED_ROCE_QP_STATE_ERR:
2418                         break;
2419                 default:
2420                         /* Invalid state change. */
2421                         status = -EINVAL;
2422                         break;
2423                 }
2424                 break;
2425         case QED_ROCE_QP_STATE_RTS:
2426                 /* RTS->XXX */
2427                 switch (new_state) {
2428                 case QED_ROCE_QP_STATE_SQD:
2429                         break;
2430                 case QED_ROCE_QP_STATE_ERR:
2431                         break;
2432                 default:
2433                         /* Invalid state change. */
2434                         status = -EINVAL;
2435                         break;
2436                 }
2437                 break;
2438         case QED_ROCE_QP_STATE_SQD:
2439                 /* SQD->XXX */
2440                 switch (new_state) {
2441                 case QED_ROCE_QP_STATE_RTS:
2442                 case QED_ROCE_QP_STATE_ERR:
2443                         break;
2444                 default:
2445                         /* Invalid state change. */
2446                         status = -EINVAL;
2447                         break;
2448                 }
2449                 break;
2450         case QED_ROCE_QP_STATE_ERR:
2451                 /* ERR->XXX */
2452                 switch (new_state) {
2453                 case QED_ROCE_QP_STATE_RESET:
2454                         if ((qp->rq.prod != qp->rq.cons) ||
2455                             (qp->sq.prod != qp->sq.cons)) {
2456                                 DP_NOTICE(dev,
2457                                           "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2458                                           qp->rq.prod, qp->rq.cons, qp->sq.prod,
2459                                           qp->sq.cons);
2460                                 status = -EINVAL;
2461                         }
2462                         break;
2463                 default:
2464                         status = -EINVAL;
2465                         break;
2466                 }
2467                 break;
2468         default:
2469                 status = -EINVAL;
2470                 break;
2471         }
2472
2473         return status;
2474 }
2475
2476 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2477                    int attr_mask, struct ib_udata *udata)
2478 {
2479         struct qedr_qp *qp = get_qedr_qp(ibqp);
2480         struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2481         struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2482         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2483         enum ib_qp_state old_qp_state, new_qp_state;
2484         enum qed_roce_qp_state cur_state;
2485         int rc = 0;
2486
2487         DP_DEBUG(dev, QEDR_MSG_QP,
2488                  "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2489                  attr->qp_state);
2490
2491         if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2492                 return -EOPNOTSUPP;
2493
2494         old_qp_state = qedr_get_ibqp_state(qp->state);
2495         if (attr_mask & IB_QP_STATE)
2496                 new_qp_state = attr->qp_state;
2497         else
2498                 new_qp_state = old_qp_state;
2499
2500         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2501                 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2502                                         ibqp->qp_type, attr_mask)) {
2503                         DP_ERR(dev,
2504                                "modify qp: invalid attribute mask=0x%x specified for\n"
2505                                "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2506                                attr_mask, qp->qp_id, ibqp->qp_type,
2507                                old_qp_state, new_qp_state);
2508                         rc = -EINVAL;
2509                         goto err;
2510                 }
2511         }
2512
2513         /* Translate the masks... */
2514         if (attr_mask & IB_QP_STATE) {
2515                 SET_FIELD(qp_params.modify_flags,
2516                           QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2517                 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2518         }
2519
2520         if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2521                 qp_params.sqd_async = true;
2522
2523         if (attr_mask & IB_QP_PKEY_INDEX) {
2524                 SET_FIELD(qp_params.modify_flags,
2525                           QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2526                 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2527                         rc = -EINVAL;
2528                         goto err;
2529                 }
2530
2531                 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2532         }
2533
2534         if (attr_mask & IB_QP_QKEY)
2535                 qp->qkey = attr->qkey;
2536
2537         if (attr_mask & IB_QP_ACCESS_FLAGS) {
2538                 SET_FIELD(qp_params.modify_flags,
2539                           QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2540                 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2541                                                   IB_ACCESS_REMOTE_READ;
2542                 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2543                                                    IB_ACCESS_REMOTE_WRITE;
2544                 qp_params.incoming_atomic_en = attr->qp_access_flags &
2545                                                IB_ACCESS_REMOTE_ATOMIC;
2546         }
2547
2548         if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2549                 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2550                         return -EINVAL;
2551
2552                 if (attr_mask & IB_QP_PATH_MTU) {
2553                         if (attr->path_mtu < IB_MTU_256 ||
2554                             attr->path_mtu > IB_MTU_4096) {
2555                                 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2556                                 rc = -EINVAL;
2557                                 goto err;
2558                         }
2559                         qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2560                                       ib_mtu_enum_to_int(iboe_get_mtu
2561                                                          (dev->ndev->mtu)));
2562                 }
2563
2564                 if (!qp->mtu) {
2565                         qp->mtu =
2566                         ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2567                         pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2568                 }
2569
2570                 SET_FIELD(qp_params.modify_flags,
2571                           QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2572
2573                 qp_params.traffic_class_tos = grh->traffic_class;
2574                 qp_params.flow_label = grh->flow_label;
2575                 qp_params.hop_limit_ttl = grh->hop_limit;
2576
2577                 qp->sgid_idx = grh->sgid_index;
2578
2579                 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2580                 if (rc) {
2581                         DP_ERR(dev,
2582                                "modify qp: problems with GID index %d (rc=%d)\n",
2583                                grh->sgid_index, rc);
2584                         return rc;
2585                 }
2586
2587                 rc = qedr_get_dmac(dev, &attr->ah_attr,
2588                                    qp_params.remote_mac_addr);
2589                 if (rc)
2590                         return rc;
2591
2592                 qp_params.use_local_mac = true;
2593                 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2594
2595                 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2596                          qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2597                          qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2598                 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2599                          qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2600                          qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2601                 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2602                          qp_params.remote_mac_addr);
2603
2604                 qp_params.mtu = qp->mtu;
2605                 qp_params.lb_indication = false;
2606         }
2607
2608         if (!qp_params.mtu) {
2609                 /* Stay with current MTU */
2610                 if (qp->mtu)
2611                         qp_params.mtu = qp->mtu;
2612                 else
2613                         qp_params.mtu =
2614                             ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2615         }
2616
2617         if (attr_mask & IB_QP_TIMEOUT) {
2618                 SET_FIELD(qp_params.modify_flags,
2619                           QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2620
2621                 /* The received timeout value is an exponent used like this:
2622                  *    "12.7.34 LOCAL ACK TIMEOUT
2623                  *    Value representing the transport (ACK) timeout for use by
2624                  *    the remote, expressed as: 4.096 * 2^timeout [usec]"
2625                  * The FW expects timeout in msec so we need to divide the usec
2626                  * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2627                  * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2628                  * The value of zero means infinite so we use a 'max_t' to make
2629                  * sure that sub 1 msec values will be configured as 1 msec.
2630                  */
2631                 if (attr->timeout)
2632                         qp_params.ack_timeout =
2633                                         1 << max_t(int, attr->timeout - 8, 0);
2634                 else
2635                         qp_params.ack_timeout = 0;
2636         }
2637
2638         if (attr_mask & IB_QP_RETRY_CNT) {
2639                 SET_FIELD(qp_params.modify_flags,
2640                           QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2641                 qp_params.retry_cnt = attr->retry_cnt;
2642         }
2643
2644         if (attr_mask & IB_QP_RNR_RETRY) {
2645                 SET_FIELD(qp_params.modify_flags,
2646                           QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2647                 qp_params.rnr_retry_cnt = attr->rnr_retry;
2648         }
2649
2650         if (attr_mask & IB_QP_RQ_PSN) {
2651                 SET_FIELD(qp_params.modify_flags,
2652                           QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2653                 qp_params.rq_psn = attr->rq_psn;
2654                 qp->rq_psn = attr->rq_psn;
2655         }
2656
2657         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2658                 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2659                         rc = -EINVAL;
2660                         DP_ERR(dev,
2661                                "unsupported max_rd_atomic=%d, supported=%d\n",
2662                                attr->max_rd_atomic,
2663                                dev->attr.max_qp_req_rd_atomic_resc);
2664                         goto err;
2665                 }
2666
2667                 SET_FIELD(qp_params.modify_flags,
2668                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2669                 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2670         }
2671
2672         if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2673                 SET_FIELD(qp_params.modify_flags,
2674                           QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2675                 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2676         }
2677
2678         if (attr_mask & IB_QP_SQ_PSN) {
2679                 SET_FIELD(qp_params.modify_flags,
2680                           QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2681                 qp_params.sq_psn = attr->sq_psn;
2682                 qp->sq_psn = attr->sq_psn;
2683         }
2684
2685         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2686                 if (attr->max_dest_rd_atomic >
2687                     dev->attr.max_qp_resp_rd_atomic_resc) {
2688                         DP_ERR(dev,
2689                                "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2690                                attr->max_dest_rd_atomic,
2691                                dev->attr.max_qp_resp_rd_atomic_resc);
2692
2693                         rc = -EINVAL;
2694                         goto err;
2695                 }
2696
2697                 SET_FIELD(qp_params.modify_flags,
2698                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2699                 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2700         }
2701
2702         if (attr_mask & IB_QP_DEST_QPN) {
2703                 SET_FIELD(qp_params.modify_flags,
2704                           QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2705
2706                 qp_params.dest_qp = attr->dest_qp_num;
2707                 qp->dest_qp_num = attr->dest_qp_num;
2708         }
2709
2710         cur_state = qp->state;
2711
2712         /* Update the QP state before the actual ramrod to prevent a race with
2713          * fast path. Modifying the QP state to error will cause the device to
2714          * flush the CQEs and while polling the flushed CQEs will considered as
2715          * a potential issue if the QP isn't in error state.
2716          */
2717         if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2718             !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2719                 qp->state = QED_ROCE_QP_STATE_ERR;
2720
2721         if (qp->qp_type != IB_QPT_GSI)
2722                 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2723                                               qp->qed_qp, &qp_params);
2724
2725         if (attr_mask & IB_QP_STATE) {
2726                 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2727                         rc = qedr_update_qp_state(dev, qp, cur_state,
2728                                                   qp_params.new_state);
2729                 qp->state = qp_params.new_state;
2730         }
2731
2732 err:
2733         return rc;
2734 }
2735
2736 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2737 {
2738         int ib_qp_acc_flags = 0;
2739
2740         if (params->incoming_rdma_write_en)
2741                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2742         if (params->incoming_rdma_read_en)
2743                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2744         if (params->incoming_atomic_en)
2745                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2746         ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2747         return ib_qp_acc_flags;
2748 }
2749
2750 int qedr_query_qp(struct ib_qp *ibqp,
2751                   struct ib_qp_attr *qp_attr,
2752                   int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2753 {
2754         struct qed_rdma_query_qp_out_params params;
2755         struct qedr_qp *qp = get_qedr_qp(ibqp);
2756         struct qedr_dev *dev = qp->dev;
2757         int rc = 0;
2758
2759         memset(&params, 0, sizeof(params));
2760
2761         rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2762         if (rc)
2763                 goto err;
2764
2765         memset(qp_attr, 0, sizeof(*qp_attr));
2766         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2767
2768         qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2769         qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2770         qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2771         qp_attr->path_mig_state = IB_MIG_MIGRATED;
2772         qp_attr->rq_psn = params.rq_psn;
2773         qp_attr->sq_psn = params.sq_psn;
2774         qp_attr->dest_qp_num = params.dest_qp;
2775
2776         qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2777
2778         qp_attr->cap.max_send_wr = qp->sq.max_wr;
2779         qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2780         qp_attr->cap.max_send_sge = qp->sq.max_sges;
2781         qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2782         qp_attr->cap.max_inline_data = dev->attr.max_inline;
2783         qp_init_attr->cap = qp_attr->cap;
2784
2785         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2786         rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2787                         params.flow_label, qp->sgid_idx,
2788                         params.hop_limit_ttl, params.traffic_class_tos);
2789         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2790         rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2791         rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2792         qp_attr->timeout = params.timeout;
2793         qp_attr->rnr_retry = params.rnr_retry;
2794         qp_attr->retry_cnt = params.retry_cnt;
2795         qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2796         qp_attr->pkey_index = params.pkey_index;
2797         qp_attr->port_num = 1;
2798         rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2799         rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2800         qp_attr->alt_pkey_index = 0;
2801         qp_attr->alt_port_num = 0;
2802         qp_attr->alt_timeout = 0;
2803         memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2804
2805         qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2806         qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2807         qp_attr->max_rd_atomic = params.max_rd_atomic;
2808         qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2809
2810         DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2811                  qp_attr->cap.max_inline_data);
2812
2813 err:
2814         return rc;
2815 }
2816
2817 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2818 {
2819         struct qedr_qp *qp = get_qedr_qp(ibqp);
2820         struct qedr_dev *dev = qp->dev;
2821         struct ib_qp_attr attr;
2822         int attr_mask = 0;
2823
2824         DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2825                  qp, qp->qp_type);
2826
2827         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2828                 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2829                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
2830                     (qp->state != QED_ROCE_QP_STATE_INIT)) {
2831
2832                         attr.qp_state = IB_QPS_ERR;
2833                         attr_mask |= IB_QP_STATE;
2834
2835                         /* Change the QP state to ERROR */
2836                         qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2837                 }
2838         } else {
2839                 /* If connection establishment started the WAIT_FOR_CONNECT
2840                  * bit will be on and we need to Wait for the establishment
2841                  * to complete before destroying the qp.
2842                  */
2843                 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2844                                      &qp->iwarp_cm_flags))
2845                         wait_for_completion(&qp->iwarp_cm_comp);
2846
2847                 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2848                  * bit will be on, and we need to wait for the disconnect to
2849                  * complete before continuing. We can use the same completion,
2850                  * iwarp_cm_comp, since this is the only place that waits for
2851                  * this completion and it is sequential. In addition,
2852                  * disconnect can't occur before the connection is fully
2853                  * established, therefore if WAIT_FOR_DISCONNECT is on it
2854                  * means WAIT_FOR_CONNECT is also on and the completion for
2855                  * CONNECT already occurred.
2856                  */
2857                 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2858                                      &qp->iwarp_cm_flags))
2859                         wait_for_completion(&qp->iwarp_cm_comp);
2860         }
2861
2862         if (qp->qp_type == IB_QPT_GSI)
2863                 qedr_destroy_gsi_qp(dev);
2864
2865         /* We need to remove the entry from the xarray before we release the
2866          * qp_id to avoid a race of the qp_id being reallocated and failing
2867          * on xa_insert
2868          */
2869         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2870                 xa_erase(&dev->qps, qp->qp_id);
2871
2872         qedr_free_qp_resources(dev, qp, udata);
2873
2874         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2875                 qedr_iw_qp_rem_ref(&qp->ibqp);
2876         else
2877                 kfree(qp);
2878
2879         return 0;
2880 }
2881
2882 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
2883                    struct ib_udata *udata)
2884 {
2885         struct qedr_ah *ah = get_qedr_ah(ibah);
2886
2887         rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
2888
2889         return 0;
2890 }
2891
2892 int qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2893 {
2894         struct qedr_ah *ah = get_qedr_ah(ibah);
2895
2896         rdma_destroy_ah_attr(&ah->attr);
2897         return 0;
2898 }
2899
2900 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2901 {
2902         struct qedr_pbl *pbl, *tmp;
2903
2904         if (info->pbl_table)
2905                 list_add_tail(&info->pbl_table->list_entry,
2906                               &info->free_pbl_list);
2907
2908         if (!list_empty(&info->inuse_pbl_list))
2909                 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2910
2911         list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2912                 list_del(&pbl->list_entry);
2913                 qedr_free_pbl(dev, &info->pbl_info, pbl);
2914         }
2915 }
2916
2917 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2918                         size_t page_list_len, bool two_layered)
2919 {
2920         struct qedr_pbl *tmp;
2921         int rc;
2922
2923         INIT_LIST_HEAD(&info->free_pbl_list);
2924         INIT_LIST_HEAD(&info->inuse_pbl_list);
2925
2926         rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2927                                   page_list_len, two_layered);
2928         if (rc)
2929                 goto done;
2930
2931         info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2932         if (IS_ERR(info->pbl_table)) {
2933                 rc = PTR_ERR(info->pbl_table);
2934                 goto done;
2935         }
2936
2937         DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2938                  &info->pbl_table->pa);
2939
2940         /* in usual case we use 2 PBLs, so we add one to free
2941          * list and allocating another one
2942          */
2943         tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2944         if (IS_ERR(tmp)) {
2945                 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2946                 goto done;
2947         }
2948
2949         list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2950
2951         DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2952
2953 done:
2954         if (rc)
2955                 free_mr_info(dev, info);
2956
2957         return rc;
2958 }
2959
2960 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2961                                u64 usr_addr, int acc, struct ib_udata *udata)
2962 {
2963         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2964         struct qedr_mr *mr;
2965         struct qedr_pd *pd;
2966         int rc = -ENOMEM;
2967
2968         pd = get_qedr_pd(ibpd);
2969         DP_DEBUG(dev, QEDR_MSG_MR,
2970                  "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2971                  pd->pd_id, start, len, usr_addr, acc);
2972
2973         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2974                 return ERR_PTR(-EINVAL);
2975
2976         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2977         if (!mr)
2978                 return ERR_PTR(rc);
2979
2980         mr->type = QEDR_MR_USER;
2981
2982         mr->umem = ib_umem_get(ibpd->device, start, len, acc);
2983         if (IS_ERR(mr->umem)) {
2984                 rc = -EFAULT;
2985                 goto err0;
2986         }
2987
2988         rc = init_mr_info(dev, &mr->info,
2989                           ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE), 1);
2990         if (rc)
2991                 goto err1;
2992
2993         qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2994                            &mr->info.pbl_info, PAGE_SHIFT);
2995
2996         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2997         if (rc) {
2998                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2999                 goto err1;
3000         }
3001
3002         /* Index only, 18 bit long, lkey = itid << 8 | key */
3003         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3004         mr->hw_mr.key = 0;
3005         mr->hw_mr.pd = pd->pd_id;
3006         mr->hw_mr.local_read = 1;
3007         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3008         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3009         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3010         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3011         mr->hw_mr.mw_bind = false;
3012         mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
3013         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3014         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3015         mr->hw_mr.page_size_log = PAGE_SHIFT;
3016         mr->hw_mr.length = len;
3017         mr->hw_mr.vaddr = usr_addr;
3018         mr->hw_mr.phy_mr = false;
3019         mr->hw_mr.dma_mr = false;
3020
3021         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3022         if (rc) {
3023                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3024                 goto err2;
3025         }
3026
3027         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3028         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3029             mr->hw_mr.remote_atomic)
3030                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3031
3032         DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
3033                  mr->ibmr.lkey);
3034         return &mr->ibmr;
3035
3036 err2:
3037         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3038 err1:
3039         qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3040 err0:
3041         kfree(mr);
3042         return ERR_PTR(rc);
3043 }
3044
3045 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
3046 {
3047         struct qedr_mr *mr = get_qedr_mr(ib_mr);
3048         struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
3049         int rc = 0;
3050
3051         rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
3052         if (rc)
3053                 return rc;
3054
3055         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3056
3057         if (mr->type != QEDR_MR_DMA)
3058                 free_mr_info(dev, &mr->info);
3059
3060         /* it could be user registered memory. */
3061         ib_umem_release(mr->umem);
3062
3063         kfree(mr);
3064
3065         return rc;
3066 }
3067
3068 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
3069                                        int max_page_list_len)
3070 {
3071         struct qedr_pd *pd = get_qedr_pd(ibpd);
3072         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3073         struct qedr_mr *mr;
3074         int rc = -ENOMEM;
3075
3076         DP_DEBUG(dev, QEDR_MSG_MR,
3077                  "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
3078                  max_page_list_len);
3079
3080         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3081         if (!mr)
3082                 return ERR_PTR(rc);
3083
3084         mr->dev = dev;
3085         mr->type = QEDR_MR_FRMR;
3086
3087         rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
3088         if (rc)
3089                 goto err0;
3090
3091         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3092         if (rc) {
3093                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
3094                 goto err0;
3095         }
3096
3097         /* Index only, 18 bit long, lkey = itid << 8 | key */
3098         mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
3099         mr->hw_mr.key = 0;
3100         mr->hw_mr.pd = pd->pd_id;
3101         mr->hw_mr.local_read = 1;
3102         mr->hw_mr.local_write = 0;
3103         mr->hw_mr.remote_read = 0;
3104         mr->hw_mr.remote_write = 0;
3105         mr->hw_mr.remote_atomic = 0;
3106         mr->hw_mr.mw_bind = false;
3107         mr->hw_mr.pbl_ptr = 0;
3108         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3109         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3110         mr->hw_mr.length = 0;
3111         mr->hw_mr.vaddr = 0;
3112         mr->hw_mr.phy_mr = true;
3113         mr->hw_mr.dma_mr = false;
3114
3115         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3116         if (rc) {
3117                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3118                 goto err1;
3119         }
3120
3121         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3122         mr->ibmr.rkey = mr->ibmr.lkey;
3123
3124         DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
3125         return mr;
3126
3127 err1:
3128         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3129 err0:
3130         kfree(mr);
3131         return ERR_PTR(rc);
3132 }
3133
3134 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
3135                             u32 max_num_sg)
3136 {
3137         struct qedr_mr *mr;
3138
3139         if (mr_type != IB_MR_TYPE_MEM_REG)
3140                 return ERR_PTR(-EINVAL);
3141
3142         mr = __qedr_alloc_mr(ibpd, max_num_sg);
3143
3144         if (IS_ERR(mr))
3145                 return ERR_PTR(-EINVAL);
3146
3147         return &mr->ibmr;
3148 }
3149
3150 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3151 {
3152         struct qedr_mr *mr = get_qedr_mr(ibmr);
3153         struct qedr_pbl *pbl_table;
3154         struct regpair *pbe;
3155         u32 pbes_in_page;
3156
3157         if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
3158                 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
3159                 return -ENOMEM;
3160         }
3161
3162         DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3163                  mr->npages, addr);
3164
3165         pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3166         pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3167         pbe = (struct regpair *)pbl_table->va;
3168         pbe +=  mr->npages % pbes_in_page;
3169         pbe->lo = cpu_to_le32((u32)addr);
3170         pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3171
3172         mr->npages++;
3173
3174         return 0;
3175 }
3176
3177 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3178 {
3179         int work = info->completed - info->completed_handled - 1;
3180
3181         DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3182         while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
3183                 struct qedr_pbl *pbl;
3184
3185                 /* Free all the page list that are possible to be freed
3186                  * (all the ones that were invalidated), under the assumption
3187                  * that if an FMR was completed successfully that means that
3188                  * if there was an invalidate operation before it also ended
3189                  */
3190                 pbl = list_first_entry(&info->inuse_pbl_list,
3191                                        struct qedr_pbl, list_entry);
3192                 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
3193                 info->completed_handled++;
3194         }
3195 }
3196
3197 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3198                    int sg_nents, unsigned int *sg_offset)
3199 {
3200         struct qedr_mr *mr = get_qedr_mr(ibmr);
3201
3202         mr->npages = 0;
3203
3204         handle_completed_mrs(mr->dev, &mr->info);
3205         return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
3206 }
3207
3208 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3209 {
3210         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3211         struct qedr_pd *pd = get_qedr_pd(ibpd);
3212         struct qedr_mr *mr;
3213         int rc;
3214
3215         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3216         if (!mr)
3217                 return ERR_PTR(-ENOMEM);
3218
3219         mr->type = QEDR_MR_DMA;
3220
3221         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3222         if (rc) {
3223                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
3224                 goto err1;
3225         }
3226
3227         /* index only, 18 bit long, lkey = itid << 8 | key */
3228         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3229         mr->hw_mr.pd = pd->pd_id;
3230         mr->hw_mr.local_read = 1;
3231         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3232         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3233         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3234         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3235         mr->hw_mr.dma_mr = true;
3236
3237         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3238         if (rc) {
3239                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3240                 goto err2;
3241         }
3242
3243         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3244         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3245             mr->hw_mr.remote_atomic)
3246                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3247
3248         DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3249         return &mr->ibmr;
3250
3251 err2:
3252         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3253 err1:
3254         kfree(mr);
3255         return ERR_PTR(rc);
3256 }
3257
3258 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3259 {
3260         return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3261 }
3262
3263 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3264 {
3265         int i, len = 0;
3266
3267         for (i = 0; i < num_sge; i++)
3268                 len += sg_list[i].length;
3269
3270         return len;
3271 }
3272
3273 static void swap_wqe_data64(u64 *p)
3274 {
3275         int i;
3276
3277         for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3278                 *p = cpu_to_be64(cpu_to_le64(*p));
3279 }
3280
3281 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3282                                        struct qedr_qp *qp, u8 *wqe_size,
3283                                        const struct ib_send_wr *wr,
3284                                        const struct ib_send_wr **bad_wr,
3285                                        u8 *bits, u8 bit)
3286 {
3287         u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3288         char *seg_prt, *wqe;
3289         int i, seg_siz;
3290
3291         if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3292                 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3293                 *bad_wr = wr;
3294                 return 0;
3295         }
3296
3297         if (!data_size)
3298                 return data_size;
3299
3300         *bits |= bit;
3301
3302         seg_prt = NULL;
3303         wqe = NULL;
3304         seg_siz = 0;
3305
3306         /* Copy data inline */
3307         for (i = 0; i < wr->num_sge; i++) {
3308                 u32 len = wr->sg_list[i].length;
3309                 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3310
3311                 while (len > 0) {
3312                         u32 cur;
3313
3314                         /* New segment required */
3315                         if (!seg_siz) {
3316                                 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3317                                 seg_prt = wqe;
3318                                 seg_siz = sizeof(struct rdma_sq_common_wqe);
3319                                 (*wqe_size)++;
3320                         }
3321
3322                         /* Calculate currently allowed length */
3323                         cur = min_t(u32, len, seg_siz);
3324                         memcpy(seg_prt, src, cur);
3325
3326                         /* Update segment variables */
3327                         seg_prt += cur;
3328                         seg_siz -= cur;
3329
3330                         /* Update sge variables */
3331                         src += cur;
3332                         len -= cur;
3333
3334                         /* Swap fully-completed segments */
3335                         if (!seg_siz)
3336                                 swap_wqe_data64((u64 *)wqe);
3337                 }
3338         }
3339
3340         /* swap last not completed segment */
3341         if (seg_siz)
3342                 swap_wqe_data64((u64 *)wqe);
3343
3344         return data_size;
3345 }
3346
3347 #define RQ_SGE_SET(sge, vaddr, vlength, vflags)                 \
3348         do {                                                    \
3349                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3350                 (sge)->length = cpu_to_le32(vlength);           \
3351                 (sge)->flags = cpu_to_le32(vflags);             \
3352         } while (0)
3353
3354 #define SRQ_HDR_SET(hdr, vwr_id, num_sge)                       \
3355         do {                                                    \
3356                 DMA_REGPAIR_LE(hdr->wr_id, vwr_id);             \
3357                 (hdr)->num_sges = num_sge;                      \
3358         } while (0)
3359
3360 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)                 \
3361         do {                                                    \
3362                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3363                 (sge)->length = cpu_to_le32(vlength);           \
3364                 (sge)->l_key = cpu_to_le32(vlkey);              \
3365         } while (0)
3366
3367 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3368                                 const struct ib_send_wr *wr)
3369 {
3370         u32 data_size = 0;
3371         int i;
3372
3373         for (i = 0; i < wr->num_sge; i++) {
3374                 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3375
3376                 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3377                 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3378                 sge->length = cpu_to_le32(wr->sg_list[i].length);
3379                 data_size += wr->sg_list[i].length;
3380         }
3381
3382         if (wqe_size)
3383                 *wqe_size += wr->num_sge;
3384
3385         return data_size;
3386 }
3387
3388 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3389                                      struct qedr_qp *qp,
3390                                      struct rdma_sq_rdma_wqe_1st *rwqe,
3391                                      struct rdma_sq_rdma_wqe_2nd *rwqe2,
3392                                      const struct ib_send_wr *wr,
3393                                      const struct ib_send_wr **bad_wr)
3394 {
3395         rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3396         DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3397
3398         if (wr->send_flags & IB_SEND_INLINE &&
3399             (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3400              wr->opcode == IB_WR_RDMA_WRITE)) {
3401                 u8 flags = 0;
3402
3403                 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3404                 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3405                                                    bad_wr, &rwqe->flags, flags);
3406         }
3407
3408         return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3409 }
3410
3411 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3412                                      struct qedr_qp *qp,
3413                                      struct rdma_sq_send_wqe_1st *swqe,
3414                                      struct rdma_sq_send_wqe_2st *swqe2,
3415                                      const struct ib_send_wr *wr,
3416                                      const struct ib_send_wr **bad_wr)
3417 {
3418         memset(swqe2, 0, sizeof(*swqe2));
3419         if (wr->send_flags & IB_SEND_INLINE) {
3420                 u8 flags = 0;
3421
3422                 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3423                 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3424                                                    bad_wr, &swqe->flags, flags);
3425         }
3426
3427         return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3428 }
3429
3430 static int qedr_prepare_reg(struct qedr_qp *qp,
3431                             struct rdma_sq_fmr_wqe_1st *fwqe1,
3432                             const struct ib_reg_wr *wr)
3433 {
3434         struct qedr_mr *mr = get_qedr_mr(wr->mr);
3435         struct rdma_sq_fmr_wqe_2nd *fwqe2;
3436
3437         fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3438         fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3439         fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3440         fwqe1->l_key = wr->key;
3441
3442         fwqe2->access_ctrl = 0;
3443
3444         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3445                    !!(wr->access & IB_ACCESS_REMOTE_READ));
3446         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3447                    !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3448         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3449                    !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3450         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3451         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3452                    !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3453         fwqe2->fmr_ctrl = 0;
3454
3455         SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3456                    ilog2(mr->ibmr.page_size) - 12);
3457
3458         fwqe2->length_hi = 0;
3459         fwqe2->length_lo = mr->ibmr.length;
3460         fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3461         fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3462
3463         qp->wqe_wr_id[qp->sq.prod].mr = mr;
3464
3465         return 0;
3466 }
3467
3468 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3469 {
3470         switch (opcode) {
3471         case IB_WR_RDMA_WRITE:
3472         case IB_WR_RDMA_WRITE_WITH_IMM:
3473                 return IB_WC_RDMA_WRITE;
3474         case IB_WR_SEND_WITH_IMM:
3475         case IB_WR_SEND:
3476         case IB_WR_SEND_WITH_INV:
3477                 return IB_WC_SEND;
3478         case IB_WR_RDMA_READ:
3479         case IB_WR_RDMA_READ_WITH_INV:
3480                 return IB_WC_RDMA_READ;
3481         case IB_WR_ATOMIC_CMP_AND_SWP:
3482                 return IB_WC_COMP_SWAP;
3483         case IB_WR_ATOMIC_FETCH_AND_ADD:
3484                 return IB_WC_FETCH_ADD;
3485         case IB_WR_REG_MR:
3486                 return IB_WC_REG_MR;
3487         case IB_WR_LOCAL_INV:
3488                 return IB_WC_LOCAL_INV;
3489         default:
3490                 return IB_WC_SEND;
3491         }
3492 }
3493
3494 static inline bool qedr_can_post_send(struct qedr_qp *qp,
3495                                       const struct ib_send_wr *wr)
3496 {
3497         int wq_is_full, err_wr, pbl_is_full;
3498         struct qedr_dev *dev = qp->dev;
3499
3500         /* prevent SQ overflow and/or processing of a bad WR */
3501         err_wr = wr->num_sge > qp->sq.max_sges;
3502         wq_is_full = qedr_wq_is_full(&qp->sq);
3503         pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3504                       QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3505         if (wq_is_full || err_wr || pbl_is_full) {
3506                 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3507                         DP_ERR(dev,
3508                                "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3509                                qp);
3510                         qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3511                 }
3512
3513                 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3514                         DP_ERR(dev,
3515                                "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3516                                qp);
3517                         qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3518                 }
3519
3520                 if (pbl_is_full &&
3521                     !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3522                         DP_ERR(dev,
3523                                "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3524                                qp);
3525                         qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3526                 }
3527                 return false;
3528         }
3529         return true;
3530 }
3531
3532 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3533                             const struct ib_send_wr **bad_wr)
3534 {
3535         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3536         struct qedr_qp *qp = get_qedr_qp(ibqp);
3537         struct rdma_sq_atomic_wqe_1st *awqe1;
3538         struct rdma_sq_atomic_wqe_2nd *awqe2;
3539         struct rdma_sq_atomic_wqe_3rd *awqe3;
3540         struct rdma_sq_send_wqe_2st *swqe2;
3541         struct rdma_sq_local_inv_wqe *iwqe;
3542         struct rdma_sq_rdma_wqe_2nd *rwqe2;
3543         struct rdma_sq_send_wqe_1st *swqe;
3544         struct rdma_sq_rdma_wqe_1st *rwqe;
3545         struct rdma_sq_fmr_wqe_1st *fwqe1;
3546         struct rdma_sq_common_wqe *wqe;
3547         u32 length;
3548         int rc = 0;
3549         bool comp;
3550
3551         if (!qedr_can_post_send(qp, wr)) {
3552                 *bad_wr = wr;
3553                 return -ENOMEM;
3554         }
3555
3556         wqe = qed_chain_produce(&qp->sq.pbl);
3557         qp->wqe_wr_id[qp->sq.prod].signaled =
3558                 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3559
3560         wqe->flags = 0;
3561         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3562                    !!(wr->send_flags & IB_SEND_SOLICITED));
3563         comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3564         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3565         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3566                    !!(wr->send_flags & IB_SEND_FENCE));
3567         wqe->prev_wqe_size = qp->prev_wqe_size;
3568
3569         qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3570
3571         switch (wr->opcode) {
3572         case IB_WR_SEND_WITH_IMM:
3573                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3574                         rc = -EINVAL;
3575                         *bad_wr = wr;
3576                         break;
3577                 }
3578                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3579                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3580                 swqe->wqe_size = 2;
3581                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3582
3583                 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3584                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3585                                                    wr, bad_wr);
3586                 swqe->length = cpu_to_le32(length);
3587                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3588                 qp->prev_wqe_size = swqe->wqe_size;
3589                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3590                 break;
3591         case IB_WR_SEND:
3592                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3593                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3594
3595                 swqe->wqe_size = 2;
3596                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3597                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3598                                                    wr, bad_wr);
3599                 swqe->length = cpu_to_le32(length);
3600                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3601                 qp->prev_wqe_size = swqe->wqe_size;
3602                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3603                 break;
3604         case IB_WR_SEND_WITH_INV:
3605                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3606                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3607                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3608                 swqe->wqe_size = 2;
3609                 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3610                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3611                                                    wr, bad_wr);
3612                 swqe->length = cpu_to_le32(length);
3613                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3614                 qp->prev_wqe_size = swqe->wqe_size;
3615                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3616                 break;
3617
3618         case IB_WR_RDMA_WRITE_WITH_IMM:
3619                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3620                         rc = -EINVAL;
3621                         *bad_wr = wr;
3622                         break;
3623                 }
3624                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3625                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3626
3627                 rwqe->wqe_size = 2;
3628                 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3629                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3630                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3631                                                    wr, bad_wr);
3632                 rwqe->length = cpu_to_le32(length);
3633                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3634                 qp->prev_wqe_size = rwqe->wqe_size;
3635                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3636                 break;
3637         case IB_WR_RDMA_WRITE:
3638                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3639                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3640
3641                 rwqe->wqe_size = 2;
3642                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3643                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3644                                                    wr, bad_wr);
3645                 rwqe->length = cpu_to_le32(length);
3646                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3647                 qp->prev_wqe_size = rwqe->wqe_size;
3648                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3649                 break;
3650         case IB_WR_RDMA_READ_WITH_INV:
3651                 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3652                 fallthrough;    /* same is identical to RDMA READ */
3653
3654         case IB_WR_RDMA_READ:
3655                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3656                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3657
3658                 rwqe->wqe_size = 2;
3659                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3660                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3661                                                    wr, bad_wr);
3662                 rwqe->length = cpu_to_le32(length);
3663                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3664                 qp->prev_wqe_size = rwqe->wqe_size;
3665                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3666                 break;
3667
3668         case IB_WR_ATOMIC_CMP_AND_SWP:
3669         case IB_WR_ATOMIC_FETCH_AND_ADD:
3670                 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3671                 awqe1->wqe_size = 4;
3672
3673                 awqe2 = qed_chain_produce(&qp->sq.pbl);
3674                 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3675                 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3676
3677                 awqe3 = qed_chain_produce(&qp->sq.pbl);
3678
3679                 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3680                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3681                         DMA_REGPAIR_LE(awqe3->swap_data,
3682                                        atomic_wr(wr)->compare_add);
3683                 } else {
3684                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3685                         DMA_REGPAIR_LE(awqe3->swap_data,
3686                                        atomic_wr(wr)->swap);
3687                         DMA_REGPAIR_LE(awqe3->cmp_data,
3688                                        atomic_wr(wr)->compare_add);
3689                 }
3690
3691                 qedr_prepare_sq_sges(qp, NULL, wr);
3692
3693                 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3694                 qp->prev_wqe_size = awqe1->wqe_size;
3695                 break;
3696
3697         case IB_WR_LOCAL_INV:
3698                 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3699                 iwqe->wqe_size = 1;
3700
3701                 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3702                 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3703                 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3704                 qp->prev_wqe_size = iwqe->wqe_size;
3705                 break;
3706         case IB_WR_REG_MR:
3707                 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3708                 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3709                 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3710                 fwqe1->wqe_size = 2;
3711
3712                 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3713                 if (rc) {
3714                         DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3715                         *bad_wr = wr;
3716                         break;
3717                 }
3718
3719                 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3720                 qp->prev_wqe_size = fwqe1->wqe_size;
3721                 break;
3722         default:
3723                 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3724                 rc = -EINVAL;
3725                 *bad_wr = wr;
3726                 break;
3727         }
3728
3729         if (*bad_wr) {
3730                 u16 value;
3731
3732                 /* Restore prod to its position before
3733                  * this WR was processed
3734                  */
3735                 value = le16_to_cpu(qp->sq.db_data.data.value);
3736                 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3737
3738                 /* Restore prev_wqe_size */
3739                 qp->prev_wqe_size = wqe->prev_wqe_size;
3740                 rc = -EINVAL;
3741                 DP_ERR(dev, "POST SEND FAILED\n");
3742         }
3743
3744         return rc;
3745 }
3746
3747 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3748                    const struct ib_send_wr **bad_wr)
3749 {
3750         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3751         struct qedr_qp *qp = get_qedr_qp(ibqp);
3752         unsigned long flags;
3753         int rc = 0;
3754
3755         *bad_wr = NULL;
3756
3757         if (qp->qp_type == IB_QPT_GSI)
3758                 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3759
3760         spin_lock_irqsave(&qp->q_lock, flags);
3761
3762         if (rdma_protocol_roce(&dev->ibdev, 1)) {
3763                 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3764                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
3765                     (qp->state != QED_ROCE_QP_STATE_SQD)) {
3766                         spin_unlock_irqrestore(&qp->q_lock, flags);
3767                         *bad_wr = wr;
3768                         DP_DEBUG(dev, QEDR_MSG_CQ,
3769                                  "QP in wrong state! QP icid=0x%x state %d\n",
3770                                  qp->icid, qp->state);
3771                         return -EINVAL;
3772                 }
3773         }
3774
3775         while (wr) {
3776                 rc = __qedr_post_send(ibqp, wr, bad_wr);
3777                 if (rc)
3778                         break;
3779
3780                 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3781
3782                 qedr_inc_sw_prod(&qp->sq);
3783
3784                 qp->sq.db_data.data.value++;
3785
3786                 wr = wr->next;
3787         }
3788
3789         /* Trigger doorbell
3790          * If there was a failure in the first WR then it will be triggered in
3791          * vane. However this is not harmful (as long as the producer value is
3792          * unchanged). For performance reasons we avoid checking for this
3793          * redundant doorbell.
3794          *
3795          * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3796          * soon as we give the doorbell, we could get a completion
3797          * for this wr, therefore we need to make sure that the
3798          * memory is updated before giving the doorbell.
3799          * During qedr_poll_cq, rmb is called before accessing the
3800          * cqe. This covers for the smp_rmb as well.
3801          */
3802         smp_wmb();
3803         writel(qp->sq.db_data.raw, qp->sq.db);
3804
3805         spin_unlock_irqrestore(&qp->q_lock, flags);
3806
3807         return rc;
3808 }
3809
3810 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3811 {
3812         u32 used;
3813
3814         /* Calculate number of elements used based on producer
3815          * count and consumer count and subtract it from max
3816          * work request supported so that we get elements left.
3817          */
3818         used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3819
3820         return hw_srq->max_wr - used;
3821 }
3822
3823 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3824                        const struct ib_recv_wr **bad_wr)
3825 {
3826         struct qedr_srq *srq = get_qedr_srq(ibsrq);
3827         struct qedr_srq_hwq_info *hw_srq;
3828         struct qedr_dev *dev = srq->dev;
3829         struct qed_chain *pbl;
3830         unsigned long flags;
3831         int status = 0;
3832         u32 num_sge;
3833
3834         spin_lock_irqsave(&srq->lock, flags);
3835
3836         hw_srq = &srq->hw_srq;
3837         pbl = &srq->hw_srq.pbl;
3838         while (wr) {
3839                 struct rdma_srq_wqe_header *hdr;
3840                 int i;
3841
3842                 if (!qedr_srq_elem_left(hw_srq) ||
3843                     wr->num_sge > srq->hw_srq.max_sges) {
3844                         DP_ERR(dev, "Can't post WR  (%d,%d) || (%d > %d)\n",
3845                                hw_srq->wr_prod_cnt,
3846                                atomic_read(&hw_srq->wr_cons_cnt),
3847                                wr->num_sge, srq->hw_srq.max_sges);
3848                         status = -ENOMEM;
3849                         *bad_wr = wr;
3850                         break;
3851                 }
3852
3853                 hdr = qed_chain_produce(pbl);
3854                 num_sge = wr->num_sge;
3855                 /* Set number of sge and work request id in header */
3856                 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3857
3858                 srq->hw_srq.wr_prod_cnt++;
3859                 hw_srq->wqe_prod++;
3860                 hw_srq->sge_prod++;
3861
3862                 DP_DEBUG(dev, QEDR_MSG_SRQ,
3863                          "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3864                          wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3865
3866                 for (i = 0; i < wr->num_sge; i++) {
3867                         struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3868
3869                         /* Set SGE length, lkey and address */
3870                         SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3871                                     wr->sg_list[i].length, wr->sg_list[i].lkey);
3872
3873                         DP_DEBUG(dev, QEDR_MSG_SRQ,
3874                                  "[%d]: len %d key %x addr %x:%x\n",
3875                                  i, srq_sge->length, srq_sge->l_key,
3876                                  srq_sge->addr.hi, srq_sge->addr.lo);
3877                         hw_srq->sge_prod++;
3878                 }
3879
3880                 /* Update WQE and SGE information before
3881                  * updating producer.
3882                  */
3883                 dma_wmb();
3884
3885                 /* SRQ producer is 8 bytes. Need to update SGE producer index
3886                  * in first 4 bytes and need to update WQE producer in
3887                  * next 4 bytes.
3888                  */
3889                 srq->hw_srq.virt_prod_pair_addr->sge_prod = cpu_to_le32(hw_srq->sge_prod);
3890                 /* Make sure sge producer is updated first */
3891                 dma_wmb();
3892                 srq->hw_srq.virt_prod_pair_addr->wqe_prod = cpu_to_le32(hw_srq->wqe_prod);
3893
3894                 wr = wr->next;
3895         }
3896
3897         DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3898                  qed_chain_get_elem_left(pbl));
3899         spin_unlock_irqrestore(&srq->lock, flags);
3900
3901         return status;
3902 }
3903
3904 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3905                    const struct ib_recv_wr **bad_wr)
3906 {
3907         struct qedr_qp *qp = get_qedr_qp(ibqp);
3908         struct qedr_dev *dev = qp->dev;
3909         unsigned long flags;
3910         int status = 0;
3911
3912         if (qp->qp_type == IB_QPT_GSI)
3913                 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3914
3915         spin_lock_irqsave(&qp->q_lock, flags);
3916
3917         if (qp->state == QED_ROCE_QP_STATE_RESET) {
3918                 spin_unlock_irqrestore(&qp->q_lock, flags);
3919                 *bad_wr = wr;
3920                 return -EINVAL;
3921         }
3922
3923         while (wr) {
3924                 int i;
3925
3926                 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3927                     QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3928                     wr->num_sge > qp->rq.max_sges) {
3929                         DP_ERR(dev, "Can't post WR  (%d < %d) || (%d > %d)\n",
3930                                qed_chain_get_elem_left_u32(&qp->rq.pbl),
3931                                QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3932                                qp->rq.max_sges);
3933                         status = -ENOMEM;
3934                         *bad_wr = wr;
3935                         break;
3936                 }
3937                 for (i = 0; i < wr->num_sge; i++) {
3938                         u32 flags = 0;
3939                         struct rdma_rq_sge *rqe =
3940                             qed_chain_produce(&qp->rq.pbl);
3941
3942                         /* First one must include the number
3943                          * of SGE in the list
3944                          */
3945                         if (!i)
3946                                 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3947                                           wr->num_sge);
3948
3949                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3950                                   wr->sg_list[i].lkey);
3951
3952                         RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3953                                    wr->sg_list[i].length, flags);
3954                 }
3955
3956                 /* Special case of no sges. FW requires between 1-4 sges...
3957                  * in this case we need to post 1 sge with length zero. this is
3958                  * because rdma write with immediate consumes an RQ.
3959                  */
3960                 if (!wr->num_sge) {
3961                         u32 flags = 0;
3962                         struct rdma_rq_sge *rqe =
3963                             qed_chain_produce(&qp->rq.pbl);
3964
3965                         /* First one must include the number
3966                          * of SGE in the list
3967                          */
3968                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3969                         SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3970
3971                         RQ_SGE_SET(rqe, 0, 0, flags);
3972                         i = 1;
3973                 }
3974
3975                 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3976                 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3977
3978                 qedr_inc_sw_prod(&qp->rq);
3979
3980                 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3981                  * soon as we give the doorbell, we could get a completion
3982                  * for this wr, therefore we need to make sure that the
3983                  * memory is update before giving the doorbell.
3984                  * During qedr_poll_cq, rmb is called before accessing the
3985                  * cqe. This covers for the smp_rmb as well.
3986                  */
3987                 smp_wmb();
3988
3989                 qp->rq.db_data.data.value++;
3990
3991                 writel(qp->rq.db_data.raw, qp->rq.db);
3992
3993                 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3994                         writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3995                 }
3996
3997                 wr = wr->next;
3998         }
3999
4000         spin_unlock_irqrestore(&qp->q_lock, flags);
4001
4002         return status;
4003 }
4004
4005 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
4006 {
4007         struct rdma_cqe_requester *resp_cqe = &cqe->req;
4008
4009         return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
4010                 cq->pbl_toggle;
4011 }
4012
4013 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
4014 {
4015         struct rdma_cqe_requester *resp_cqe = &cqe->req;
4016         struct qedr_qp *qp;
4017
4018         qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
4019                                                    resp_cqe->qp_handle.lo,
4020                                                    u64);
4021         return qp;
4022 }
4023
4024 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
4025 {
4026         struct rdma_cqe_requester *resp_cqe = &cqe->req;
4027
4028         return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
4029 }
4030
4031 /* Return latest CQE (needs processing) */
4032 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
4033 {
4034         return cq->latest_cqe;
4035 }
4036
4037 /* In fmr we need to increase the number of fmr completed counter for the fmr
4038  * algorithm determining whether we can free a pbl or not.
4039  * we need to perform this whether the work request was signaled or not. for
4040  * this purpose we call this function from the condition that checks if a wr
4041  * should be skipped, to make sure we don't miss it ( possibly this fmr
4042  * operation was not signalted)
4043  */
4044 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
4045 {
4046         if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
4047                 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4048 }
4049
4050 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
4051                        struct qedr_cq *cq, int num_entries,
4052                        struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
4053                        int force)
4054 {
4055         u16 cnt = 0;
4056
4057         while (num_entries && qp->sq.wqe_cons != hw_cons) {
4058                 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
4059                         qedr_chk_if_fmr(qp);
4060                         /* skip WC */
4061                         goto next_cqe;
4062                 }
4063
4064                 /* fill WC */
4065                 wc->status = status;
4066                 wc->vendor_err = 0;
4067                 wc->wc_flags = 0;
4068                 wc->src_qp = qp->id;
4069                 wc->qp = &qp->ibqp;
4070
4071                 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
4072                 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
4073
4074                 switch (wc->opcode) {
4075                 case IB_WC_RDMA_WRITE:
4076                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4077                         break;
4078                 case IB_WC_COMP_SWAP:
4079                 case IB_WC_FETCH_ADD:
4080                         wc->byte_len = 8;
4081                         break;
4082                 case IB_WC_REG_MR:
4083                         qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4084                         break;
4085                 case IB_WC_RDMA_READ:
4086                 case IB_WC_SEND:
4087                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4088                         break;
4089                 default:
4090                         break;
4091                 }
4092
4093                 num_entries--;
4094                 wc++;
4095                 cnt++;
4096 next_cqe:
4097                 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
4098                         qed_chain_consume(&qp->sq.pbl);
4099                 qedr_inc_sw_cons(&qp->sq);
4100         }
4101
4102         return cnt;
4103 }
4104
4105 static int qedr_poll_cq_req(struct qedr_dev *dev,
4106                             struct qedr_qp *qp, struct qedr_cq *cq,
4107                             int num_entries, struct ib_wc *wc,
4108                             struct rdma_cqe_requester *req)
4109 {
4110         int cnt = 0;
4111
4112         switch (req->status) {
4113         case RDMA_CQE_REQ_STS_OK:
4114                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4115                                   IB_WC_SUCCESS, 0);
4116                 break;
4117         case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
4118                 if (qp->state != QED_ROCE_QP_STATE_ERR)
4119                         DP_DEBUG(dev, QEDR_MSG_CQ,
4120                                  "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4121                                  cq->icid, qp->icid);
4122                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4123                                   IB_WC_WR_FLUSH_ERR, 1);
4124                 break;
4125         default:
4126                 /* process all WQE before the cosumer */
4127                 qp->state = QED_ROCE_QP_STATE_ERR;
4128                 cnt = process_req(dev, qp, cq, num_entries, wc,
4129                                   req->sq_cons - 1, IB_WC_SUCCESS, 0);
4130                 wc += cnt;
4131                 /* if we have extra WC fill it with actual error info */
4132                 if (cnt < num_entries) {
4133                         enum ib_wc_status wc_status;
4134
4135                         switch (req->status) {
4136                         case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4137                                 DP_ERR(dev,
4138                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4139                                        cq->icid, qp->icid);
4140                                 wc_status = IB_WC_BAD_RESP_ERR;
4141                                 break;
4142                         case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4143                                 DP_ERR(dev,
4144                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4145                                        cq->icid, qp->icid);
4146                                 wc_status = IB_WC_LOC_LEN_ERR;
4147                                 break;
4148                         case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4149                                 DP_ERR(dev,
4150                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4151                                        cq->icid, qp->icid);
4152                                 wc_status = IB_WC_LOC_QP_OP_ERR;
4153                                 break;
4154                         case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4155                                 DP_ERR(dev,
4156                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4157                                        cq->icid, qp->icid);
4158                                 wc_status = IB_WC_LOC_PROT_ERR;
4159                                 break;
4160                         case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4161                                 DP_ERR(dev,
4162                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4163                                        cq->icid, qp->icid);
4164                                 wc_status = IB_WC_MW_BIND_ERR;
4165                                 break;
4166                         case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4167                                 DP_ERR(dev,
4168                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4169                                        cq->icid, qp->icid);
4170                                 wc_status = IB_WC_REM_INV_REQ_ERR;
4171                                 break;
4172                         case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4173                                 DP_ERR(dev,
4174                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4175                                        cq->icid, qp->icid);
4176                                 wc_status = IB_WC_REM_ACCESS_ERR;
4177                                 break;
4178                         case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4179                                 DP_ERR(dev,
4180                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4181                                        cq->icid, qp->icid);
4182                                 wc_status = IB_WC_REM_OP_ERR;
4183                                 break;
4184                         case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4185                                 DP_ERR(dev,
4186                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4187                                        cq->icid, qp->icid);
4188                                 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4189                                 break;
4190                         case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4191                                 DP_ERR(dev,
4192                                        "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4193                                        cq->icid, qp->icid);
4194                                 wc_status = IB_WC_RETRY_EXC_ERR;
4195                                 break;
4196                         default:
4197                                 DP_ERR(dev,
4198                                        "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4199                                        cq->icid, qp->icid);
4200                                 wc_status = IB_WC_GENERAL_ERR;
4201                         }
4202                         cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4203                                            wc_status, 1);
4204                 }
4205         }
4206
4207         return cnt;
4208 }
4209
4210 static inline int qedr_cqe_resp_status_to_ib(u8 status)
4211 {
4212         switch (status) {
4213         case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
4214                 return IB_WC_LOC_ACCESS_ERR;
4215         case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
4216                 return IB_WC_LOC_LEN_ERR;
4217         case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
4218                 return IB_WC_LOC_QP_OP_ERR;
4219         case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
4220                 return IB_WC_LOC_PROT_ERR;
4221         case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
4222                 return IB_WC_MW_BIND_ERR;
4223         case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
4224                 return IB_WC_REM_INV_RD_REQ_ERR;
4225         case RDMA_CQE_RESP_STS_OK:
4226                 return IB_WC_SUCCESS;
4227         default:
4228                 return IB_WC_GENERAL_ERR;
4229         }
4230 }
4231
4232 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4233                                           struct ib_wc *wc)
4234 {
4235         wc->status = IB_WC_SUCCESS;
4236         wc->byte_len = le32_to_cpu(resp->length);
4237
4238         if (resp->flags & QEDR_RESP_IMM) {
4239                 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
4240                 wc->wc_flags |= IB_WC_WITH_IMM;
4241
4242                 if (resp->flags & QEDR_RESP_RDMA)
4243                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4244
4245                 if (resp->flags & QEDR_RESP_INV)
4246                         return -EINVAL;
4247
4248         } else if (resp->flags & QEDR_RESP_INV) {
4249                 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4250                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4251
4252                 if (resp->flags & QEDR_RESP_RDMA)
4253                         return -EINVAL;
4254
4255         } else if (resp->flags & QEDR_RESP_RDMA) {
4256                 return -EINVAL;
4257         }
4258
4259         return 0;
4260 }
4261
4262 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4263                                struct qedr_cq *cq, struct ib_wc *wc,
4264                                struct rdma_cqe_responder *resp, u64 wr_id)
4265 {
4266         /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4267         wc->opcode = IB_WC_RECV;
4268         wc->wc_flags = 0;
4269
4270         if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4271                 if (qedr_set_ok_cqe_resp_wc(resp, wc))
4272                         DP_ERR(dev,
4273                                "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4274                                cq, cq->icid, resp->flags);
4275
4276         } else {
4277                 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4278                 if (wc->status == IB_WC_GENERAL_ERR)
4279                         DP_ERR(dev,
4280                                "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4281                                cq, cq->icid, resp->status);
4282         }
4283
4284         /* Fill the rest of the WC */
4285         wc->vendor_err = 0;
4286         wc->src_qp = qp->id;
4287         wc->qp = &qp->ibqp;
4288         wc->wr_id = wr_id;
4289 }
4290
4291 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4292                                 struct qedr_cq *cq, struct ib_wc *wc,
4293                                 struct rdma_cqe_responder *resp)
4294 {
4295         struct qedr_srq *srq = qp->srq;
4296         u64 wr_id;
4297
4298         wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4299                          le32_to_cpu(resp->srq_wr_id.lo), u64);
4300
4301         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4302                 wc->status = IB_WC_WR_FLUSH_ERR;
4303                 wc->vendor_err = 0;
4304                 wc->wr_id = wr_id;
4305                 wc->byte_len = 0;
4306                 wc->src_qp = qp->id;
4307                 wc->qp = &qp->ibqp;
4308                 wc->wr_id = wr_id;
4309         } else {
4310                 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4311         }
4312         atomic_inc(&srq->hw_srq.wr_cons_cnt);
4313
4314         return 1;
4315 }
4316 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4317                             struct qedr_cq *cq, struct ib_wc *wc,
4318                             struct rdma_cqe_responder *resp)
4319 {
4320         u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4321
4322         __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4323
4324         while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4325                 qed_chain_consume(&qp->rq.pbl);
4326         qedr_inc_sw_cons(&qp->rq);
4327
4328         return 1;
4329 }
4330
4331 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4332                               int num_entries, struct ib_wc *wc, u16 hw_cons)
4333 {
4334         u16 cnt = 0;
4335
4336         while (num_entries && qp->rq.wqe_cons != hw_cons) {
4337                 /* fill WC */
4338                 wc->status = IB_WC_WR_FLUSH_ERR;
4339                 wc->vendor_err = 0;
4340                 wc->wc_flags = 0;
4341                 wc->src_qp = qp->id;
4342                 wc->byte_len = 0;
4343                 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4344                 wc->qp = &qp->ibqp;
4345                 num_entries--;
4346                 wc++;
4347                 cnt++;
4348                 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4349                         qed_chain_consume(&qp->rq.pbl);
4350                 qedr_inc_sw_cons(&qp->rq);
4351         }
4352
4353         return cnt;
4354 }
4355
4356 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4357                                  struct rdma_cqe_responder *resp, int *update)
4358 {
4359         if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4360                 consume_cqe(cq);
4361                 *update |= 1;
4362         }
4363 }
4364
4365 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4366                                  struct qedr_cq *cq, int num_entries,
4367                                  struct ib_wc *wc,
4368                                  struct rdma_cqe_responder *resp)
4369 {
4370         int cnt;
4371
4372         cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4373         consume_cqe(cq);
4374
4375         return cnt;
4376 }
4377
4378 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4379                              struct qedr_cq *cq, int num_entries,
4380                              struct ib_wc *wc, struct rdma_cqe_responder *resp,
4381                              int *update)
4382 {
4383         int cnt;
4384
4385         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4386                 cnt = process_resp_flush(qp, cq, num_entries, wc,
4387                                          resp->rq_cons_or_srq_id);
4388                 try_consume_resp_cqe(cq, qp, resp, update);
4389         } else {
4390                 cnt = process_resp_one(dev, qp, cq, wc, resp);
4391                 consume_cqe(cq);
4392                 *update |= 1;
4393         }
4394
4395         return cnt;
4396 }
4397
4398 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4399                                 struct rdma_cqe_requester *req, int *update)
4400 {
4401         if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4402                 consume_cqe(cq);
4403                 *update |= 1;
4404         }
4405 }
4406
4407 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4408 {
4409         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4410         struct qedr_cq *cq = get_qedr_cq(ibcq);
4411         union rdma_cqe *cqe;
4412         u32 old_cons, new_cons;
4413         unsigned long flags;
4414         int update = 0;
4415         int done = 0;
4416
4417         if (cq->destroyed) {
4418                 DP_ERR(dev,
4419                        "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4420                        cq, cq->icid);
4421                 return 0;
4422         }
4423
4424         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4425                 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4426
4427         spin_lock_irqsave(&cq->cq_lock, flags);
4428         cqe = cq->latest_cqe;
4429         old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4430         while (num_entries && is_valid_cqe(cq, cqe)) {
4431                 struct qedr_qp *qp;
4432                 int cnt = 0;
4433
4434                 /* prevent speculative reads of any field of CQE */
4435                 rmb();
4436
4437                 qp = cqe_get_qp(cqe);
4438                 if (!qp) {
4439                         WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4440                         break;
4441                 }
4442
4443                 wc->qp = &qp->ibqp;
4444
4445                 switch (cqe_get_type(cqe)) {
4446                 case RDMA_CQE_TYPE_REQUESTER:
4447                         cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4448                                                &cqe->req);
4449                         try_consume_req_cqe(cq, qp, &cqe->req, &update);
4450                         break;
4451                 case RDMA_CQE_TYPE_RESPONDER_RQ:
4452                         cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4453                                                 &cqe->resp, &update);
4454                         break;
4455                 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4456                         cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4457                                                     wc, &cqe->resp);
4458                         update = 1;
4459                         break;
4460                 case RDMA_CQE_TYPE_INVALID:
4461                 default:
4462                         DP_ERR(dev, "Error: invalid CQE type = %d\n",
4463                                cqe_get_type(cqe));
4464                 }
4465                 num_entries -= cnt;
4466                 wc += cnt;
4467                 done += cnt;
4468
4469                 cqe = get_cqe(cq);
4470         }
4471         new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4472
4473         cq->cq_cons += new_cons - old_cons;
4474
4475         if (update)
4476                 /* doorbell notifies abount latest VALID entry,
4477                  * but chain already point to the next INVALID one
4478                  */
4479                 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4480
4481         spin_unlock_irqrestore(&cq->cq_lock, flags);
4482         return done;
4483 }
4484
4485 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4486                      u8 port_num, const struct ib_wc *in_wc,
4487                      const struct ib_grh *in_grh, const struct ib_mad *in,
4488                      struct ib_mad *out_mad, size_t *out_mad_size,
4489                      u16 *out_mad_pkey_index)
4490 {
4491         return IB_MAD_RESULT_SUCCESS;
4492 }