drivers/infiniband/hw/qedr/verbs.c

   1 /* QLogic qedr NIC Driver
   2  * Copyright (c) 2015-2016  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/dma-mapping.h>
  33 #include <linux/crc32.h>
  34 #include <net/ip.h>
  35 #include <net/ipv6.h>
  36 #include <net/udp.h>
  37 #include <linux/iommu.h>
  38
  39 #include <rdma/ib_verbs.h>
  40 #include <rdma/ib_user_verbs.h>
  41 #include <rdma/iw_cm.h>
  42 #include <rdma/ib_umem.h>
  43 #include <rdma/ib_addr.h>
  44 #include <rdma/ib_cache.h>
  45 #include <rdma/uverbs_ioctl.h>
  46
  47 #include <linux/qed/common_hsi.h>
  48 #include "qedr_hsi_rdma.h"
  49 #include <linux/qed/qed_if.h>
  50 #include "qedr.h"
  51 #include "verbs.h"
  52 #include <rdma/qedr-abi.h>
  53 #include "qedr_roce_cm.h"
  54 #include "qedr_iw_cm.h"
  55
  56 #define QEDR_SRQ_WQE_ELEM_SIZE  sizeof(union rdma_srq_elm)
  57 #define RDMA_MAX_SGE_PER_SRQ    (4)
  58 #define RDMA_MAX_SRQ_WQE_SIZE   (RDMA_MAX_SGE_PER_SRQ + 1)
  59
  60 #define DB_ADDR_SHIFT(addr)             ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
  61
  62 enum {
  63         QEDR_USER_MMAP_IO_WC = 0,
  64         QEDR_USER_MMAP_PHYS_PAGE,
  65 };
  66
  67 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
  68                                         size_t len)
  69 {
  70         size_t min_len = min_t(size_t, len, udata->outlen);
  71
  72         return ib_copy_to_udata(udata, src, min_len);
  73 }
  74
  75 int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
  76 {
  77         if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
  78                 return -EINVAL;
  79
  80         *pkey = QEDR_ROCE_PKEY_DEFAULT;
  81         return 0;
  82 }
  83
  84 int qedr_iw_query_gid(struct ib_device *ibdev, u32 port,
  85                       int index, union ib_gid *sgid)
  86 {
  87         struct qedr_dev *dev = get_qedr_dev(ibdev);
  88
  89         memset(sgid->raw, 0, sizeof(sgid->raw));
  90         ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
  91
  92         DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
  93                  sgid->global.interface_id, sgid->global.subnet_prefix);
  94
  95         return 0;
  96 }
  97
  98 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
  99 {
 100         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
 101         struct qedr_device_attr *qattr = &dev->attr;
 102         struct qedr_srq *srq = get_qedr_srq(ibsrq);
 103
 104         srq_attr->srq_limit = srq->srq_limit;
 105         srq_attr->max_wr = qattr->max_srq_wr;
 106         srq_attr->max_sge = qattr->max_sge;
 107
 108         return 0;
 109 }
 110
 111 int qedr_query_device(struct ib_device *ibdev,
 112                       struct ib_device_attr *attr, struct ib_udata *udata)
 113 {
 114         struct qedr_dev *dev = get_qedr_dev(ibdev);
 115         struct qedr_device_attr *qattr = &dev->attr;
 116
 117         if (!dev->rdma_ctx) {
 118                 DP_ERR(dev,
 119                        "qedr_query_device called with invalid params rdma_ctx=%p\n",
 120                        dev->rdma_ctx);
 121                 return -EINVAL;
 122         }
 123
 124         memset(attr, 0, sizeof(*attr));
 125
 126         attr->fw_ver = qattr->fw_ver;
 127         attr->sys_image_guid = qattr->sys_image_guid;
 128         attr->max_mr_size = qattr->max_mr_size;
 129         attr->page_size_cap = qattr->page_size_caps;
 130         attr->vendor_id = qattr->vendor_id;
 131         attr->vendor_part_id = qattr->vendor_part_id;
 132         attr->hw_ver = qattr->hw_ver;
 133         attr->max_qp = qattr->max_qp;
 134         attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
 135         attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
 136             IB_DEVICE_RC_RNR_NAK_GEN |
 137             IB_DEVICE_MEM_MGT_EXTENSIONS;
 138         attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
 139
 140         if (!rdma_protocol_iwarp(&dev->ibdev, 1))
 141                 attr->device_cap_flags |= IB_DEVICE_XRC;
 142         attr->max_send_sge = qattr->max_sge;
 143         attr->max_recv_sge = qattr->max_sge;
 144         attr->max_sge_rd = qattr->max_sge;
 145         attr->max_cq = qattr->max_cq;
 146         attr->max_cqe = qattr->max_cqe;
 147         attr->max_mr = qattr->max_mr;
 148         attr->max_mw = qattr->max_mw;
 149         attr->max_pd = qattr->max_pd;
 150         attr->atomic_cap = dev->atomic_cap;
 151         attr->max_qp_init_rd_atom =
 152             1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
 153         attr->max_qp_rd_atom =
 154             min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
 155                 attr->max_qp_init_rd_atom);
 156
 157         attr->max_srq = qattr->max_srq;
 158         attr->max_srq_sge = qattr->max_srq_sge;
 159         attr->max_srq_wr = qattr->max_srq_wr;
 160
 161         attr->local_ca_ack_delay = qattr->dev_ack_delay;
 162         attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
 163         attr->max_pkeys = qattr->max_pkey;
 164         attr->max_ah = qattr->max_ah;
 165
 166         return 0;
 167 }
 168
 169 static inline void get_link_speed_and_width(int speed, u16 *ib_speed,
 170                                             u8 *ib_width)
 171 {
 172         switch (speed) {
 173         case 1000:
 174                 *ib_speed = IB_SPEED_SDR;
 175                 *ib_width = IB_WIDTH_1X;
 176                 break;
 177         case 10000:
 178                 *ib_speed = IB_SPEED_QDR;
 179                 *ib_width = IB_WIDTH_1X;
 180                 break;
 181
 182         case 20000:
 183                 *ib_speed = IB_SPEED_DDR;
 184                 *ib_width = IB_WIDTH_4X;
 185                 break;
 186
 187         case 25000:
 188                 *ib_speed = IB_SPEED_EDR;
 189                 *ib_width = IB_WIDTH_1X;
 190                 break;
 191
 192         case 40000:
 193                 *ib_speed = IB_SPEED_QDR;
 194                 *ib_width = IB_WIDTH_4X;
 195                 break;
 196
 197         case 50000:
 198                 *ib_speed = IB_SPEED_HDR;
 199                 *ib_width = IB_WIDTH_1X;
 200                 break;
 201
 202         case 100000:
 203                 *ib_speed = IB_SPEED_EDR;
 204                 *ib_width = IB_WIDTH_4X;
 205                 break;
 206
 207         default:
 208                 /* Unsupported */
 209                 *ib_speed = IB_SPEED_SDR;
 210                 *ib_width = IB_WIDTH_1X;
 211         }
 212 }
 213
 214 int qedr_query_port(struct ib_device *ibdev, u32 port,
 215                     struct ib_port_attr *attr)
 216 {
 217         struct qedr_dev *dev;
 218         struct qed_rdma_port *rdma_port;
 219
 220         dev = get_qedr_dev(ibdev);
 221
 222         if (!dev->rdma_ctx) {
 223                 DP_ERR(dev, "rdma_ctx is NULL\n");
 224                 return -EINVAL;
 225         }
 226
 227         rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
 228
 229         /* *attr being zeroed by the caller, avoid zeroing it here */
 230         if (rdma_port->port_state == QED_RDMA_PORT_UP) {
 231                 attr->state = IB_PORT_ACTIVE;
 232                 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 233         } else {
 234                 attr->state = IB_PORT_DOWN;
 235                 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 236         }
 237         attr->max_mtu = IB_MTU_4096;
 238         attr->lid = 0;
 239         attr->lmc = 0;
 240         attr->sm_lid = 0;
 241         attr->sm_sl = 0;
 242         attr->ip_gids = true;
 243         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
 244                 attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu);
 245                 attr->gid_tbl_len = 1;
 246         } else {
 247                 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
 248                 attr->gid_tbl_len = QEDR_MAX_SGID;
 249                 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
 250         }
 251         attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
 252         attr->qkey_viol_cntr = 0;
 253         get_link_speed_and_width(rdma_port->link_speed,
 254                                  &attr->active_speed, &attr->active_width);
 255         attr->max_msg_sz = rdma_port->max_msg_size;
 256         attr->max_vl_num = 4;
 257
 258         return 0;
 259 }
 260
 261 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
 262 {
 263         struct ib_device *ibdev = uctx->device;
 264         int rc;
 265         struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
 266         struct qedr_alloc_ucontext_resp uresp = {};
 267         struct qedr_alloc_ucontext_req ureq = {};
 268         struct qedr_dev *dev = get_qedr_dev(ibdev);
 269         struct qed_rdma_add_user_out_params oparams;
 270         struct qedr_user_mmap_entry *entry;
 271
 272         if (!udata)
 273                 return -EFAULT;
 274
 275         if (udata->inlen) {
 276                 rc = ib_copy_from_udata(&ureq, udata,
 277                                         min(sizeof(ureq), udata->inlen));
 278                 if (rc) {
 279                         DP_ERR(dev, "Problem copying data from user space\n");
 280                         return -EFAULT;
 281                 }
 282                 ctx->edpm_mode = !!(ureq.context_flags &
 283                                     QEDR_ALLOC_UCTX_EDPM_MODE);
 284                 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
 285         }
 286
 287         rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
 288         if (rc) {
 289                 DP_ERR(dev,
 290                        "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
 291                        rc);
 292                 return rc;
 293         }
 294
 295         ctx->dpi = oparams.dpi;
 296         ctx->dpi_addr = oparams.dpi_addr;
 297         ctx->dpi_phys_addr = oparams.dpi_phys_addr;
 298         ctx->dpi_size = oparams.dpi_size;
 299         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 300         if (!entry) {
 301                 rc = -ENOMEM;
 302                 goto err;
 303         }
 304
 305         entry->io_address = ctx->dpi_phys_addr;
 306         entry->length = ctx->dpi_size;
 307         entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
 308         entry->dpi = ctx->dpi;
 309         entry->dev = dev;
 310         rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,
 311                                          ctx->dpi_size);
 312         if (rc) {
 313                 kfree(entry);
 314                 goto err;
 315         }
 316         ctx->db_mmap_entry = &entry->rdma_entry;
 317
 318         if (!dev->user_dpm_enabled)
 319                 uresp.dpm_flags = 0;
 320         else if (rdma_protocol_iwarp(&dev->ibdev, 1))
 321                 uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
 322         else
 323                 uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
 324                                   QEDR_DPM_TYPE_ROCE_LEGACY |
 325                                   QEDR_DPM_TYPE_ROCE_EDPM_MODE;
 326
 327         if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) {
 328                 uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
 329                 uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
 330                 uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
 331                 uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE;
 332         }
 333
 334         uresp.wids_enabled = 1;
 335         uresp.wid_count = oparams.wid_count;
 336         uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);
 337         uresp.db_size = ctx->dpi_size;
 338         uresp.max_send_wr = dev->attr.max_sqe;
 339         uresp.max_recv_wr = dev->attr.max_rqe;
 340         uresp.max_srq_wr = dev->attr.max_srq_wr;
 341         uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
 342         uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
 343         uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
 344         uresp.max_cqes = QEDR_MAX_CQES;
 345
 346         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 347         if (rc)
 348                 goto err;
 349
 350         ctx->dev = dev;
 351
 352         DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
 353                  &ctx->ibucontext);
 354         return 0;
 355
 356 err:
 357         if (!ctx->db_mmap_entry)
 358                 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
 359         else
 360                 rdma_user_mmap_entry_remove(ctx->db_mmap_entry);
 361
 362         return rc;
 363 }
 364
 365 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
 366 {
 367         struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
 368
 369         DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
 370                  uctx);
 371
 372         rdma_user_mmap_entry_remove(uctx->db_mmap_entry);
 373 }
 374
 375 void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
 376 {
 377         struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
 378         struct qedr_dev *dev = entry->dev;
 379
 380         if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
 381                 free_page((unsigned long)entry->address);
 382         else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
 383                 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
 384
 385         kfree(entry);
 386 }
 387
 388 int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
 389 {
 390         struct ib_device *dev = ucontext->device;
 391         size_t length = vma->vm_end - vma->vm_start;
 392         struct rdma_user_mmap_entry *rdma_entry;
 393         struct qedr_user_mmap_entry *entry;
 394         int rc = 0;
 395         u64 pfn;
 396
 397         ibdev_dbg(dev,
 398                   "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
 399                   vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
 400
 401         rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
 402         if (!rdma_entry) {
 403                 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
 404                           vma->vm_pgoff);
 405                 return -EINVAL;
 406         }
 407         entry = get_qedr_mmap_entry(rdma_entry);
 408         ibdev_dbg(dev,
 409                   "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
 410                   entry->io_address, length, entry->mmap_flag);
 411
 412         switch (entry->mmap_flag) {
 413         case QEDR_USER_MMAP_IO_WC:
 414                 pfn = entry->io_address >> PAGE_SHIFT;
 415                 rc = rdma_user_mmap_io(ucontext, vma, pfn, length,
 416                                        pgprot_writecombine(vma->vm_page_prot),
 417                                        rdma_entry);
 418                 break;
 419         case QEDR_USER_MMAP_PHYS_PAGE:
 420                 rc = vm_insert_page(vma, vma->vm_start,
 421                                     virt_to_page(entry->address));
 422                 break;
 423         default:
 424                 rc = -EINVAL;
 425         }
 426
 427         if (rc)
 428                 ibdev_dbg(dev,
 429                           "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
 430                           entry->io_address, length, entry->mmap_flag, rc);
 431
 432         rdma_user_mmap_entry_put(rdma_entry);
 433         return rc;
 434 }
 435
 436 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 437 {
 438         struct ib_device *ibdev = ibpd->device;
 439         struct qedr_dev *dev = get_qedr_dev(ibdev);
 440         struct qedr_pd *pd = get_qedr_pd(ibpd);
 441         u16 pd_id;
 442         int rc;
 443
 444         DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
 445                  udata ? "User Lib" : "Kernel");
 446
 447         if (!dev->rdma_ctx) {
 448                 DP_ERR(dev, "invalid RDMA context\n");
 449                 return -EINVAL;
 450         }
 451
 452         rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
 453         if (rc)
 454                 return rc;
 455
 456         pd->pd_id = pd_id;
 457
 458         if (udata) {
 459                 struct qedr_alloc_pd_uresp uresp = {
 460                         .pd_id = pd_id,
 461                 };
 462                 struct qedr_ucontext *context = rdma_udata_to_drv_context(
 463                         udata, struct qedr_ucontext, ibucontext);
 464
 465                 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 466                 if (rc) {
 467                         DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
 468                         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
 469                         return rc;
 470                 }
 471
 472                 pd->uctx = context;
 473                 pd->uctx->pd = pd;
 474         }
 475
 476         return 0;
 477 }
 478
 479 int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 480 {
 481         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
 482         struct qedr_pd *pd = get_qedr_pd(ibpd);
 483
 484         DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
 485         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
 486         return 0;
 487 }
 488
 489
 490 int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
 491 {
 492         struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
 493         struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd);
 494
 495         return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id);
 496 }
 497
 498 int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
 499 {
 500         struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
 501         u16 xrcd_id = get_qedr_xrcd(ibxrcd)->xrcd_id;
 502
 503         dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id);
 504         return 0;
 505 }
 506 static void qedr_free_pbl(struct qedr_dev *dev,
 507                           struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
 508 {
 509         struct pci_dev *pdev = dev->pdev;
 510         int i;
 511
 512         for (i = 0; i < pbl_info->num_pbls; i++) {
 513                 if (!pbl[i].va)
 514                         continue;
 515                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 516                                   pbl[i].va, pbl[i].pa);
 517         }
 518
 519         kfree(pbl);
 520 }
 521
 522 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
 523 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
 524
 525 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
 526 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
 527 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
 528
 529 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
 530                                            struct qedr_pbl_info *pbl_info,
 531                                            gfp_t flags)
 532 {
 533         struct pci_dev *pdev = dev->pdev;
 534         struct qedr_pbl *pbl_table;
 535         dma_addr_t *pbl_main_tbl;
 536         dma_addr_t pa;
 537         void *va;
 538         int i;
 539
 540         pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
 541         if (!pbl_table)
 542                 return ERR_PTR(-ENOMEM);
 543
 544         for (i = 0; i < pbl_info->num_pbls; i++) {
 545                 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
 546                                         flags);
 547                 if (!va)
 548                         goto err;
 549
 550                 pbl_table[i].va = va;
 551                 pbl_table[i].pa = pa;
 552         }
 553
 554         /* Two-Layer PBLs, if we have more than one pbl we need to initialize
 555          * the first one with physical pointers to all of the rest
 556          */
 557         pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
 558         for (i = 0; i < pbl_info->num_pbls - 1; i++)
 559                 pbl_main_tbl[i] = pbl_table[i + 1].pa;
 560
 561         return pbl_table;
 562
 563 err:
 564         for (i--; i >= 0; i--)
 565                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 566                                   pbl_table[i].va, pbl_table[i].pa);
 567
 568         qedr_free_pbl(dev, pbl_info, pbl_table);
 569
 570         return ERR_PTR(-ENOMEM);
 571 }
 572
 573 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
 574                                 struct qedr_pbl_info *pbl_info,
 575                                 u32 num_pbes, int two_layer_capable)
 576 {
 577         u32 pbl_capacity;
 578         u32 pbl_size;
 579         u32 num_pbls;
 580
 581         if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
 582                 if (num_pbes > MAX_PBES_TWO_LAYER) {
 583                         DP_ERR(dev, "prepare pbl table: too many pages %d\n",
 584                                num_pbes);
 585                         return -EINVAL;
 586                 }
 587
 588                 /* calculate required pbl page size */
 589                 pbl_size = MIN_FW_PBL_PAGE_SIZE;
 590                 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
 591                                NUM_PBES_ON_PAGE(pbl_size);
 592
 593                 while (pbl_capacity < num_pbes) {
 594                         pbl_size *= 2;
 595                         pbl_capacity = pbl_size / sizeof(u64);
 596                         pbl_capacity = pbl_capacity * pbl_capacity;
 597                 }
 598
 599                 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
 600                 num_pbls++;     /* One for the layer0 ( points to the pbls) */
 601                 pbl_info->two_layered = true;
 602         } else {
 603                 /* One layered PBL */
 604                 num_pbls = 1;
 605                 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
 606                                  roundup_pow_of_two((num_pbes * sizeof(u64))));
 607                 pbl_info->two_layered = false;
 608         }
 609
 610         pbl_info->num_pbls = num_pbls;
 611         pbl_info->pbl_size = pbl_size;
 612         pbl_info->num_pbes = num_pbes;
 613
 614         DP_DEBUG(dev, QEDR_MSG_MR,
 615                  "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
 616                  pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
 617
 618         return 0;
 619 }
 620
 621 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
 622                                struct qedr_pbl *pbl,
 623                                struct qedr_pbl_info *pbl_info, u32 pg_shift)
 624 {
 625         int pbe_cnt, total_num_pbes = 0;
 626         struct qedr_pbl *pbl_tbl;
 627         struct ib_block_iter biter;
 628         struct regpair *pbe;
 629
 630         if (!pbl_info->num_pbes)
 631                 return;
 632
 633         /* If we have a two layered pbl, the first pbl points to the rest
 634          * of the pbls and the first entry lays on the second pbl in the table
 635          */
 636         if (pbl_info->two_layered)
 637                 pbl_tbl = &pbl[1];
 638         else
 639                 pbl_tbl = pbl;
 640
 641         pbe = (struct regpair *)pbl_tbl->va;
 642         if (!pbe) {
 643                 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
 644                 return;
 645         }
 646
 647         pbe_cnt = 0;
 648
 649         rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) {
 650                 u64 pg_addr = rdma_block_iter_dma_address(&biter);
 651
 652                 pbe->lo = cpu_to_le32(pg_addr);
 653                 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
 654
 655                 pbe_cnt++;
 656                 total_num_pbes++;
 657                 pbe++;
 658
 659                 if (total_num_pbes == pbl_info->num_pbes)
 660                         return;
 661
 662                 /* If the given pbl is full storing the pbes, move to next pbl.
 663                  */
 664                 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
 665                         pbl_tbl++;
 666                         pbe = (struct regpair *)pbl_tbl->va;
 667                         pbe_cnt = 0;
 668                 }
 669         }
 670 }
 671
 672 static int qedr_db_recovery_add(struct qedr_dev *dev,
 673                                 void __iomem *db_addr,
 674                                 void *db_data,
 675                                 enum qed_db_rec_width db_width,
 676                                 enum qed_db_rec_space db_space)
 677 {
 678         if (!db_data) {
 679                 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
 680                 return 0;
 681         }
 682
 683         return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
 684                                                  db_width, db_space);
 685 }
 686
 687 static void qedr_db_recovery_del(struct qedr_dev *dev,
 688                                  void __iomem *db_addr,
 689                                  void *db_data)
 690 {
 691         if (!db_data) {
 692                 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
 693                 return;
 694         }
 695
 696         /* Ignore return code as there is not much we can do about it. Error
 697          * log will be printed inside.
 698          */
 699         dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
 700 }
 701
 702 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
 703                               struct qedr_cq *cq, struct ib_udata *udata,
 704                               u32 db_offset)
 705 {
 706         struct qedr_create_cq_uresp uresp;
 707         int rc;
 708
 709         memset(&uresp, 0, sizeof(uresp));
 710
 711         uresp.db_offset = db_offset;
 712         uresp.icid = cq->icid;
 713         if (cq->q.db_mmap_entry)
 714                 uresp.db_rec_addr =
 715                         rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
 716
 717         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 718         if (rc)
 719                 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
 720
 721         return rc;
 722 }
 723
 724 static void consume_cqe(struct qedr_cq *cq)
 725 {
 726         if (cq->latest_cqe == cq->toggle_cqe)
 727                 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 728
 729         cq->latest_cqe = qed_chain_consume(&cq->pbl);
 730 }
 731
 732 static inline int qedr_align_cq_entries(int entries)
 733 {
 734         u64 size, aligned_size;
 735
 736         /* We allocate an extra entry that we don't report to the FW. */
 737         size = (entries + 1) * QEDR_CQE_SIZE;
 738         aligned_size = ALIGN(size, PAGE_SIZE);
 739
 740         return aligned_size / QEDR_CQE_SIZE;
 741 }
 742
 743 static int qedr_init_user_db_rec(struct ib_udata *udata,
 744                                  struct qedr_dev *dev, struct qedr_userq *q,
 745                                  bool requires_db_rec)
 746 {
 747         struct qedr_ucontext *uctx =
 748                 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
 749                                           ibucontext);
 750         struct qedr_user_mmap_entry *entry;
 751         int rc;
 752
 753         /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
 754         if (requires_db_rec == 0 || !uctx->db_rec)
 755                 return 0;
 756
 757         /* Allocate a page for doorbell recovery, add to mmap */
 758         q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
 759         if (!q->db_rec_data) {
 760                 DP_ERR(dev, "get_zeroed_page failed\n");
 761                 return -ENOMEM;
 762         }
 763
 764         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 765         if (!entry)
 766                 goto err_free_db_data;
 767
 768         entry->address = q->db_rec_data;
 769         entry->length = PAGE_SIZE;
 770         entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
 771         rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,
 772                                          &entry->rdma_entry,
 773                                          PAGE_SIZE);
 774         if (rc)
 775                 goto err_free_entry;
 776
 777         q->db_mmap_entry = &entry->rdma_entry;
 778
 779         return 0;
 780
 781 err_free_entry:
 782         kfree(entry);
 783
 784 err_free_db_data:
 785         free_page((unsigned long)q->db_rec_data);
 786         q->db_rec_data = NULL;
 787         return -ENOMEM;
 788 }
 789
 790 static inline int qedr_init_user_queue(struct ib_udata *udata,
 791                                        struct qedr_dev *dev,
 792                                        struct qedr_userq *q, u64 buf_addr,
 793                                        size_t buf_len, bool requires_db_rec,
 794                                        int access,
 795                                        int alloc_and_init)
 796 {
 797         u32 fw_pages;
 798         int rc;
 799
 800         q->buf_addr = buf_addr;
 801         q->buf_len = buf_len;
 802         q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
 803         if (IS_ERR(q->umem)) {
 804                 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
 805                        PTR_ERR(q->umem));
 806                 return PTR_ERR(q->umem);
 807         }
 808
 809         fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT);
 810         rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
 811         if (rc)
 812                 goto err0;
 813
 814         if (alloc_and_init) {
 815                 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
 816                 if (IS_ERR(q->pbl_tbl)) {
 817                         rc = PTR_ERR(q->pbl_tbl);
 818                         goto err0;
 819                 }
 820                 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
 821                                    FW_PAGE_SHIFT);
 822         } else {
 823                 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
 824                 if (!q->pbl_tbl) {
 825                         rc = -ENOMEM;
 826                         goto err0;
 827                 }
 828         }
 829
 830         /* mmap the user address used to store doorbell data for recovery */
 831         return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
 832
 833 err0:
 834         ib_umem_release(q->umem);
 835         q->umem = NULL;
 836
 837         return rc;
 838 }
 839
 840 static inline void qedr_init_cq_params(struct qedr_cq *cq,
 841                                        struct qedr_ucontext *ctx,
 842                                        struct qedr_dev *dev, int vector,
 843                                        int chain_entries, int page_cnt,
 844                                        u64 pbl_ptr,
 845                                        struct qed_rdma_create_cq_in_params
 846                                        *params)
 847 {
 848         memset(params, 0, sizeof(*params));
 849         params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
 850         params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
 851         params->cnq_id = vector;
 852         params->cq_size = chain_entries - 1;
 853         params->dpi = (ctx) ? ctx->dpi : dev->dpi;
 854         params->pbl_num_pages = page_cnt;
 855         params->pbl_ptr = pbl_ptr;
 856         params->pbl_two_level = 0;
 857 }
 858
 859 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
 860 {
 861         cq->db.data.agg_flags = flags;
 862         cq->db.data.value = cpu_to_le32(cons);
 863         writeq(cq->db.raw, cq->db_addr);
 864 }
 865
 866 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 867 {
 868         struct qedr_cq *cq = get_qedr_cq(ibcq);
 869         unsigned long sflags;
 870         struct qedr_dev *dev;
 871
 872         dev = get_qedr_dev(ibcq->device);
 873
 874         if (cq->destroyed) {
 875                 DP_ERR(dev,
 876                        "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
 877                        cq, cq->icid);
 878                 return -EINVAL;
 879         }
 880
 881
 882         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
 883                 return 0;
 884
 885         spin_lock_irqsave(&cq->cq_lock, sflags);
 886
 887         cq->arm_flags = 0;
 888
 889         if (flags & IB_CQ_SOLICITED)
 890                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
 891
 892         if (flags & IB_CQ_NEXT_COMP)
 893                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
 894
 895         doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
 896
 897         spin_unlock_irqrestore(&cq->cq_lock, sflags);
 898
 899         return 0;
 900 }
 901
 902 int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 903                    struct ib_udata *udata)
 904 {
 905         struct ib_device *ibdev = ibcq->device;
 906         struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
 907                 udata, struct qedr_ucontext, ibucontext);
 908         struct qed_rdma_destroy_cq_out_params destroy_oparams;
 909         struct qed_rdma_destroy_cq_in_params destroy_iparams;
 910         struct qed_chain_init_params chain_params = {
 911                 .mode           = QED_CHAIN_MODE_PBL,
 912                 .intended_use   = QED_CHAIN_USE_TO_CONSUME,
 913                 .cnt_type       = QED_CHAIN_CNT_TYPE_U32,
 914                 .elem_size      = sizeof(union rdma_cqe),
 915         };
 916         struct qedr_dev *dev = get_qedr_dev(ibdev);
 917         struct qed_rdma_create_cq_in_params params;
 918         struct qedr_create_cq_ureq ureq = {};
 919         int vector = attr->comp_vector;
 920         int entries = attr->cqe;
 921         struct qedr_cq *cq = get_qedr_cq(ibcq);
 922         int chain_entries;
 923         u32 db_offset;
 924         int page_cnt;
 925         u64 pbl_ptr;
 926         u16 icid;
 927         int rc;
 928
 929         DP_DEBUG(dev, QEDR_MSG_INIT,
 930                  "create_cq: called from %s. entries=%d, vector=%d\n",
 931                  udata ? "User Lib" : "Kernel", entries, vector);
 932
 933         if (attr->flags)
 934                 return -EOPNOTSUPP;
 935
 936         if (entries > QEDR_MAX_CQES) {
 937                 DP_ERR(dev,
 938                        "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
 939                        entries, QEDR_MAX_CQES);
 940                 return -EINVAL;
 941         }
 942
 943         chain_entries = qedr_align_cq_entries(entries);
 944         chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
 945         chain_params.num_elems = chain_entries;
 946
 947         /* calc db offset. user will add DPI base, kernel will add db addr */
 948         db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 949
 950         if (udata) {
 951                 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
 952                                                          udata->inlen))) {
 953                         DP_ERR(dev,
 954                                "create cq: problem copying data from user space\n");
 955                         goto err0;
 956                 }
 957
 958                 if (!ureq.len) {
 959                         DP_ERR(dev,
 960                                "create cq: cannot create a cq with 0 entries\n");
 961                         goto err0;
 962                 }
 963
 964                 cq->cq_type = QEDR_CQ_TYPE_USER;
 965
 966                 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
 967                                           ureq.len, true, IB_ACCESS_LOCAL_WRITE,
 968                                           1);
 969                 if (rc)
 970                         goto err0;
 971
 972                 pbl_ptr = cq->q.pbl_tbl->pa;
 973                 page_cnt = cq->q.pbl_info.num_pbes;
 974
 975                 cq->ibcq.cqe = chain_entries;
 976                 cq->q.db_addr = ctx->dpi_addr + db_offset;
 977         } else {
 978                 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
 979
 980                 rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl,
 981                                                    &chain_params);
 982                 if (rc)
 983                         goto err0;
 984
 985                 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
 986                 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
 987                 cq->ibcq.cqe = cq->pbl.capacity;
 988         }
 989
 990         qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
 991                             pbl_ptr, &params);
 992
 993         rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
 994         if (rc)
 995                 goto err1;
 996
 997         cq->icid = icid;
 998         cq->sig = QEDR_CQ_MAGIC_NUMBER;
 999         spin_lock_init(&cq->cq_lock);
1000
1001         if (udata) {
1002                 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
1003                 if (rc)
1004                         goto err2;
1005
1006                 rc = qedr_db_recovery_add(dev, cq->q.db_addr,
1007                                           &cq->q.db_rec_data->db_data,
1008                                           DB_REC_WIDTH_64B,
1009                                           DB_REC_USER);
1010                 if (rc)
1011                         goto err2;
1012
1013         } else {
1014                 /* Generate doorbell address. */
1015                 cq->db.data.icid = cq->icid;
1016                 cq->db_addr = dev->db_addr + db_offset;
1017                 cq->db.data.params = DB_AGG_CMD_MAX <<
1018                     RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1019
1020                 /* point to the very last element, passing it we will toggle */
1021                 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
1022                 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1023                 cq->latest_cqe = NULL;
1024                 consume_cqe(cq);
1025                 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
1026
1027                 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
1028                                           DB_REC_WIDTH_64B, DB_REC_KERNEL);
1029                 if (rc)
1030                         goto err2;
1031         }
1032
1033         DP_DEBUG(dev, QEDR_MSG_CQ,
1034                  "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1035                  cq->icid, cq, params.cq_size);
1036
1037         return 0;
1038
1039 err2:
1040         destroy_iparams.icid = cq->icid;
1041         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1042                                   &destroy_oparams);
1043 err1:
1044         if (udata) {
1045                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1046                 ib_umem_release(cq->q.umem);
1047                 if (cq->q.db_mmap_entry)
1048                         rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1049         } else {
1050                 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1051         }
1052 err0:
1053         return -EINVAL;
1054 }
1055
1056 #define QEDR_DESTROY_CQ_MAX_ITERATIONS          (10)
1057 #define QEDR_DESTROY_CQ_ITER_DURATION           (10)
1058
1059 int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1060 {
1061         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1062         struct qed_rdma_destroy_cq_out_params oparams;
1063         struct qed_rdma_destroy_cq_in_params iparams;
1064         struct qedr_cq *cq = get_qedr_cq(ibcq);
1065         int iter;
1066
1067         DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1068
1069         cq->destroyed = 1;
1070
1071         /* GSIs CQs are handled by driver, so they don't exist in the FW */
1072         if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1073                 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1074                 return 0;
1075         }
1076
1077         iparams.icid = cq->icid;
1078         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1079         dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1080
1081         if (udata) {
1082                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1083                 ib_umem_release(cq->q.umem);
1084
1085                 if (cq->q.db_rec_data) {
1086                         qedr_db_recovery_del(dev, cq->q.db_addr,
1087                                              &cq->q.db_rec_data->db_data);
1088                         rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1089                 }
1090         } else {
1091                 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1092         }
1093
1094         /* We don't want the IRQ handler to handle a non-existing CQ so we
1095          * wait until all CNQ interrupts, if any, are received. This will always
1096          * happen and will always happen very fast. If not, then a serious error
1097          * has occured. That is why we can use a long delay.
1098          * We spin for a short time so we don’t lose time on context switching
1099          * in case all the completions are handled in that span. Otherwise
1100          * we sleep for a while and check again. Since the CNQ may be
1101          * associated with (only) the current CPU we use msleep to allow the
1102          * current CPU to be freed.
1103          * The CNQ notification is increased in qedr_irq_handler().
1104          */
1105         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1106         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1107                 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1108                 iter--;
1109         }
1110
1111         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1112         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1113                 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1114                 iter--;
1115         }
1116
1117         /* Note that we don't need to have explicit code to wait for the
1118          * completion of the event handler because it is invoked from the EQ.
1119          * Since the destroy CQ ramrod has also been received on the EQ we can
1120          * be certain that there's no event handler in process.
1121          */
1122         return 0;
1123 }
1124
1125 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1126                                           struct ib_qp_attr *attr,
1127                                           int attr_mask,
1128                                           struct qed_rdma_modify_qp_in_params
1129                                           *qp_params)
1130 {
1131         const struct ib_gid_attr *gid_attr;
1132         enum rdma_network_type nw_type;
1133         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1134         u32 ipv4_addr;
1135         int ret;
1136         int i;
1137
1138         gid_attr = grh->sgid_attr;
1139         ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1140         if (ret)
1141                 return ret;
1142
1143         nw_type = rdma_gid_attr_network_type(gid_attr);
1144         switch (nw_type) {
1145         case RDMA_NETWORK_IPV6:
1146                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1147                        sizeof(qp_params->sgid));
1148                 memcpy(&qp_params->dgid.bytes[0],
1149                        &grh->dgid,
1150                        sizeof(qp_params->dgid));
1151                 qp_params->roce_mode = ROCE_V2_IPV6;
1152                 SET_FIELD(qp_params->modify_flags,
1153                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1154                 break;
1155         case RDMA_NETWORK_ROCE_V1:
1156                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1157                        sizeof(qp_params->sgid));
1158                 memcpy(&qp_params->dgid.bytes[0],
1159                        &grh->dgid,
1160                        sizeof(qp_params->dgid));
1161                 qp_params->roce_mode = ROCE_V1;
1162                 break;
1163         case RDMA_NETWORK_IPV4:
1164                 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1165                 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1166                 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1167                 qp_params->sgid.ipv4_addr = ipv4_addr;
1168                 ipv4_addr =
1169                     qedr_get_ipv4_from_gid(grh->dgid.raw);
1170                 qp_params->dgid.ipv4_addr = ipv4_addr;
1171                 SET_FIELD(qp_params->modify_flags,
1172                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1173                 qp_params->roce_mode = ROCE_V2_IPV4;
1174                 break;
1175         default:
1176                 return -EINVAL;
1177         }
1178
1179         for (i = 0; i < 4; i++) {
1180                 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1181                 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1182         }
1183
1184         if (qp_params->vlan_id >= VLAN_CFI_MASK)
1185                 qp_params->vlan_id = 0;
1186
1187         return 0;
1188 }
1189
1190 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1191                                struct ib_qp_init_attr *attrs,
1192                                struct ib_udata *udata)
1193 {
1194         struct qedr_device_attr *qattr = &dev->attr;
1195
1196         /* QP0... attrs->qp_type == IB_QPT_GSI */
1197         if (attrs->qp_type != IB_QPT_RC &&
1198             attrs->qp_type != IB_QPT_GSI &&
1199             attrs->qp_type != IB_QPT_XRC_INI &&
1200             attrs->qp_type != IB_QPT_XRC_TGT) {
1201                 DP_DEBUG(dev, QEDR_MSG_QP,
1202                          "create qp: unsupported qp type=0x%x requested\n",
1203                          attrs->qp_type);
1204                 return -EOPNOTSUPP;
1205         }
1206
1207         if (attrs->cap.max_send_wr > qattr->max_sqe) {
1208                 DP_ERR(dev,
1209                        "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1210                        attrs->cap.max_send_wr, qattr->max_sqe);
1211                 return -EINVAL;
1212         }
1213
1214         if (attrs->cap.max_inline_data > qattr->max_inline) {
1215                 DP_ERR(dev,
1216                        "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1217                        attrs->cap.max_inline_data, qattr->max_inline);
1218                 return -EINVAL;
1219         }
1220
1221         if (attrs->cap.max_send_sge > qattr->max_sge) {
1222                 DP_ERR(dev,
1223                        "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1224                        attrs->cap.max_send_sge, qattr->max_sge);
1225                 return -EINVAL;
1226         }
1227
1228         if (attrs->cap.max_recv_sge > qattr->max_sge) {
1229                 DP_ERR(dev,
1230                        "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1231                        attrs->cap.max_recv_sge, qattr->max_sge);
1232                 return -EINVAL;
1233         }
1234
1235         /* verify consumer QPs are not trying to use GSI QP's CQ.
1236          * TGT QP isn't associated with RQ/SQ
1237          */
1238         if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) &&
1239             (attrs->qp_type != IB_QPT_XRC_TGT) &&
1240             (attrs->qp_type != IB_QPT_XRC_INI)) {
1241                 struct qedr_cq *send_cq = get_qedr_cq(attrs->send_cq);
1242                 struct qedr_cq *recv_cq = get_qedr_cq(attrs->recv_cq);
1243
1244                 if ((send_cq->cq_type == QEDR_CQ_TYPE_GSI) ||
1245                     (recv_cq->cq_type == QEDR_CQ_TYPE_GSI)) {
1246                         DP_ERR(dev,
1247                                "create qp: consumer QP cannot use GSI CQs.\n");
1248                         return -EINVAL;
1249                 }
1250         }
1251
1252         return 0;
1253 }
1254
1255 static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1256                                struct qedr_srq *srq, struct ib_udata *udata)
1257 {
1258         struct qedr_create_srq_uresp uresp = {};
1259         int rc;
1260
1261         uresp.srq_id = srq->srq_id;
1262
1263         rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1264         if (rc)
1265                 DP_ERR(dev, "create srq: problem copying data to user space\n");
1266
1267         return rc;
1268 }
1269
1270 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1271                                struct qedr_create_qp_uresp *uresp,
1272                                struct qedr_qp *qp)
1273 {
1274         /* iWARP requires two doorbells per RQ. */
1275         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1276                 uresp->rq_db_offset =
1277                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1278                 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1279         } else {
1280                 uresp->rq_db_offset =
1281                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1282         }
1283
1284         uresp->rq_icid = qp->icid;
1285         if (qp->urq.db_mmap_entry)
1286                 uresp->rq_db_rec_addr =
1287                         rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
1288 }
1289
1290 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1291                                struct qedr_create_qp_uresp *uresp,
1292                                struct qedr_qp *qp)
1293 {
1294         uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1295
1296         /* iWARP uses the same cid for rq and sq */
1297         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1298                 uresp->sq_icid = qp->icid;
1299         else
1300                 uresp->sq_icid = qp->icid + 1;
1301
1302         if (qp->usq.db_mmap_entry)
1303                 uresp->sq_db_rec_addr =
1304                         rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
1305 }
1306
1307 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1308                               struct qedr_qp *qp, struct ib_udata *udata,
1309                               struct qedr_create_qp_uresp *uresp)
1310 {
1311         int rc;
1312
1313         memset(uresp, 0, sizeof(*uresp));
1314
1315         if (qedr_qp_has_sq(qp))
1316                 qedr_copy_sq_uresp(dev, uresp, qp);
1317
1318         if (qedr_qp_has_rq(qp))
1319                 qedr_copy_rq_uresp(dev, uresp, qp);
1320
1321         uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1322         uresp->qp_id = qp->qp_id;
1323
1324         rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
1325         if (rc)
1326                 DP_ERR(dev,
1327                        "create qp: failed a copy to user space with qp icid=0x%x.\n",
1328                        qp->icid);
1329
1330         return rc;
1331 }
1332
1333 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1334 {
1335         qed_chain_reset(&qph->pbl);
1336         qph->prod = 0;
1337         qph->cons = 0;
1338         qph->wqe_cons = 0;
1339         qph->db_data.data.value = cpu_to_le16(0);
1340 }
1341
1342 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1343                                       struct qedr_qp *qp,
1344                                       struct qedr_pd *pd,
1345                                       struct ib_qp_init_attr *attrs)
1346 {
1347         spin_lock_init(&qp->q_lock);
1348         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1349                 kref_init(&qp->refcnt);
1350                 init_completion(&qp->iwarp_cm_comp);
1351                 init_completion(&qp->qp_rel_comp);
1352         }
1353
1354         qp->pd = pd;
1355         qp->qp_type = attrs->qp_type;
1356         qp->max_inline_data = attrs->cap.max_inline_data;
1357         qp->state = QED_ROCE_QP_STATE_RESET;
1358
1359         qp->prev_wqe_size = 0;
1360
1361         qp->signaled = attrs->sq_sig_type == IB_SIGNAL_ALL_WR;
1362         qp->dev = dev;
1363         if (qedr_qp_has_sq(qp)) {
1364                 qedr_reset_qp_hwq_info(&qp->sq);
1365                 qp->sq.max_sges = attrs->cap.max_send_sge;
1366                 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1367                 DP_DEBUG(dev, QEDR_MSG_QP,
1368                          "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1369                          qp->sq.max_sges, qp->sq_cq->icid);
1370         }
1371
1372         if (attrs->srq)
1373                 qp->srq = get_qedr_srq(attrs->srq);
1374
1375         if (qedr_qp_has_rq(qp)) {
1376                 qedr_reset_qp_hwq_info(&qp->rq);
1377                 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1378                 qp->rq.max_sges = attrs->cap.max_recv_sge;
1379                 DP_DEBUG(dev, QEDR_MSG_QP,
1380                          "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1381                          qp->rq.max_sges, qp->rq_cq->icid);
1382         }
1383
1384         DP_DEBUG(dev, QEDR_MSG_QP,
1385                  "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1386                  pd->pd_id, qp->qp_type, qp->max_inline_data,
1387                  qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1388         DP_DEBUG(dev, QEDR_MSG_QP,
1389                  "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1390                  qp->sq.max_sges, qp->sq_cq->icid);
1391 }
1392
1393 static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1394 {
1395         int rc = 0;
1396
1397         if (qedr_qp_has_sq(qp)) {
1398                 qp->sq.db = dev->db_addr +
1399                             DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1400                 qp->sq.db_data.data.icid = qp->icid + 1;
1401                 rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data,
1402                                           DB_REC_WIDTH_32B, DB_REC_KERNEL);
1403                 if (rc)
1404                         return rc;
1405         }
1406
1407         if (qedr_qp_has_rq(qp)) {
1408                 qp->rq.db = dev->db_addr +
1409                             DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1410                 qp->rq.db_data.data.icid = qp->icid;
1411                 rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data,
1412                                           DB_REC_WIDTH_32B, DB_REC_KERNEL);
1413                 if (rc && qedr_qp_has_sq(qp))
1414                         qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
1415         }
1416
1417         return rc;
1418 }
1419
1420 static int qedr_check_srq_params(struct qedr_dev *dev,
1421                                  struct ib_srq_init_attr *attrs,
1422                                  struct ib_udata *udata)
1423 {
1424         struct qedr_device_attr *qattr = &dev->attr;
1425
1426         if (attrs->attr.max_wr > qattr->max_srq_wr) {
1427                 DP_ERR(dev,
1428                        "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1429                        attrs->attr.max_wr, qattr->max_srq_wr);
1430                 return -EINVAL;
1431         }
1432
1433         if (attrs->attr.max_sge > qattr->max_sge) {
1434                 DP_ERR(dev,
1435                        "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1436                        attrs->attr.max_sge, qattr->max_sge);
1437         }
1438
1439         if (!udata && attrs->srq_type == IB_SRQT_XRC) {
1440                 DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n");
1441                 return -EINVAL;
1442         }
1443
1444         return 0;
1445 }
1446
1447 static void qedr_free_srq_user_params(struct qedr_srq *srq)
1448 {
1449         qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1450         ib_umem_release(srq->usrq.umem);
1451         ib_umem_release(srq->prod_umem);
1452 }
1453
1454 static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1455 {
1456         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1457         struct qedr_dev *dev = srq->dev;
1458
1459         dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1460
1461         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1462                           hw_srq->virt_prod_pair_addr,
1463                           hw_srq->phy_prod_pair_addr);
1464 }
1465
1466 static int qedr_init_srq_user_params(struct ib_udata *udata,
1467                                      struct qedr_srq *srq,
1468                                      struct qedr_create_srq_ureq *ureq,
1469                                      int access)
1470 {
1471         struct scatterlist *sg;
1472         int rc;
1473
1474         rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1475                                   ureq->srq_len, false, access, 1);
1476         if (rc)
1477                 return rc;
1478
1479         srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,
1480                                      sizeof(struct rdma_srq_producers), access);
1481         if (IS_ERR(srq->prod_umem)) {
1482                 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1483                 ib_umem_release(srq->usrq.umem);
1484                 DP_ERR(srq->dev,
1485                        "create srq: failed ib_umem_get for producer, got %ld\n",
1486                        PTR_ERR(srq->prod_umem));
1487                 return PTR_ERR(srq->prod_umem);
1488         }
1489
1490         sg = srq->prod_umem->sgt_append.sgt.sgl;
1491         srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1492
1493         return 0;
1494 }
1495
1496 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1497                                         struct qedr_dev *dev,
1498                                         struct ib_srq_init_attr *init_attr)
1499 {
1500         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1501         struct qed_chain_init_params params = {
1502                 .mode           = QED_CHAIN_MODE_PBL,
1503                 .intended_use   = QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1504                 .cnt_type       = QED_CHAIN_CNT_TYPE_U32,
1505                 .elem_size      = QEDR_SRQ_WQE_ELEM_SIZE,
1506         };
1507         dma_addr_t phy_prod_pair_addr;
1508         u32 num_elems;
1509         void *va;
1510         int rc;
1511
1512         va = dma_alloc_coherent(&dev->pdev->dev,
1513                                 sizeof(struct rdma_srq_producers),
1514                                 &phy_prod_pair_addr, GFP_KERNEL);
1515         if (!va) {
1516                 DP_ERR(dev,
1517                        "create srq: failed to allocate dma memory for producer\n");
1518                 return -ENOMEM;
1519         }
1520
1521         hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1522         hw_srq->virt_prod_pair_addr = va;
1523
1524         num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1525         params.num_elems = num_elems;
1526
1527         rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, &params);
1528         if (rc)
1529                 goto err0;
1530
1531         hw_srq->num_elems = num_elems;
1532
1533         return 0;
1534
1535 err0:
1536         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1537                           va, phy_prod_pair_addr);
1538         return rc;
1539 }
1540
1541 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1542                     struct ib_udata *udata)
1543 {
1544         struct qed_rdma_destroy_srq_in_params destroy_in_params;
1545         struct qed_rdma_create_srq_in_params in_params = {};
1546         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1547         struct qed_rdma_create_srq_out_params out_params;
1548         struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
1549         struct qedr_create_srq_ureq ureq = {};
1550         u64 pbl_base_addr, phy_prod_pair_addr;
1551         struct qedr_srq_hwq_info *hw_srq;
1552         u32 page_cnt, page_size;
1553         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1554         int rc = 0;
1555
1556         DP_DEBUG(dev, QEDR_MSG_QP,
1557                  "create SRQ called from %s (pd %p)\n",
1558                  (udata) ? "User lib" : "kernel", pd);
1559
1560         if (init_attr->srq_type != IB_SRQT_BASIC &&
1561             init_attr->srq_type != IB_SRQT_XRC)
1562                 return -EOPNOTSUPP;
1563
1564         rc = qedr_check_srq_params(dev, init_attr, udata);
1565         if (rc)
1566                 return -EINVAL;
1567
1568         srq->dev = dev;
1569         srq->is_xrc = (init_attr->srq_type == IB_SRQT_XRC);
1570         hw_srq = &srq->hw_srq;
1571         spin_lock_init(&srq->lock);
1572
1573         hw_srq->max_wr = init_attr->attr.max_wr;
1574         hw_srq->max_sges = init_attr->attr.max_sge;
1575
1576         if (udata) {
1577                 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1578                                                          udata->inlen))) {
1579                         DP_ERR(dev,
1580                                "create srq: problem copying data from user space\n");
1581                         goto err0;
1582                 }
1583
1584                 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0);
1585                 if (rc)
1586                         goto err0;
1587
1588                 page_cnt = srq->usrq.pbl_info.num_pbes;
1589                 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1590                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1591                 page_size = PAGE_SIZE;
1592         } else {
1593                 struct qed_chain *pbl;
1594
1595                 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1596                 if (rc)
1597                         goto err0;
1598
1599                 pbl = &hw_srq->pbl;
1600                 page_cnt = qed_chain_get_page_cnt(pbl);
1601                 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1602                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1603                 page_size = QED_CHAIN_PAGE_SIZE;
1604         }
1605
1606         in_params.pd_id = pd->pd_id;
1607         in_params.pbl_base_addr = pbl_base_addr;
1608         in_params.prod_pair_addr = phy_prod_pair_addr;
1609         in_params.num_pages = page_cnt;
1610         in_params.page_size = page_size;
1611         if (srq->is_xrc) {
1612                 struct qedr_xrcd *xrcd = get_qedr_xrcd(init_attr->ext.xrc.xrcd);
1613                 struct qedr_cq *cq = get_qedr_cq(init_attr->ext.cq);
1614
1615                 in_params.is_xrc = 1;
1616                 in_params.xrcd_id = xrcd->xrcd_id;
1617                 in_params.cq_cid = cq->icid;
1618         }
1619
1620         rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1621         if (rc)
1622                 goto err1;
1623
1624         srq->srq_id = out_params.srq_id;
1625
1626         if (udata) {
1627                 rc = qedr_copy_srq_uresp(dev, srq, udata);
1628                 if (rc)
1629                         goto err2;
1630         }
1631
1632         rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
1633         if (rc)
1634                 goto err2;
1635
1636         DP_DEBUG(dev, QEDR_MSG_SRQ,
1637                  "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1638         return 0;
1639
1640 err2:
1641         destroy_in_params.srq_id = srq->srq_id;
1642
1643         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1644 err1:
1645         if (udata)
1646                 qedr_free_srq_user_params(srq);
1647         else
1648                 qedr_free_srq_kernel_params(srq);
1649 err0:
1650         return -EFAULT;
1651 }
1652
1653 int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1654 {
1655         struct qed_rdma_destroy_srq_in_params in_params = {};
1656         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1657         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1658
1659         xa_erase_irq(&dev->srqs, srq->srq_id);
1660         in_params.srq_id = srq->srq_id;
1661         in_params.is_xrc = srq->is_xrc;
1662         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1663
1664         if (ibsrq->uobject)
1665                 qedr_free_srq_user_params(srq);
1666         else
1667                 qedr_free_srq_kernel_params(srq);
1668
1669         DP_DEBUG(dev, QEDR_MSG_SRQ,
1670                  "destroy srq: destroyed srq with srq_id=0x%0x\n",
1671                  srq->srq_id);
1672         return 0;
1673 }
1674
1675 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1676                     enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1677 {
1678         struct qed_rdma_modify_srq_in_params in_params = {};
1679         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1680         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1681         int rc;
1682
1683         if (attr_mask & IB_SRQ_MAX_WR) {
1684                 DP_ERR(dev,
1685                        "modify srq: invalid attribute mask=0x%x specified for %p\n",
1686                        attr_mask, srq);
1687                 return -EINVAL;
1688         }
1689
1690         if (attr_mask & IB_SRQ_LIMIT) {
1691                 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1692                         DP_ERR(dev,
1693                                "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1694                                attr->srq_limit, srq->hw_srq.max_wr);
1695                         return -EINVAL;
1696                 }
1697
1698                 in_params.srq_id = srq->srq_id;
1699                 in_params.wqe_limit = attr->srq_limit;
1700                 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1701                 if (rc)
1702                         return rc;
1703         }
1704
1705         srq->srq_limit = attr->srq_limit;
1706
1707         DP_DEBUG(dev, QEDR_MSG_SRQ,
1708                  "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1709
1710         return 0;
1711 }
1712
1713 static enum qed_rdma_qp_type qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type)
1714 {
1715         switch (ib_qp_type) {
1716         case IB_QPT_RC:
1717                 return QED_RDMA_QP_TYPE_RC;
1718         case IB_QPT_XRC_INI:
1719                 return QED_RDMA_QP_TYPE_XRC_INI;
1720         case IB_QPT_XRC_TGT:
1721                 return QED_RDMA_QP_TYPE_XRC_TGT;
1722         default:
1723                 return QED_RDMA_QP_TYPE_INVAL;
1724         }
1725 }
1726
1727 static inline void
1728 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1729                               struct qedr_pd *pd,
1730                               struct qedr_qp *qp,
1731                               struct ib_qp_init_attr *attrs,
1732                               bool fmr_and_reserved_lkey,
1733                               struct qed_rdma_create_qp_in_params *params)
1734 {
1735         /* QP handle to be written in an async event */
1736         params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1737         params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1738
1739         params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1740         params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1741         params->qp_type = qedr_ib_to_qed_qp_type(attrs->qp_type);
1742         params->stats_queue = 0;
1743
1744         if (pd) {
1745                 params->pd = pd->pd_id;
1746                 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1747         }
1748
1749         if (qedr_qp_has_sq(qp))
1750                 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1751
1752         if (qedr_qp_has_rq(qp))
1753                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1754
1755         if (qedr_qp_has_srq(qp)) {
1756                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1757                 params->srq_id = qp->srq->srq_id;
1758                 params->use_srq = true;
1759         } else {
1760                 params->srq_id = 0;
1761                 params->use_srq = false;
1762         }
1763 }
1764
1765 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1766 {
1767         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1768                  "qp=%p. "
1769                  "sq_addr=0x%llx, "
1770                  "sq_len=%zd, "
1771                  "rq_addr=0x%llx, "
1772                  "rq_len=%zd"
1773                  "\n",
1774                  qp,
1775                  qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0,
1776                  qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0,
1777                  qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0,
1778                  qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0);
1779 }
1780
1781 static inline void
1782 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1783                             struct qedr_qp *qp,
1784                             struct qed_rdma_create_qp_out_params *out_params)
1785 {
1786         qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1787         qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1788
1789         qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1790                            &qp->usq.pbl_info, FW_PAGE_SHIFT);
1791         if (!qp->srq) {
1792                 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1793                 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1794         }
1795
1796         qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1797                            &qp->urq.pbl_info, FW_PAGE_SHIFT);
1798 }
1799
1800 static void qedr_cleanup_user(struct qedr_dev *dev,
1801                               struct qedr_ucontext *ctx,
1802                               struct qedr_qp *qp)
1803 {
1804         if (qedr_qp_has_sq(qp)) {
1805                 ib_umem_release(qp->usq.umem);
1806                 qp->usq.umem = NULL;
1807         }
1808
1809         if (qedr_qp_has_rq(qp)) {
1810                 ib_umem_release(qp->urq.umem);
1811                 qp->urq.umem = NULL;
1812         }
1813
1814         if (rdma_protocol_roce(&dev->ibdev, 1)) {
1815                 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1816                 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1817         } else {
1818                 kfree(qp->usq.pbl_tbl);
1819                 kfree(qp->urq.pbl_tbl);
1820         }
1821
1822         if (qp->usq.db_rec_data) {
1823                 qedr_db_recovery_del(dev, qp->usq.db_addr,
1824                                      &qp->usq.db_rec_data->db_data);
1825                 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1826         }
1827
1828         if (qp->urq.db_rec_data) {
1829                 qedr_db_recovery_del(dev, qp->urq.db_addr,
1830                                      &qp->urq.db_rec_data->db_data);
1831                 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1832         }
1833
1834         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1835                 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1836                                      &qp->urq.db_rec_db2_data);
1837 }
1838
1839 static int qedr_create_user_qp(struct qedr_dev *dev,
1840                                struct qedr_qp *qp,
1841                                struct ib_pd *ibpd,
1842                                struct ib_udata *udata,
1843                                struct ib_qp_init_attr *attrs)
1844 {
1845         struct qed_rdma_create_qp_in_params in_params;
1846         struct qed_rdma_create_qp_out_params out_params;
1847         struct qedr_create_qp_uresp uresp = {};
1848         struct qedr_create_qp_ureq ureq = {};
1849         int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1850         struct qedr_ucontext *ctx = NULL;
1851         struct qedr_pd *pd = NULL;
1852         int rc = 0;
1853
1854         qp->create_type = QEDR_QP_CREATE_USER;
1855
1856         if (ibpd) {
1857                 pd = get_qedr_pd(ibpd);
1858                 ctx = pd->uctx;
1859         }
1860
1861         if (udata) {
1862                 rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1863                                         udata->inlen));
1864                 if (rc) {
1865                         DP_ERR(dev, "Problem copying data from user space\n");
1866                         return rc;
1867                 }
1868         }
1869
1870         if (qedr_qp_has_sq(qp)) {
1871                 /* SQ - read access only (0) */
1872                 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1873                                           ureq.sq_len, true, 0, alloc_and_init);
1874                 if (rc)
1875                         return rc;
1876         }
1877
1878         if (qedr_qp_has_rq(qp)) {
1879                 /* RQ - read access only (0) */
1880                 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1881                                           ureq.rq_len, true, 0, alloc_and_init);
1882                 if (rc)
1883                         return rc;
1884         }
1885
1886         memset(&in_params, 0, sizeof(in_params));
1887         qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1888         in_params.qp_handle_lo = ureq.qp_handle_lo;
1889         in_params.qp_handle_hi = ureq.qp_handle_hi;
1890
1891         if (qp->qp_type == IB_QPT_XRC_TGT) {
1892                 struct qedr_xrcd *xrcd = get_qedr_xrcd(attrs->xrcd);
1893
1894                 in_params.xrcd_id = xrcd->xrcd_id;
1895                 in_params.qp_handle_lo = qp->qp_id;
1896                 in_params.use_srq = 1;
1897         }
1898
1899         if (qedr_qp_has_sq(qp)) {
1900                 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1901                 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1902         }
1903
1904         if (qedr_qp_has_rq(qp)) {
1905                 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1906                 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1907         }
1908
1909         if (ctx)
1910                 SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
1911
1912         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1913                                               &in_params, &out_params);
1914
1915         if (!qp->qed_qp) {
1916                 rc = -ENOMEM;
1917                 goto err1;
1918         }
1919
1920         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1921                 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1922
1923         qp->qp_id = out_params.qp_id;
1924         qp->icid = out_params.icid;
1925
1926         if (udata) {
1927                 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1928                 if (rc)
1929                         goto err;
1930         }
1931
1932         /* db offset was calculated in copy_qp_uresp, now set in the user q */
1933         if (qedr_qp_has_sq(qp)) {
1934                 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1935                 qp->sq.max_wr = attrs->cap.max_send_wr;
1936                 rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1937                                           &qp->usq.db_rec_data->db_data,
1938                                           DB_REC_WIDTH_32B,
1939                                           DB_REC_USER);
1940                 if (rc)
1941                         goto err;
1942         }
1943
1944         if (qedr_qp_has_rq(qp)) {
1945                 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1946                 qp->rq.max_wr = attrs->cap.max_recv_wr;
1947                 rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1948                                           &qp->urq.db_rec_data->db_data,
1949                                           DB_REC_WIDTH_32B,
1950                                           DB_REC_USER);
1951                 if (rc)
1952                         goto err;
1953         }
1954
1955         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1956                 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1957
1958                 /* calculate the db_rec_db2 data since it is constant so no
1959                  * need to reflect from user
1960                  */
1961                 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1962                 qp->urq.db_rec_db2_data.data.value =
1963                         cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1964
1965                 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1966                                           &qp->urq.db_rec_db2_data,
1967                                           DB_REC_WIDTH_32B,
1968                                           DB_REC_USER);
1969                 if (rc)
1970                         goto err;
1971         }
1972         qedr_qp_user_print(dev, qp);
1973         return rc;
1974 err:
1975         rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1976         if (rc)
1977                 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1978
1979 err1:
1980         qedr_cleanup_user(dev, ctx, qp);
1981         return rc;
1982 }
1983
1984 static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1985 {
1986         int rc;
1987
1988         qp->sq.db = dev->db_addr +
1989             DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1990         qp->sq.db_data.data.icid = qp->icid;
1991
1992         rc = qedr_db_recovery_add(dev, qp->sq.db,
1993                                   &qp->sq.db_data,
1994                                   DB_REC_WIDTH_32B,
1995                                   DB_REC_KERNEL);
1996         if (rc)
1997                 return rc;
1998
1999         qp->rq.db = dev->db_addr +
2000                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2001         qp->rq.db_data.data.icid = qp->icid;
2002         qp->rq.iwarp_db2 = dev->db_addr +
2003                            DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2004         qp->rq.iwarp_db2_data.data.icid = qp->icid;
2005         qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
2006
2007         rc = qedr_db_recovery_add(dev, qp->rq.db,
2008                                   &qp->rq.db_data,
2009                                   DB_REC_WIDTH_32B,
2010                                   DB_REC_KERNEL);
2011         if (rc)
2012                 return rc;
2013
2014         rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
2015                                   &qp->rq.iwarp_db2_data,
2016                                   DB_REC_WIDTH_32B,
2017                                   DB_REC_KERNEL);
2018         return rc;
2019 }
2020
2021 static int
2022 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
2023                            struct qedr_qp *qp,
2024                            struct qed_rdma_create_qp_in_params *in_params,
2025                            u32 n_sq_elems, u32 n_rq_elems)
2026 {
2027         struct qed_rdma_create_qp_out_params out_params;
2028         struct qed_chain_init_params params = {
2029                 .mode           = QED_CHAIN_MODE_PBL,
2030                 .cnt_type       = QED_CHAIN_CNT_TYPE_U32,
2031         };
2032         int rc;
2033
2034         params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2035         params.num_elems = n_sq_elems;
2036         params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2037
2038         rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2039         if (rc)
2040                 return rc;
2041
2042         in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
2043         in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
2044
2045         params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2046         params.num_elems = n_rq_elems;
2047         params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2048
2049         rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2050         if (rc)
2051                 return rc;
2052
2053         in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
2054         in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
2055
2056         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2057                                               in_params, &out_params);
2058
2059         if (!qp->qed_qp)
2060                 return -EINVAL;
2061
2062         qp->qp_id = out_params.qp_id;
2063         qp->icid = out_params.icid;
2064
2065         return qedr_set_roce_db_info(dev, qp);
2066 }
2067
2068 static int
2069 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
2070                             struct qedr_qp *qp,
2071                             struct qed_rdma_create_qp_in_params *in_params,
2072                             u32 n_sq_elems, u32 n_rq_elems)
2073 {
2074         struct qed_rdma_create_qp_out_params out_params;
2075         struct qed_chain_init_params params = {
2076                 .mode           = QED_CHAIN_MODE_PBL,
2077                 .cnt_type       = QED_CHAIN_CNT_TYPE_U32,
2078         };
2079         int rc;
2080
2081         in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
2082                                                      QEDR_SQE_ELEMENT_SIZE,
2083                                                      QED_CHAIN_PAGE_SIZE,
2084                                                      QED_CHAIN_MODE_PBL);
2085         in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
2086                                                      QEDR_RQE_ELEMENT_SIZE,
2087                                                      QED_CHAIN_PAGE_SIZE,
2088                                                      QED_CHAIN_MODE_PBL);
2089
2090         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2091                                               in_params, &out_params);
2092
2093         if (!qp->qed_qp)
2094                 return -EINVAL;
2095
2096         /* Now we allocate the chain */
2097
2098         params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2099         params.num_elems = n_sq_elems;
2100         params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2101         params.ext_pbl_virt = out_params.sq_pbl_virt;
2102         params.ext_pbl_phys = out_params.sq_pbl_phys;
2103
2104         rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2105         if (rc)
2106                 goto err;
2107
2108         params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2109         params.num_elems = n_rq_elems;
2110         params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2111         params.ext_pbl_virt = out_params.rq_pbl_virt;
2112         params.ext_pbl_phys = out_params.rq_pbl_phys;
2113
2114         rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2115         if (rc)
2116                 goto err;
2117
2118         qp->qp_id = out_params.qp_id;
2119         qp->icid = out_params.icid;
2120
2121         return qedr_set_iwarp_db_info(dev, qp);
2122
2123 err:
2124         dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2125
2126         return rc;
2127 }
2128
2129 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
2130 {
2131         dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2132         kfree(qp->wqe_wr_id);
2133
2134         dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2135         kfree(qp->rqe_wr_id);
2136
2137         /* GSI qp is not registered to db mechanism so no need to delete */
2138         if (qp->qp_type == IB_QPT_GSI)
2139                 return;
2140
2141         qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2142
2143         if (!qp->srq) {
2144                 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
2145
2146                 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2147                         qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2148                                              &qp->rq.iwarp_db2_data);
2149         }
2150 }
2151
2152 static int qedr_create_kernel_qp(struct qedr_dev *dev,
2153                                  struct qedr_qp *qp,
2154                                  struct ib_pd *ibpd,
2155                                  struct ib_qp_init_attr *attrs)
2156 {
2157         struct qed_rdma_create_qp_in_params in_params;
2158         struct qedr_pd *pd = get_qedr_pd(ibpd);
2159         int rc = -EINVAL;
2160         u32 n_rq_elems;
2161         u32 n_sq_elems;
2162         u32 n_sq_entries;
2163
2164         memset(&in_params, 0, sizeof(in_params));
2165         qp->create_type = QEDR_QP_CREATE_KERNEL;
2166
2167         /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2168          * the ring. The ring should allow at least a single WR, even if the
2169          * user requested none, due to allocation issues.
2170          * We should add an extra WR since the prod and cons indices of
2171          * wqe_wr_id are managed in such a way that the WQ is considered full
2172          * when (prod+1)%max_wr==cons. We currently don't do that because we
2173          * double the number of entries due an iSER issue that pushes far more
2174          * WRs than indicated. If we decline its ib_post_send() then we get
2175          * error prints in the dmesg we'd like to avoid.
2176          */
2177         qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2178                               dev->attr.max_sqe);
2179
2180         qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
2181                                 GFP_KERNEL);
2182         if (!qp->wqe_wr_id) {
2183                 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2184                 return -ENOMEM;
2185         }
2186
2187         /* QP handle to be written in CQE */
2188         in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2189         in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
2190
2191         /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2192          * the ring. There ring should allow at least a single WR, even if the
2193          * user requested none, due to allocation issues.
2194          */
2195         qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
2196
2197         /* Allocate driver internal RQ array */
2198         qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
2199                                 GFP_KERNEL);
2200         if (!qp->rqe_wr_id) {
2201                 DP_ERR(dev,
2202                        "create qp: failed RQ shadow memory allocation\n");
2203                 kfree(qp->wqe_wr_id);
2204                 return -ENOMEM;
2205         }
2206
2207         qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
2208
2209         n_sq_entries = attrs->cap.max_send_wr;
2210         n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2211         n_sq_entries = max_t(u32, n_sq_entries, 1);
2212         n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2213
2214         n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2215
2216         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2217                 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2218                                                  n_sq_elems, n_rq_elems);
2219         else
2220                 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2221                                                 n_sq_elems, n_rq_elems);
2222         if (rc)
2223                 qedr_cleanup_kernel(dev, qp);
2224
2225         return rc;
2226 }
2227
2228 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2229                                   struct ib_udata *udata)
2230 {
2231         struct qedr_ucontext *ctx =
2232                 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2233                                           ibucontext);
2234         int rc;
2235
2236         if (qp->qp_type != IB_QPT_GSI) {
2237                 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2238                 if (rc)
2239                         return rc;
2240         }
2241
2242         if (qp->create_type == QEDR_QP_CREATE_USER)
2243                 qedr_cleanup_user(dev, ctx, qp);
2244         else
2245                 qedr_cleanup_kernel(dev, qp);
2246
2247         return 0;
2248 }
2249
2250 int qedr_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
2251                    struct ib_udata *udata)
2252 {
2253         struct qedr_xrcd *xrcd = NULL;
2254         struct ib_pd *ibpd = ibqp->pd;
2255         struct qedr_pd *pd = get_qedr_pd(ibpd);
2256         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2257         struct qedr_qp *qp = get_qedr_qp(ibqp);
2258         int rc = 0;
2259
2260         if (attrs->create_flags)
2261                 return -EOPNOTSUPP;
2262
2263         if (attrs->qp_type == IB_QPT_XRC_TGT)
2264                 xrcd = get_qedr_xrcd(attrs->xrcd);
2265         else
2266                 pd = get_qedr_pd(ibpd);
2267
2268         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2269                  udata ? "user library" : "kernel", pd);
2270
2271         rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
2272         if (rc)
2273                 return rc;
2274
2275         DP_DEBUG(dev, QEDR_MSG_QP,
2276                  "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2277                  udata ? "user library" : "kernel", attrs->event_handler, pd,
2278                  get_qedr_cq(attrs->send_cq),
2279                  get_qedr_cq(attrs->send_cq)->icid,
2280                  get_qedr_cq(attrs->recv_cq),
2281                  attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
2282
2283         qedr_set_common_qp_params(dev, qp, pd, attrs);
2284
2285         if (attrs->qp_type == IB_QPT_GSI)
2286                 return qedr_create_gsi_qp(dev, attrs, qp);
2287
2288         if (udata || xrcd)
2289                 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2290         else
2291                 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2292
2293         if (rc)
2294                 return rc;
2295
2296         qp->ibqp.qp_num = qp->qp_id;
2297
2298         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2299                 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
2300                 if (rc)
2301                         goto out_free_qp_resources;
2302         }
2303
2304         return 0;
2305
2306 out_free_qp_resources:
2307         qedr_free_qp_resources(dev, qp, udata);
2308         return -EFAULT;
2309 }
2310
2311 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
2312 {
2313         switch (qp_state) {
2314         case QED_ROCE_QP_STATE_RESET:
2315                 return IB_QPS_RESET;
2316         case QED_ROCE_QP_STATE_INIT:
2317                 return IB_QPS_INIT;
2318         case QED_ROCE_QP_STATE_RTR:
2319                 return IB_QPS_RTR;
2320         case QED_ROCE_QP_STATE_RTS:
2321                 return IB_QPS_RTS;
2322         case QED_ROCE_QP_STATE_SQD:
2323                 return IB_QPS_SQD;
2324         case QED_ROCE_QP_STATE_ERR:
2325                 return IB_QPS_ERR;
2326         case QED_ROCE_QP_STATE_SQE:
2327                 return IB_QPS_SQE;
2328         }
2329         return IB_QPS_ERR;
2330 }
2331
2332 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2333                                         enum ib_qp_state qp_state)
2334 {
2335         switch (qp_state) {
2336         case IB_QPS_RESET:
2337                 return QED_ROCE_QP_STATE_RESET;
2338         case IB_QPS_INIT:
2339                 return QED_ROCE_QP_STATE_INIT;
2340         case IB_QPS_RTR:
2341                 return QED_ROCE_QP_STATE_RTR;
2342         case IB_QPS_RTS:
2343                 return QED_ROCE_QP_STATE_RTS;
2344         case IB_QPS_SQD:
2345                 return QED_ROCE_QP_STATE_SQD;
2346         case IB_QPS_ERR:
2347                 return QED_ROCE_QP_STATE_ERR;
2348         default:
2349                 return QED_ROCE_QP_STATE_ERR;
2350         }
2351 }
2352
2353 static int qedr_update_qp_state(struct qedr_dev *dev,
2354                                 struct qedr_qp *qp,
2355                                 enum qed_roce_qp_state cur_state,
2356                                 enum qed_roce_qp_state new_state)
2357 {
2358         int status = 0;
2359
2360         if (new_state == cur_state)
2361                 return 0;
2362
2363         switch (cur_state) {
2364         case QED_ROCE_QP_STATE_RESET:
2365                 switch (new_state) {
2366                 case QED_ROCE_QP_STATE_INIT:
2367                         break;
2368                 default:
2369                         status = -EINVAL;
2370                         break;
2371                 }
2372                 break;
2373         case QED_ROCE_QP_STATE_INIT:
2374                 switch (new_state) {
2375                 case QED_ROCE_QP_STATE_RTR:
2376                         /* Update doorbell (in case post_recv was
2377                          * done before move to RTR)
2378                          */
2379
2380                         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2381                                 writel(qp->rq.db_data.raw, qp->rq.db);
2382                         }
2383                         break;
2384                 case QED_ROCE_QP_STATE_ERR:
2385                         break;
2386                 default:
2387                         /* Invalid state change. */
2388                         status = -EINVAL;
2389                         break;
2390                 }
2391                 break;
2392         case QED_ROCE_QP_STATE_RTR:
2393                 /* RTR->XXX */
2394                 switch (new_state) {
2395                 case QED_ROCE_QP_STATE_RTS:
2396                         break;
2397                 case QED_ROCE_QP_STATE_ERR:
2398                         break;
2399                 default:
2400                         /* Invalid state change. */
2401                         status = -EINVAL;
2402                         break;
2403                 }
2404                 break;
2405         case QED_ROCE_QP_STATE_RTS:
2406                 /* RTS->XXX */
2407                 switch (new_state) {
2408                 case QED_ROCE_QP_STATE_SQD:
2409                         break;
2410                 case QED_ROCE_QP_STATE_ERR:
2411                         break;
2412                 default:
2413                         /* Invalid state change. */
2414                         status = -EINVAL;
2415                         break;
2416                 }
2417                 break;
2418         case QED_ROCE_QP_STATE_SQD:
2419                 /* SQD->XXX */
2420                 switch (new_state) {
2421                 case QED_ROCE_QP_STATE_RTS:
2422                 case QED_ROCE_QP_STATE_ERR:
2423                         break;
2424                 default:
2425                         /* Invalid state change. */
2426                         status = -EINVAL;
2427                         break;
2428                 }
2429                 break;
2430         case QED_ROCE_QP_STATE_ERR:
2431                 /* ERR->XXX */
2432                 switch (new_state) {
2433                 case QED_ROCE_QP_STATE_RESET:
2434                         if ((qp->rq.prod != qp->rq.cons) ||
2435                             (qp->sq.prod != qp->sq.cons)) {
2436                                 DP_NOTICE(dev,
2437                                           "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2438                                           qp->rq.prod, qp->rq.cons, qp->sq.prod,
2439                                           qp->sq.cons);
2440                                 status = -EINVAL;
2441                         }
2442                         break;
2443                 default:
2444                         status = -EINVAL;
2445                         break;
2446                 }
2447                 break;
2448         default:
2449                 status = -EINVAL;
2450                 break;
2451         }
2452
2453         return status;
2454 }
2455
2456 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2457                    int attr_mask, struct ib_udata *udata)
2458 {
2459         struct qedr_qp *qp = get_qedr_qp(ibqp);
2460         struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2461         struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2462         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2463         enum ib_qp_state old_qp_state, new_qp_state;
2464         enum qed_roce_qp_state cur_state;
2465         int rc = 0;
2466
2467         DP_DEBUG(dev, QEDR_MSG_QP,
2468                  "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2469                  attr->qp_state);
2470
2471         if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2472                 return -EOPNOTSUPP;
2473
2474         old_qp_state = qedr_get_ibqp_state(qp->state);
2475         if (attr_mask & IB_QP_STATE)
2476                 new_qp_state = attr->qp_state;
2477         else
2478                 new_qp_state = old_qp_state;
2479
2480         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2481                 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2482                                         ibqp->qp_type, attr_mask)) {
2483                         DP_ERR(dev,
2484                                "modify qp: invalid attribute mask=0x%x specified for\n"
2485                                "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2486                                attr_mask, qp->qp_id, ibqp->qp_type,
2487                                old_qp_state, new_qp_state);
2488                         rc = -EINVAL;
2489                         goto err;
2490                 }
2491         }
2492
2493         /* Translate the masks... */
2494         if (attr_mask & IB_QP_STATE) {
2495                 SET_FIELD(qp_params.modify_flags,
2496                           QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2497                 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2498         }
2499
2500         if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2501                 qp_params.sqd_async = true;
2502
2503         if (attr_mask & IB_QP_PKEY_INDEX) {
2504                 SET_FIELD(qp_params.modify_flags,
2505                           QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2506                 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2507                         rc = -EINVAL;
2508                         goto err;
2509                 }
2510
2511                 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2512         }
2513
2514         if (attr_mask & IB_QP_QKEY)
2515                 qp->qkey = attr->qkey;
2516
2517         if (attr_mask & IB_QP_ACCESS_FLAGS) {
2518                 SET_FIELD(qp_params.modify_flags,
2519                           QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2520                 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2521                                                   IB_ACCESS_REMOTE_READ;
2522                 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2523                                                    IB_ACCESS_REMOTE_WRITE;
2524                 qp_params.incoming_atomic_en = attr->qp_access_flags &
2525                                                IB_ACCESS_REMOTE_ATOMIC;
2526         }
2527
2528         if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2529                 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2530                         return -EINVAL;
2531
2532                 if (attr_mask & IB_QP_PATH_MTU) {
2533                         if (attr->path_mtu < IB_MTU_256 ||
2534                             attr->path_mtu > IB_MTU_4096) {
2535                                 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2536                                 rc = -EINVAL;
2537                                 goto err;
2538                         }
2539                         qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2540                                       ib_mtu_enum_to_int(iboe_get_mtu
2541                                                          (dev->ndev->mtu)));
2542                 }
2543
2544                 if (!qp->mtu) {
2545                         qp->mtu =
2546                         ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2547                         pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2548                 }
2549
2550                 SET_FIELD(qp_params.modify_flags,
2551                           QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2552
2553                 qp_params.traffic_class_tos = grh->traffic_class;
2554                 qp_params.flow_label = grh->flow_label;
2555                 qp_params.hop_limit_ttl = grh->hop_limit;
2556
2557                 qp->sgid_idx = grh->sgid_index;
2558
2559                 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2560                 if (rc) {
2561                         DP_ERR(dev,
2562                                "modify qp: problems with GID index %d (rc=%d)\n",
2563                                grh->sgid_index, rc);
2564                         return rc;
2565                 }
2566
2567                 rc = qedr_get_dmac(dev, &attr->ah_attr,
2568                                    qp_params.remote_mac_addr);
2569                 if (rc)
2570                         return rc;
2571
2572                 qp_params.use_local_mac = true;
2573                 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2574
2575                 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2576                          qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2577                          qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2578                 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2579                          qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2580                          qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2581                 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2582                          qp_params.remote_mac_addr);
2583
2584                 qp_params.mtu = qp->mtu;
2585                 qp_params.lb_indication = false;
2586         }
2587
2588         if (!qp_params.mtu) {
2589                 /* Stay with current MTU */
2590                 if (qp->mtu)
2591                         qp_params.mtu = qp->mtu;
2592                 else
2593                         qp_params.mtu =
2594                             ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2595         }
2596
2597         if (attr_mask & IB_QP_TIMEOUT) {
2598                 SET_FIELD(qp_params.modify_flags,
2599                           QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2600
2601                 /* The received timeout value is an exponent used like this:
2602                  *    "12.7.34 LOCAL ACK TIMEOUT
2603                  *    Value representing the transport (ACK) timeout for use by
2604                  *    the remote, expressed as: 4.096 * 2^timeout [usec]"
2605                  * The FW expects timeout in msec so we need to divide the usec
2606                  * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2607                  * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2608                  * The value of zero means infinite so we use a 'max_t' to make
2609                  * sure that sub 1 msec values will be configured as 1 msec.
2610                  */
2611                 if (attr->timeout)
2612                         qp_params.ack_timeout =
2613                                         1 << max_t(int, attr->timeout - 8, 0);
2614                 else
2615                         qp_params.ack_timeout = 0;
2616
2617                 qp->timeout = attr->timeout;
2618         }
2619
2620         if (attr_mask & IB_QP_RETRY_CNT) {
2621                 SET_FIELD(qp_params.modify_flags,
2622                           QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2623                 qp_params.retry_cnt = attr->retry_cnt;
2624         }
2625
2626         if (attr_mask & IB_QP_RNR_RETRY) {
2627                 SET_FIELD(qp_params.modify_flags,
2628                           QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2629                 qp_params.rnr_retry_cnt = attr->rnr_retry;
2630         }
2631
2632         if (attr_mask & IB_QP_RQ_PSN) {
2633                 SET_FIELD(qp_params.modify_flags,
2634                           QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2635                 qp_params.rq_psn = attr->rq_psn;
2636                 qp->rq_psn = attr->rq_psn;
2637         }
2638
2639         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2640                 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2641                         rc = -EINVAL;
2642                         DP_ERR(dev,
2643                                "unsupported max_rd_atomic=%d, supported=%d\n",
2644                                attr->max_rd_atomic,
2645                                dev->attr.max_qp_req_rd_atomic_resc);
2646                         goto err;
2647                 }
2648
2649                 SET_FIELD(qp_params.modify_flags,
2650                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2651                 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2652         }
2653
2654         if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2655                 SET_FIELD(qp_params.modify_flags,
2656                           QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2657                 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2658         }
2659
2660         if (attr_mask & IB_QP_SQ_PSN) {
2661                 SET_FIELD(qp_params.modify_flags,
2662                           QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2663                 qp_params.sq_psn = attr->sq_psn;
2664                 qp->sq_psn = attr->sq_psn;
2665         }
2666
2667         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2668                 if (attr->max_dest_rd_atomic >
2669                     dev->attr.max_qp_resp_rd_atomic_resc) {
2670                         DP_ERR(dev,
2671                                "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2672                                attr->max_dest_rd_atomic,
2673                                dev->attr.max_qp_resp_rd_atomic_resc);
2674
2675                         rc = -EINVAL;
2676                         goto err;
2677                 }
2678
2679                 SET_FIELD(qp_params.modify_flags,
2680                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2681                 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2682         }
2683
2684         if (attr_mask & IB_QP_DEST_QPN) {
2685                 SET_FIELD(qp_params.modify_flags,
2686                           QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2687
2688                 qp_params.dest_qp = attr->dest_qp_num;
2689                 qp->dest_qp_num = attr->dest_qp_num;
2690         }
2691
2692         cur_state = qp->state;
2693
2694         /* Update the QP state before the actual ramrod to prevent a race with
2695          * fast path. Modifying the QP state to error will cause the device to
2696          * flush the CQEs and while polling the flushed CQEs will considered as
2697          * a potential issue if the QP isn't in error state.
2698          */
2699         if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2700             !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2701                 qp->state = QED_ROCE_QP_STATE_ERR;
2702
2703         if (qp->qp_type != IB_QPT_GSI)
2704                 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2705                                               qp->qed_qp, &qp_params);
2706
2707         if (attr_mask & IB_QP_STATE) {
2708                 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2709                         rc = qedr_update_qp_state(dev, qp, cur_state,
2710                                                   qp_params.new_state);
2711                 qp->state = qp_params.new_state;
2712         }
2713
2714 err:
2715         return rc;
2716 }
2717
2718 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2719 {
2720         int ib_qp_acc_flags = 0;
2721
2722         if (params->incoming_rdma_write_en)
2723                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2724         if (params->incoming_rdma_read_en)
2725                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2726         if (params->incoming_atomic_en)
2727                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2728         ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2729         return ib_qp_acc_flags;
2730 }
2731
2732 int qedr_query_qp(struct ib_qp *ibqp,
2733                   struct ib_qp_attr *qp_attr,
2734                   int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2735 {
2736         struct qed_rdma_query_qp_out_params params;
2737         struct qedr_qp *qp = get_qedr_qp(ibqp);
2738         struct qedr_dev *dev = qp->dev;
2739         int rc = 0;
2740
2741         memset(&params, 0, sizeof(params));
2742         memset(qp_attr, 0, sizeof(*qp_attr));
2743         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2744
2745         if (qp->qp_type != IB_QPT_GSI) {
2746                 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2747                 if (rc)
2748                         goto err;
2749                 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2750         } else {
2751                 qp_attr->qp_state = qedr_get_ibqp_state(QED_ROCE_QP_STATE_RTS);
2752         }
2753
2754         qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2755         qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2756         qp_attr->path_mig_state = IB_MIG_MIGRATED;
2757         qp_attr->rq_psn = params.rq_psn;
2758         qp_attr->sq_psn = params.sq_psn;
2759         qp_attr->dest_qp_num = params.dest_qp;
2760
2761         qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2762
2763         qp_attr->cap.max_send_wr = qp->sq.max_wr;
2764         qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2765         qp_attr->cap.max_send_sge = qp->sq.max_sges;
2766         qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2767         qp_attr->cap.max_inline_data = dev->attr.max_inline;
2768         qp_init_attr->cap = qp_attr->cap;
2769
2770         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2771         rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2772                         params.flow_label, qp->sgid_idx,
2773                         params.hop_limit_ttl, params.traffic_class_tos);
2774         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2775         rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2776         rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2777         qp_attr->timeout = qp->timeout;
2778         qp_attr->rnr_retry = params.rnr_retry;
2779         qp_attr->retry_cnt = params.retry_cnt;
2780         qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2781         qp_attr->pkey_index = params.pkey_index;
2782         qp_attr->port_num = 1;
2783         rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2784         rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2785         qp_attr->alt_pkey_index = 0;
2786         qp_attr->alt_port_num = 0;
2787         qp_attr->alt_timeout = 0;
2788         memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2789
2790         qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2791         qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2792         qp_attr->max_rd_atomic = params.max_rd_atomic;
2793         qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2794
2795         DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2796                  qp_attr->cap.max_inline_data);
2797
2798 err:
2799         return rc;
2800 }
2801
2802 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2803 {
2804         struct qedr_qp *qp = get_qedr_qp(ibqp);
2805         struct qedr_dev *dev = qp->dev;
2806         struct ib_qp_attr attr;
2807         int attr_mask = 0;
2808
2809         DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2810                  qp, qp->qp_type);
2811
2812         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2813                 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2814                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
2815                     (qp->state != QED_ROCE_QP_STATE_INIT)) {
2816
2817                         attr.qp_state = IB_QPS_ERR;
2818                         attr_mask |= IB_QP_STATE;
2819
2820                         /* Change the QP state to ERROR */
2821                         qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2822                 }
2823         } else {
2824                 /* If connection establishment started the WAIT_FOR_CONNECT
2825                  * bit will be on and we need to Wait for the establishment
2826                  * to complete before destroying the qp.
2827                  */
2828                 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2829                                      &qp->iwarp_cm_flags))
2830                         wait_for_completion(&qp->iwarp_cm_comp);
2831
2832                 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2833                  * bit will be on, and we need to wait for the disconnect to
2834                  * complete before continuing. We can use the same completion,
2835                  * iwarp_cm_comp, since this is the only place that waits for
2836                  * this completion and it is sequential. In addition,
2837                  * disconnect can't occur before the connection is fully
2838                  * established, therefore if WAIT_FOR_DISCONNECT is on it
2839                  * means WAIT_FOR_CONNECT is also on and the completion for
2840                  * CONNECT already occurred.
2841                  */
2842                 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2843                                      &qp->iwarp_cm_flags))
2844                         wait_for_completion(&qp->iwarp_cm_comp);
2845         }
2846
2847         if (qp->qp_type == IB_QPT_GSI)
2848                 qedr_destroy_gsi_qp(dev);
2849
2850         /* We need to remove the entry from the xarray before we release the
2851          * qp_id to avoid a race of the qp_id being reallocated and failing
2852          * on xa_insert
2853          */
2854         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2855                 xa_erase(&dev->qps, qp->qp_id);
2856
2857         qedr_free_qp_resources(dev, qp, udata);
2858
2859         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2860                 qedr_iw_qp_rem_ref(&qp->ibqp);
2861                 wait_for_completion(&qp->qp_rel_comp);
2862         }
2863
2864         return 0;
2865 }
2866
2867 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
2868                    struct ib_udata *udata)
2869 {
2870         struct qedr_ah *ah = get_qedr_ah(ibah);
2871
2872         rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
2873
2874         return 0;
2875 }
2876
2877 int qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2878 {
2879         struct qedr_ah *ah = get_qedr_ah(ibah);
2880
2881         rdma_destroy_ah_attr(&ah->attr);
2882         return 0;
2883 }
2884
2885 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2886 {
2887         struct qedr_pbl *pbl, *tmp;
2888
2889         if (info->pbl_table)
2890                 list_add_tail(&info->pbl_table->list_entry,
2891                               &info->free_pbl_list);
2892
2893         if (!list_empty(&info->inuse_pbl_list))
2894                 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2895
2896         list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2897                 list_del(&pbl->list_entry);
2898                 qedr_free_pbl(dev, &info->pbl_info, pbl);
2899         }
2900 }
2901
2902 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2903                         size_t page_list_len, bool two_layered)
2904 {
2905         struct qedr_pbl *tmp;
2906         int rc;
2907
2908         INIT_LIST_HEAD(&info->free_pbl_list);
2909         INIT_LIST_HEAD(&info->inuse_pbl_list);
2910
2911         rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2912                                   page_list_len, two_layered);
2913         if (rc)
2914                 goto done;
2915
2916         info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2917         if (IS_ERR(info->pbl_table)) {
2918                 rc = PTR_ERR(info->pbl_table);
2919                 goto done;
2920         }
2921
2922         DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2923                  &info->pbl_table->pa);
2924
2925         /* in usual case we use 2 PBLs, so we add one to free
2926          * list and allocating another one
2927          */
2928         tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2929         if (IS_ERR(tmp)) {
2930                 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2931                 goto done;
2932         }
2933
2934         list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2935
2936         DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2937
2938 done:
2939         if (rc)
2940                 free_mr_info(dev, info);
2941
2942         return rc;
2943 }
2944
2945 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2946                                u64 usr_addr, int acc, struct ib_udata *udata)
2947 {
2948         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2949         struct qedr_mr *mr;
2950         struct qedr_pd *pd;
2951         int rc = -ENOMEM;
2952
2953         pd = get_qedr_pd(ibpd);
2954         DP_DEBUG(dev, QEDR_MSG_MR,
2955                  "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2956                  pd->pd_id, start, len, usr_addr, acc);
2957
2958         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2959                 return ERR_PTR(-EINVAL);
2960
2961         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2962         if (!mr)
2963                 return ERR_PTR(rc);
2964
2965         mr->type = QEDR_MR_USER;
2966
2967         mr->umem = ib_umem_get(ibpd->device, start, len, acc);
2968         if (IS_ERR(mr->umem)) {
2969                 rc = -EFAULT;
2970                 goto err0;
2971         }
2972
2973         rc = init_mr_info(dev, &mr->info,
2974                           ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE), 1);
2975         if (rc)
2976                 goto err1;
2977
2978         qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2979                            &mr->info.pbl_info, PAGE_SHIFT);
2980
2981         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2982         if (rc) {
2983                 if (rc == -EINVAL)
2984                         DP_ERR(dev, "Out of MR resources\n");
2985                 else
2986                         DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
2987
2988                 goto err1;
2989         }
2990
2991         /* Index only, 18 bit long, lkey = itid << 8 | key */
2992         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2993         mr->hw_mr.key = 0;
2994         mr->hw_mr.pd = pd->pd_id;
2995         mr->hw_mr.local_read = 1;
2996         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2997         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2998         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2999         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3000         mr->hw_mr.mw_bind = false;
3001         mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
3002         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3003         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3004         mr->hw_mr.page_size_log = PAGE_SHIFT;
3005         mr->hw_mr.length = len;
3006         mr->hw_mr.vaddr = usr_addr;
3007         mr->hw_mr.phy_mr = false;
3008         mr->hw_mr.dma_mr = false;
3009
3010         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3011         if (rc) {
3012                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3013                 goto err2;
3014         }
3015
3016         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3017         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3018             mr->hw_mr.remote_atomic)
3019                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3020
3021         DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
3022                  mr->ibmr.lkey);
3023         return &mr->ibmr;
3024
3025 err2:
3026         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3027 err1:
3028         qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3029 err0:
3030         kfree(mr);
3031         return ERR_PTR(rc);
3032 }
3033
3034 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
3035 {
3036         struct qedr_mr *mr = get_qedr_mr(ib_mr);
3037         struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
3038         int rc = 0;
3039
3040         rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
3041         if (rc)
3042                 return rc;
3043
3044         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3045
3046         if (mr->type != QEDR_MR_DMA)
3047                 free_mr_info(dev, &mr->info);
3048
3049         /* it could be user registered memory. */
3050         ib_umem_release(mr->umem);
3051
3052         kfree(mr);
3053
3054         return rc;
3055 }
3056
3057 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
3058                                        int max_page_list_len)
3059 {
3060         struct qedr_pd *pd = get_qedr_pd(ibpd);
3061         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3062         struct qedr_mr *mr;
3063         int rc = -ENOMEM;
3064
3065         DP_DEBUG(dev, QEDR_MSG_MR,
3066                  "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
3067                  max_page_list_len);
3068
3069         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3070         if (!mr)
3071                 return ERR_PTR(rc);
3072
3073         mr->dev = dev;
3074         mr->type = QEDR_MR_FRMR;
3075
3076         rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
3077         if (rc)
3078                 goto err0;
3079
3080         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3081         if (rc) {
3082                 if (rc == -EINVAL)
3083                         DP_ERR(dev, "Out of MR resources\n");
3084                 else
3085                         DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3086
3087                 goto err1;
3088         }
3089
3090         /* Index only, 18 bit long, lkey = itid << 8 | key */
3091         mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
3092         mr->hw_mr.key = 0;
3093         mr->hw_mr.pd = pd->pd_id;
3094         mr->hw_mr.local_read = 1;
3095         mr->hw_mr.local_write = 0;
3096         mr->hw_mr.remote_read = 0;
3097         mr->hw_mr.remote_write = 0;
3098         mr->hw_mr.remote_atomic = 0;
3099         mr->hw_mr.mw_bind = false;
3100         mr->hw_mr.pbl_ptr = 0;
3101         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3102         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3103         mr->hw_mr.length = 0;
3104         mr->hw_mr.vaddr = 0;
3105         mr->hw_mr.phy_mr = true;
3106         mr->hw_mr.dma_mr = false;
3107
3108         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3109         if (rc) {
3110                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3111                 goto err2;
3112         }
3113
3114         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3115         mr->ibmr.rkey = mr->ibmr.lkey;
3116
3117         DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
3118         return mr;
3119
3120 err2:
3121         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3122 err1:
3123         qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3124 err0:
3125         kfree(mr);
3126         return ERR_PTR(rc);
3127 }
3128
3129 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
3130                             u32 max_num_sg)
3131 {
3132         struct qedr_mr *mr;
3133
3134         if (mr_type != IB_MR_TYPE_MEM_REG)
3135                 return ERR_PTR(-EINVAL);
3136
3137         mr = __qedr_alloc_mr(ibpd, max_num_sg);
3138
3139         if (IS_ERR(mr))
3140                 return ERR_PTR(-EINVAL);
3141
3142         return &mr->ibmr;
3143 }
3144
3145 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3146 {
3147         struct qedr_mr *mr = get_qedr_mr(ibmr);
3148         struct qedr_pbl *pbl_table;
3149         struct regpair *pbe;
3150         u32 pbes_in_page;
3151
3152         if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
3153                 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
3154                 return -ENOMEM;
3155         }
3156
3157         DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3158                  mr->npages, addr);
3159
3160         pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3161         pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3162         pbe = (struct regpair *)pbl_table->va;
3163         pbe +=  mr->npages % pbes_in_page;
3164         pbe->lo = cpu_to_le32((u32)addr);
3165         pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3166
3167         mr->npages++;
3168
3169         return 0;
3170 }
3171
3172 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3173 {
3174         int work = info->completed - info->completed_handled - 1;
3175
3176         DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3177         while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
3178                 struct qedr_pbl *pbl;
3179
3180                 /* Free all the page list that are possible to be freed
3181                  * (all the ones that were invalidated), under the assumption
3182                  * that if an FMR was completed successfully that means that
3183                  * if there was an invalidate operation before it also ended
3184                  */
3185                 pbl = list_first_entry(&info->inuse_pbl_list,
3186                                        struct qedr_pbl, list_entry);
3187                 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
3188                 info->completed_handled++;
3189         }
3190 }
3191
3192 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3193                    int sg_nents, unsigned int *sg_offset)
3194 {
3195         struct qedr_mr *mr = get_qedr_mr(ibmr);
3196
3197         mr->npages = 0;
3198
3199         handle_completed_mrs(mr->dev, &mr->info);
3200         return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
3201 }
3202
3203 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3204 {
3205         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3206         struct qedr_pd *pd = get_qedr_pd(ibpd);
3207         struct qedr_mr *mr;
3208         int rc;
3209
3210         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3211         if (!mr)
3212                 return ERR_PTR(-ENOMEM);
3213
3214         mr->type = QEDR_MR_DMA;
3215
3216         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3217         if (rc) {
3218                 if (rc == -EINVAL)
3219                         DP_ERR(dev, "Out of MR resources\n");
3220                 else
3221                         DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3222
3223                 goto err1;
3224         }
3225
3226         /* index only, 18 bit long, lkey = itid << 8 | key */
3227         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3228         mr->hw_mr.pd = pd->pd_id;
3229         mr->hw_mr.local_read = 1;
3230         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3231         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3232         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3233         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3234         mr->hw_mr.dma_mr = true;
3235
3236         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3237         if (rc) {
3238                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3239                 goto err2;
3240         }
3241
3242         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3243         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3244             mr->hw_mr.remote_atomic)
3245                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3246
3247         DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3248         return &mr->ibmr;
3249
3250 err2:
3251         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3252 err1:
3253         kfree(mr);
3254         return ERR_PTR(rc);
3255 }
3256
3257 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3258 {
3259         return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3260 }
3261
3262 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3263 {
3264         int i, len = 0;
3265
3266         for (i = 0; i < num_sge; i++)
3267                 len += sg_list[i].length;
3268
3269         return len;
3270 }
3271
3272 static void swap_wqe_data64(u64 *p)
3273 {
3274         int i;
3275
3276         for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3277                 *p = cpu_to_be64(cpu_to_le64(*p));
3278 }
3279
3280 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3281                                        struct qedr_qp *qp, u8 *wqe_size,
3282                                        const struct ib_send_wr *wr,
3283                                        const struct ib_send_wr **bad_wr,
3284                                        u8 *bits, u8 bit)
3285 {
3286         u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3287         char *seg_prt, *wqe;
3288         int i, seg_siz;
3289
3290         if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3291                 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3292                 *bad_wr = wr;
3293                 return 0;
3294         }
3295
3296         if (!data_size)
3297                 return data_size;
3298
3299         *bits |= bit;
3300
3301         seg_prt = NULL;
3302         wqe = NULL;
3303         seg_siz = 0;
3304
3305         /* Copy data inline */
3306         for (i = 0; i < wr->num_sge; i++) {
3307                 u32 len = wr->sg_list[i].length;
3308                 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3309
3310                 while (len > 0) {
3311                         u32 cur;
3312
3313                         /* New segment required */
3314                         if (!seg_siz) {
3315                                 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3316                                 seg_prt = wqe;
3317                                 seg_siz = sizeof(struct rdma_sq_common_wqe);
3318                                 (*wqe_size)++;
3319                         }
3320
3321                         /* Calculate currently allowed length */
3322                         cur = min_t(u32, len, seg_siz);
3323                         memcpy(seg_prt, src, cur);
3324
3325                         /* Update segment variables */
3326                         seg_prt += cur;
3327                         seg_siz -= cur;
3328
3329                         /* Update sge variables */
3330                         src += cur;
3331                         len -= cur;
3332
3333                         /* Swap fully-completed segments */
3334                         if (!seg_siz)
3335                                 swap_wqe_data64((u64 *)wqe);
3336                 }
3337         }
3338
3339         /* swap last not completed segment */
3340         if (seg_siz)
3341                 swap_wqe_data64((u64 *)wqe);
3342
3343         return data_size;
3344 }
3345
3346 #define RQ_SGE_SET(sge, vaddr, vlength, vflags)                 \
3347         do {                                                    \
3348                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3349                 (sge)->length = cpu_to_le32(vlength);           \
3350                 (sge)->flags = cpu_to_le32(vflags);             \
3351         } while (0)
3352
3353 #define SRQ_HDR_SET(hdr, vwr_id, num_sge)                       \
3354         do {                                                    \
3355                 DMA_REGPAIR_LE(hdr->wr_id, vwr_id);             \
3356                 (hdr)->num_sges = num_sge;                      \
3357         } while (0)
3358
3359 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)                 \
3360         do {                                                    \
3361                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3362                 (sge)->length = cpu_to_le32(vlength);           \
3363                 (sge)->l_key = cpu_to_le32(vlkey);              \
3364         } while (0)
3365
3366 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3367                                 const struct ib_send_wr *wr)
3368 {
3369         u32 data_size = 0;
3370         int i;
3371
3372         for (i = 0; i < wr->num_sge; i++) {
3373                 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3374
3375                 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3376                 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3377                 sge->length = cpu_to_le32(wr->sg_list[i].length);
3378                 data_size += wr->sg_list[i].length;
3379         }
3380
3381         if (wqe_size)
3382                 *wqe_size += wr->num_sge;
3383
3384         return data_size;
3385 }
3386
3387 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3388                                      struct qedr_qp *qp,
3389                                      struct rdma_sq_rdma_wqe_1st *rwqe,
3390                                      struct rdma_sq_rdma_wqe_2nd *rwqe2,
3391                                      const struct ib_send_wr *wr,
3392                                      const struct ib_send_wr **bad_wr)
3393 {
3394         rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3395         DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3396
3397         if (wr->send_flags & IB_SEND_INLINE &&
3398             (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3399              wr->opcode == IB_WR_RDMA_WRITE)) {
3400                 u8 flags = 0;
3401
3402                 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3403                 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3404                                                    bad_wr, &rwqe->flags, flags);
3405         }
3406
3407         return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3408 }
3409
3410 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3411                                      struct qedr_qp *qp,
3412                                      struct rdma_sq_send_wqe_1st *swqe,
3413                                      struct rdma_sq_send_wqe_2st *swqe2,
3414                                      const struct ib_send_wr *wr,
3415                                      const struct ib_send_wr **bad_wr)
3416 {
3417         memset(swqe2, 0, sizeof(*swqe2));
3418         if (wr->send_flags & IB_SEND_INLINE) {
3419                 u8 flags = 0;
3420
3421                 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3422                 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3423                                                    bad_wr, &swqe->flags, flags);
3424         }
3425
3426         return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3427 }
3428
3429 static int qedr_prepare_reg(struct qedr_qp *qp,
3430                             struct rdma_sq_fmr_wqe_1st *fwqe1,
3431                             const struct ib_reg_wr *wr)
3432 {
3433         struct qedr_mr *mr = get_qedr_mr(wr->mr);
3434         struct rdma_sq_fmr_wqe_2nd *fwqe2;
3435
3436         fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3437         fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3438         fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3439         fwqe1->l_key = wr->key;
3440
3441         fwqe2->access_ctrl = 0;
3442
3443         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3444                    !!(wr->access & IB_ACCESS_REMOTE_READ));
3445         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3446                    !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3447         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3448                    !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3449         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3450         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3451                    !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3452         fwqe2->fmr_ctrl = 0;
3453
3454         SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3455                    ilog2(mr->ibmr.page_size) - 12);
3456
3457         fwqe2->length_hi = 0;
3458         fwqe2->length_lo = mr->ibmr.length;
3459         fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3460         fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3461
3462         qp->wqe_wr_id[qp->sq.prod].mr = mr;
3463
3464         return 0;
3465 }
3466
3467 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3468 {
3469         switch (opcode) {
3470         case IB_WR_RDMA_WRITE:
3471         case IB_WR_RDMA_WRITE_WITH_IMM:
3472                 return IB_WC_RDMA_WRITE;
3473         case IB_WR_SEND_WITH_IMM:
3474         case IB_WR_SEND:
3475         case IB_WR_SEND_WITH_INV:
3476                 return IB_WC_SEND;
3477         case IB_WR_RDMA_READ:
3478         case IB_WR_RDMA_READ_WITH_INV:
3479                 return IB_WC_RDMA_READ;
3480         case IB_WR_ATOMIC_CMP_AND_SWP:
3481                 return IB_WC_COMP_SWAP;
3482         case IB_WR_ATOMIC_FETCH_AND_ADD:
3483                 return IB_WC_FETCH_ADD;
3484         case IB_WR_REG_MR:
3485                 return IB_WC_REG_MR;
3486         case IB_WR_LOCAL_INV:
3487                 return IB_WC_LOCAL_INV;
3488         default:
3489                 return IB_WC_SEND;
3490         }
3491 }
3492
3493 static inline bool qedr_can_post_send(struct qedr_qp *qp,
3494                                       const struct ib_send_wr *wr)
3495 {
3496         int wq_is_full, err_wr, pbl_is_full;
3497         struct qedr_dev *dev = qp->dev;
3498
3499         /* prevent SQ overflow and/or processing of a bad WR */
3500         err_wr = wr->num_sge > qp->sq.max_sges;
3501         wq_is_full = qedr_wq_is_full(&qp->sq);
3502         pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3503                       QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3504         if (wq_is_full || err_wr || pbl_is_full) {
3505                 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3506                         DP_ERR(dev,
3507                                "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3508                                qp);
3509                         qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3510                 }
3511
3512                 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3513                         DP_ERR(dev,
3514                                "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3515                                qp);
3516                         qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3517                 }
3518
3519                 if (pbl_is_full &&
3520                     !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3521                         DP_ERR(dev,
3522                                "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3523                                qp);
3524                         qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3525                 }
3526                 return false;
3527         }
3528         return true;
3529 }
3530
3531 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3532                             const struct ib_send_wr **bad_wr)
3533 {
3534         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3535         struct qedr_qp *qp = get_qedr_qp(ibqp);
3536         struct rdma_sq_atomic_wqe_1st *awqe1;
3537         struct rdma_sq_atomic_wqe_2nd *awqe2;
3538         struct rdma_sq_atomic_wqe_3rd *awqe3;
3539         struct rdma_sq_send_wqe_2st *swqe2;
3540         struct rdma_sq_local_inv_wqe *iwqe;
3541         struct rdma_sq_rdma_wqe_2nd *rwqe2;
3542         struct rdma_sq_send_wqe_1st *swqe;
3543         struct rdma_sq_rdma_wqe_1st *rwqe;
3544         struct rdma_sq_fmr_wqe_1st *fwqe1;
3545         struct rdma_sq_common_wqe *wqe;
3546         u32 length;
3547         int rc = 0;
3548         bool comp;
3549
3550         if (!qedr_can_post_send(qp, wr)) {
3551                 *bad_wr = wr;
3552                 return -ENOMEM;
3553         }
3554
3555         wqe = qed_chain_produce(&qp->sq.pbl);
3556         qp->wqe_wr_id[qp->sq.prod].signaled =
3557                 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3558
3559         wqe->flags = 0;
3560         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3561                    !!(wr->send_flags & IB_SEND_SOLICITED));
3562         comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3563         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3564         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3565                    !!(wr->send_flags & IB_SEND_FENCE));
3566         wqe->prev_wqe_size = qp->prev_wqe_size;
3567
3568         qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3569
3570         switch (wr->opcode) {
3571         case IB_WR_SEND_WITH_IMM:
3572                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3573                         rc = -EINVAL;
3574                         *bad_wr = wr;
3575                         break;
3576                 }
3577                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3578                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3579                 swqe->wqe_size = 2;
3580                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3581
3582                 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3583                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3584                                                    wr, bad_wr);
3585                 swqe->length = cpu_to_le32(length);
3586                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3587                 qp->prev_wqe_size = swqe->wqe_size;
3588                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3589                 break;
3590         case IB_WR_SEND:
3591                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3592                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3593
3594                 swqe->wqe_size = 2;
3595                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3596                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3597                                                    wr, bad_wr);
3598                 swqe->length = cpu_to_le32(length);
3599                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3600                 qp->prev_wqe_size = swqe->wqe_size;
3601                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3602                 break;
3603         case IB_WR_SEND_WITH_INV:
3604                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3605                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3606                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3607                 swqe->wqe_size = 2;
3608                 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3609                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3610                                                    wr, bad_wr);
3611                 swqe->length = cpu_to_le32(length);
3612                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3613                 qp->prev_wqe_size = swqe->wqe_size;
3614                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3615                 break;
3616
3617         case IB_WR_RDMA_WRITE_WITH_IMM:
3618                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3619                         rc = -EINVAL;
3620                         *bad_wr = wr;
3621                         break;
3622                 }
3623                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3624                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3625
3626                 rwqe->wqe_size = 2;
3627                 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3628                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3629                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3630                                                    wr, bad_wr);
3631                 rwqe->length = cpu_to_le32(length);
3632                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3633                 qp->prev_wqe_size = rwqe->wqe_size;
3634                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3635                 break;
3636         case IB_WR_RDMA_WRITE:
3637                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3638                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3639
3640                 rwqe->wqe_size = 2;
3641                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3642                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3643                                                    wr, bad_wr);
3644                 rwqe->length = cpu_to_le32(length);
3645                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3646                 qp->prev_wqe_size = rwqe->wqe_size;
3647                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3648                 break;
3649         case IB_WR_RDMA_READ_WITH_INV:
3650                 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3651                 fallthrough;    /* same is identical to RDMA READ */
3652
3653         case IB_WR_RDMA_READ:
3654                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3655                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3656
3657                 rwqe->wqe_size = 2;
3658                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3659                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3660                                                    wr, bad_wr);
3661                 rwqe->length = cpu_to_le32(length);
3662                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3663                 qp->prev_wqe_size = rwqe->wqe_size;
3664                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3665                 break;
3666
3667         case IB_WR_ATOMIC_CMP_AND_SWP:
3668         case IB_WR_ATOMIC_FETCH_AND_ADD:
3669                 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3670                 awqe1->wqe_size = 4;
3671
3672                 awqe2 = qed_chain_produce(&qp->sq.pbl);
3673                 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3674                 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3675
3676                 awqe3 = qed_chain_produce(&qp->sq.pbl);
3677
3678                 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3679                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3680                         DMA_REGPAIR_LE(awqe3->swap_data,
3681                                        atomic_wr(wr)->compare_add);
3682                 } else {
3683                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3684                         DMA_REGPAIR_LE(awqe3->swap_data,
3685                                        atomic_wr(wr)->swap);
3686                         DMA_REGPAIR_LE(awqe3->cmp_data,
3687                                        atomic_wr(wr)->compare_add);
3688                 }
3689
3690                 qedr_prepare_sq_sges(qp, NULL, wr);
3691
3692                 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3693                 qp->prev_wqe_size = awqe1->wqe_size;
3694                 break;
3695
3696         case IB_WR_LOCAL_INV:
3697                 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3698                 iwqe->wqe_size = 1;
3699
3700                 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3701                 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3702                 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3703                 qp->prev_wqe_size = iwqe->wqe_size;
3704                 break;
3705         case IB_WR_REG_MR:
3706                 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3707                 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3708                 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3709                 fwqe1->wqe_size = 2;
3710
3711                 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3712                 if (rc) {
3713                         DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3714                         *bad_wr = wr;
3715                         break;
3716                 }
3717
3718                 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3719                 qp->prev_wqe_size = fwqe1->wqe_size;
3720                 break;
3721         default:
3722                 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3723                 rc = -EINVAL;
3724                 *bad_wr = wr;
3725                 break;
3726         }
3727
3728         if (*bad_wr) {
3729                 u16 value;
3730
3731                 /* Restore prod to its position before
3732                  * this WR was processed
3733                  */
3734                 value = le16_to_cpu(qp->sq.db_data.data.value);
3735                 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3736
3737                 /* Restore prev_wqe_size */
3738                 qp->prev_wqe_size = wqe->prev_wqe_size;
3739                 rc = -EINVAL;
3740                 DP_ERR(dev, "POST SEND FAILED\n");
3741         }
3742
3743         return rc;
3744 }
3745
3746 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3747                    const struct ib_send_wr **bad_wr)
3748 {
3749         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3750         struct qedr_qp *qp = get_qedr_qp(ibqp);
3751         unsigned long flags;
3752         int rc = 0;
3753
3754         *bad_wr = NULL;
3755
3756         if (qp->qp_type == IB_QPT_GSI)
3757                 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3758
3759         spin_lock_irqsave(&qp->q_lock, flags);
3760
3761         if (rdma_protocol_roce(&dev->ibdev, 1)) {
3762                 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3763                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
3764                     (qp->state != QED_ROCE_QP_STATE_SQD)) {
3765                         spin_unlock_irqrestore(&qp->q_lock, flags);
3766                         *bad_wr = wr;
3767                         DP_DEBUG(dev, QEDR_MSG_CQ,
3768                                  "QP in wrong state! QP icid=0x%x state %d\n",
3769                                  qp->icid, qp->state);
3770                         return -EINVAL;
3771                 }
3772         }
3773
3774         while (wr) {
3775                 rc = __qedr_post_send(ibqp, wr, bad_wr);
3776                 if (rc)
3777                         break;
3778
3779                 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3780
3781                 qedr_inc_sw_prod(&qp->sq);
3782
3783                 qp->sq.db_data.data.value++;
3784
3785                 wr = wr->next;
3786         }
3787
3788         /* Trigger doorbell
3789          * If there was a failure in the first WR then it will be triggered in
3790          * vane. However this is not harmful (as long as the producer value is
3791          * unchanged). For performance reasons we avoid checking for this
3792          * redundant doorbell.
3793          *
3794          * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3795          * soon as we give the doorbell, we could get a completion
3796          * for this wr, therefore we need to make sure that the
3797          * memory is updated before giving the doorbell.
3798          * During qedr_poll_cq, rmb is called before accessing the
3799          * cqe. This covers for the smp_rmb as well.
3800          */
3801         smp_wmb();
3802         writel(qp->sq.db_data.raw, qp->sq.db);
3803
3804         spin_unlock_irqrestore(&qp->q_lock, flags);
3805
3806         return rc;
3807 }
3808
3809 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3810 {
3811         u32 used;
3812
3813         /* Calculate number of elements used based on producer
3814          * count and consumer count and subtract it from max
3815          * work request supported so that we get elements left.
3816          */
3817         used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3818
3819         return hw_srq->max_wr - used;
3820 }
3821
3822 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3823                        const struct ib_recv_wr **bad_wr)
3824 {
3825         struct qedr_srq *srq = get_qedr_srq(ibsrq);
3826         struct qedr_srq_hwq_info *hw_srq;
3827         struct qedr_dev *dev = srq->dev;
3828         struct qed_chain *pbl;
3829         unsigned long flags;
3830         int status = 0;
3831         u32 num_sge;
3832
3833         spin_lock_irqsave(&srq->lock, flags);
3834
3835         hw_srq = &srq->hw_srq;
3836         pbl = &srq->hw_srq.pbl;
3837         while (wr) {
3838                 struct rdma_srq_wqe_header *hdr;
3839                 int i;
3840
3841                 if (!qedr_srq_elem_left(hw_srq) ||
3842                     wr->num_sge > srq->hw_srq.max_sges) {
3843                         DP_ERR(dev, "Can't post WR  (%d,%d) || (%d > %d)\n",
3844                                hw_srq->wr_prod_cnt,
3845                                atomic_read(&hw_srq->wr_cons_cnt),
3846                                wr->num_sge, srq->hw_srq.max_sges);
3847                         status = -ENOMEM;
3848                         *bad_wr = wr;
3849                         break;
3850                 }
3851
3852                 hdr = qed_chain_produce(pbl);
3853                 num_sge = wr->num_sge;
3854                 /* Set number of sge and work request id in header */
3855                 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3856
3857                 srq->hw_srq.wr_prod_cnt++;
3858                 hw_srq->wqe_prod++;
3859                 hw_srq->sge_prod++;
3860
3861                 DP_DEBUG(dev, QEDR_MSG_SRQ,
3862                          "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3863                          wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3864
3865                 for (i = 0; i < wr->num_sge; i++) {
3866                         struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3867
3868                         /* Set SGE length, lkey and address */
3869                         SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3870                                     wr->sg_list[i].length, wr->sg_list[i].lkey);
3871
3872                         DP_DEBUG(dev, QEDR_MSG_SRQ,
3873                                  "[%d]: len %d key %x addr %x:%x\n",
3874                                  i, srq_sge->length, srq_sge->l_key,
3875                                  srq_sge->addr.hi, srq_sge->addr.lo);
3876                         hw_srq->sge_prod++;
3877                 }
3878
3879                 /* Update WQE and SGE information before
3880                  * updating producer.
3881                  */
3882                 dma_wmb();
3883
3884                 /* SRQ producer is 8 bytes. Need to update SGE producer index
3885                  * in first 4 bytes and need to update WQE producer in
3886                  * next 4 bytes.
3887                  */
3888                 srq->hw_srq.virt_prod_pair_addr->sge_prod = cpu_to_le32(hw_srq->sge_prod);
3889                 /* Make sure sge producer is updated first */
3890                 dma_wmb();
3891                 srq->hw_srq.virt_prod_pair_addr->wqe_prod = cpu_to_le32(hw_srq->wqe_prod);
3892
3893                 wr = wr->next;
3894         }
3895
3896         DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3897                  qed_chain_get_elem_left(pbl));
3898         spin_unlock_irqrestore(&srq->lock, flags);
3899
3900         return status;
3901 }
3902
3903 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3904                    const struct ib_recv_wr **bad_wr)
3905 {
3906         struct qedr_qp *qp = get_qedr_qp(ibqp);
3907         struct qedr_dev *dev = qp->dev;
3908         unsigned long flags;
3909         int status = 0;
3910
3911         if (qp->qp_type == IB_QPT_GSI)
3912                 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3913
3914         spin_lock_irqsave(&qp->q_lock, flags);
3915
3916         while (wr) {
3917                 int i;
3918
3919                 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3920                     QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3921                     wr->num_sge > qp->rq.max_sges) {
3922                         DP_ERR(dev, "Can't post WR  (%d < %d) || (%d > %d)\n",
3923                                qed_chain_get_elem_left_u32(&qp->rq.pbl),
3924                                QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3925                                qp->rq.max_sges);
3926                         status = -ENOMEM;
3927                         *bad_wr = wr;
3928                         break;
3929                 }
3930                 for (i = 0; i < wr->num_sge; i++) {
3931                         u32 flags = 0;
3932                         struct rdma_rq_sge *rqe =
3933                             qed_chain_produce(&qp->rq.pbl);
3934
3935                         /* First one must include the number
3936                          * of SGE in the list
3937                          */
3938                         if (!i)
3939                                 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3940                                           wr->num_sge);
3941
3942                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3943                                   wr->sg_list[i].lkey);
3944
3945                         RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3946                                    wr->sg_list[i].length, flags);
3947                 }
3948
3949                 /* Special case of no sges. FW requires between 1-4 sges...
3950                  * in this case we need to post 1 sge with length zero. this is
3951                  * because rdma write with immediate consumes an RQ.
3952                  */
3953                 if (!wr->num_sge) {
3954                         u32 flags = 0;
3955                         struct rdma_rq_sge *rqe =
3956                             qed_chain_produce(&qp->rq.pbl);
3957
3958                         /* First one must include the number
3959                          * of SGE in the list
3960                          */
3961                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3962                         SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3963
3964                         RQ_SGE_SET(rqe, 0, 0, flags);
3965                         i = 1;
3966                 }
3967
3968                 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3969                 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3970
3971                 qedr_inc_sw_prod(&qp->rq);
3972
3973                 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3974                  * soon as we give the doorbell, we could get a completion
3975                  * for this wr, therefore we need to make sure that the
3976                  * memory is update before giving the doorbell.
3977                  * During qedr_poll_cq, rmb is called before accessing the
3978                  * cqe. This covers for the smp_rmb as well.
3979                  */
3980                 smp_wmb();
3981
3982                 qp->rq.db_data.data.value++;
3983
3984                 writel(qp->rq.db_data.raw, qp->rq.db);
3985
3986                 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3987                         writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3988                 }
3989
3990                 wr = wr->next;
3991         }
3992
3993         spin_unlock_irqrestore(&qp->q_lock, flags);
3994
3995         return status;
3996 }
3997
3998 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3999 {
4000         struct rdma_cqe_requester *resp_cqe = &cqe->req;
4001
4002         return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
4003                 cq->pbl_toggle;
4004 }
4005
4006 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
4007 {
4008         struct rdma_cqe_requester *resp_cqe = &cqe->req;
4009         struct qedr_qp *qp;
4010
4011         qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
4012                                                    resp_cqe->qp_handle.lo,
4013                                                    u64);
4014         return qp;
4015 }
4016
4017 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
4018 {
4019         struct rdma_cqe_requester *resp_cqe = &cqe->req;
4020
4021         return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
4022 }
4023
4024 /* Return latest CQE (needs processing) */
4025 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
4026 {
4027         return cq->latest_cqe;
4028 }
4029
4030 /* In fmr we need to increase the number of fmr completed counter for the fmr
4031  * algorithm determining whether we can free a pbl or not.
4032  * we need to perform this whether the work request was signaled or not. for
4033  * this purpose we call this function from the condition that checks if a wr
4034  * should be skipped, to make sure we don't miss it ( possibly this fmr
4035  * operation was not signalted)
4036  */
4037 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
4038 {
4039         if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
4040                 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4041 }
4042
4043 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
4044                        struct qedr_cq *cq, int num_entries,
4045                        struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
4046                        int force)
4047 {
4048         u16 cnt = 0;
4049
4050         while (num_entries && qp->sq.wqe_cons != hw_cons) {
4051                 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
4052                         qedr_chk_if_fmr(qp);
4053                         /* skip WC */
4054                         goto next_cqe;
4055                 }
4056
4057                 /* fill WC */
4058                 wc->status = status;
4059                 wc->vendor_err = 0;
4060                 wc->wc_flags = 0;
4061                 wc->src_qp = qp->id;
4062                 wc->qp = &qp->ibqp;
4063
4064                 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
4065                 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
4066
4067                 switch (wc->opcode) {
4068                 case IB_WC_RDMA_WRITE:
4069                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4070                         break;
4071                 case IB_WC_COMP_SWAP:
4072                 case IB_WC_FETCH_ADD:
4073                         wc->byte_len = 8;
4074                         break;
4075                 case IB_WC_REG_MR:
4076                         qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4077                         break;
4078                 case IB_WC_RDMA_READ:
4079                 case IB_WC_SEND:
4080                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4081                         break;
4082                 default:
4083                         break;
4084                 }
4085
4086                 num_entries--;
4087                 wc++;
4088                 cnt++;
4089 next_cqe:
4090                 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
4091                         qed_chain_consume(&qp->sq.pbl);
4092                 qedr_inc_sw_cons(&qp->sq);
4093         }
4094
4095         return cnt;
4096 }
4097
4098 static int qedr_poll_cq_req(struct qedr_dev *dev,
4099                             struct qedr_qp *qp, struct qedr_cq *cq,
4100                             int num_entries, struct ib_wc *wc,
4101                             struct rdma_cqe_requester *req)
4102 {
4103         int cnt = 0;
4104
4105         switch (req->status) {
4106         case RDMA_CQE_REQ_STS_OK:
4107                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4108                                   IB_WC_SUCCESS, 0);
4109                 break;
4110         case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
4111                 if (qp->state != QED_ROCE_QP_STATE_ERR)
4112                         DP_DEBUG(dev, QEDR_MSG_CQ,
4113                                  "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4114                                  cq->icid, qp->icid);
4115                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4116                                   IB_WC_WR_FLUSH_ERR, 1);
4117                 break;
4118         default:
4119                 /* process all WQE before the cosumer */
4120                 qp->state = QED_ROCE_QP_STATE_ERR;
4121                 cnt = process_req(dev, qp, cq, num_entries, wc,
4122                                   req->sq_cons - 1, IB_WC_SUCCESS, 0);
4123                 wc += cnt;
4124                 /* if we have extra WC fill it with actual error info */
4125                 if (cnt < num_entries) {
4126                         enum ib_wc_status wc_status;
4127
4128                         switch (req->status) {
4129                         case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4130                                 DP_ERR(dev,
4131                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4132                                        cq->icid, qp->icid);
4133                                 wc_status = IB_WC_BAD_RESP_ERR;
4134                                 break;
4135                         case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4136                                 DP_ERR(dev,
4137                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4138                                        cq->icid, qp->icid);
4139                                 wc_status = IB_WC_LOC_LEN_ERR;
4140                                 break;
4141                         case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4142                                 DP_ERR(dev,
4143                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4144                                        cq->icid, qp->icid);
4145                                 wc_status = IB_WC_LOC_QP_OP_ERR;
4146                                 break;
4147                         case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4148                                 DP_ERR(dev,
4149                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4150                                        cq->icid, qp->icid);
4151                                 wc_status = IB_WC_LOC_PROT_ERR;
4152                                 break;
4153                         case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4154                                 DP_ERR(dev,
4155                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4156                                        cq->icid, qp->icid);
4157                                 wc_status = IB_WC_MW_BIND_ERR;
4158                                 break;
4159                         case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4160                                 DP_ERR(dev,
4161                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4162                                        cq->icid, qp->icid);
4163                                 wc_status = IB_WC_REM_INV_REQ_ERR;
4164                                 break;
4165                         case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4166                                 DP_ERR(dev,
4167                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4168                                        cq->icid, qp->icid);
4169                                 wc_status = IB_WC_REM_ACCESS_ERR;
4170                                 break;
4171                         case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4172                                 DP_ERR(dev,
4173                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4174                                        cq->icid, qp->icid);
4175                                 wc_status = IB_WC_REM_OP_ERR;
4176                                 break;
4177                         case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4178                                 DP_ERR(dev,
4179                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4180                                        cq->icid, qp->icid);
4181                                 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4182                                 break;
4183                         case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4184                                 DP_ERR(dev,
4185                                        "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4186                                        cq->icid, qp->icid);
4187                                 wc_status = IB_WC_RETRY_EXC_ERR;
4188                                 break;
4189                         default:
4190                                 DP_ERR(dev,
4191                                        "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4192                                        cq->icid, qp->icid);
4193                                 wc_status = IB_WC_GENERAL_ERR;
4194                         }
4195                         cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4196                                            wc_status, 1);
4197                 }
4198         }
4199
4200         return cnt;
4201 }
4202
4203 static inline int qedr_cqe_resp_status_to_ib(u8 status)
4204 {
4205         switch (status) {
4206         case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
4207                 return IB_WC_LOC_ACCESS_ERR;
4208         case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
4209                 return IB_WC_LOC_LEN_ERR;
4210         case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
4211                 return IB_WC_LOC_QP_OP_ERR;
4212         case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
4213                 return IB_WC_LOC_PROT_ERR;
4214         case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
4215                 return IB_WC_MW_BIND_ERR;
4216         case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
4217                 return IB_WC_REM_INV_RD_REQ_ERR;
4218         case RDMA_CQE_RESP_STS_OK:
4219                 return IB_WC_SUCCESS;
4220         default:
4221                 return IB_WC_GENERAL_ERR;
4222         }
4223 }
4224
4225 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4226                                           struct ib_wc *wc)
4227 {
4228         wc->status = IB_WC_SUCCESS;
4229         wc->byte_len = le32_to_cpu(resp->length);
4230
4231         if (resp->flags & QEDR_RESP_IMM) {
4232                 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
4233                 wc->wc_flags |= IB_WC_WITH_IMM;
4234
4235                 if (resp->flags & QEDR_RESP_RDMA)
4236                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4237
4238                 if (resp->flags & QEDR_RESP_INV)
4239                         return -EINVAL;
4240
4241         } else if (resp->flags & QEDR_RESP_INV) {
4242                 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4243                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4244
4245                 if (resp->flags & QEDR_RESP_RDMA)
4246                         return -EINVAL;
4247
4248         } else if (resp->flags & QEDR_RESP_RDMA) {
4249                 return -EINVAL;
4250         }
4251
4252         return 0;
4253 }
4254
4255 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4256                                struct qedr_cq *cq, struct ib_wc *wc,
4257                                struct rdma_cqe_responder *resp, u64 wr_id)
4258 {
4259         /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4260         wc->opcode = IB_WC_RECV;
4261         wc->wc_flags = 0;
4262
4263         if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4264                 if (qedr_set_ok_cqe_resp_wc(resp, wc))
4265                         DP_ERR(dev,
4266                                "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4267                                cq, cq->icid, resp->flags);
4268
4269         } else {
4270                 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4271                 if (wc->status == IB_WC_GENERAL_ERR)
4272                         DP_ERR(dev,
4273                                "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4274                                cq, cq->icid, resp->status);
4275         }
4276
4277         /* Fill the rest of the WC */
4278         wc->vendor_err = 0;
4279         wc->src_qp = qp->id;
4280         wc->qp = &qp->ibqp;
4281         wc->wr_id = wr_id;
4282 }
4283
4284 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4285                                 struct qedr_cq *cq, struct ib_wc *wc,
4286                                 struct rdma_cqe_responder *resp)
4287 {
4288         struct qedr_srq *srq = qp->srq;
4289         u64 wr_id;
4290
4291         wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4292                          le32_to_cpu(resp->srq_wr_id.lo), u64);
4293
4294         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4295                 wc->status = IB_WC_WR_FLUSH_ERR;
4296                 wc->vendor_err = 0;
4297                 wc->wr_id = wr_id;
4298                 wc->byte_len = 0;
4299                 wc->src_qp = qp->id;
4300                 wc->qp = &qp->ibqp;
4301                 wc->wr_id = wr_id;
4302         } else {
4303                 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4304         }
4305         atomic_inc(&srq->hw_srq.wr_cons_cnt);
4306
4307         return 1;
4308 }
4309 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4310                             struct qedr_cq *cq, struct ib_wc *wc,
4311                             struct rdma_cqe_responder *resp)
4312 {
4313         u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4314
4315         __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4316
4317         while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4318                 qed_chain_consume(&qp->rq.pbl);
4319         qedr_inc_sw_cons(&qp->rq);
4320
4321         return 1;
4322 }
4323
4324 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4325                               int num_entries, struct ib_wc *wc, u16 hw_cons)
4326 {
4327         u16 cnt = 0;
4328
4329         while (num_entries && qp->rq.wqe_cons != hw_cons) {
4330                 /* fill WC */
4331                 wc->status = IB_WC_WR_FLUSH_ERR;
4332                 wc->vendor_err = 0;
4333                 wc->wc_flags = 0;
4334                 wc->src_qp = qp->id;
4335                 wc->byte_len = 0;
4336                 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4337                 wc->qp = &qp->ibqp;
4338                 num_entries--;
4339                 wc++;
4340                 cnt++;
4341                 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4342                         qed_chain_consume(&qp->rq.pbl);
4343                 qedr_inc_sw_cons(&qp->rq);
4344         }
4345
4346         return cnt;
4347 }
4348
4349 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4350                                  struct rdma_cqe_responder *resp, int *update)
4351 {
4352         if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4353                 consume_cqe(cq);
4354                 *update |= 1;
4355         }
4356 }
4357
4358 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4359                                  struct qedr_cq *cq, int num_entries,
4360                                  struct ib_wc *wc,
4361                                  struct rdma_cqe_responder *resp)
4362 {
4363         int cnt;
4364
4365         cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4366         consume_cqe(cq);
4367
4368         return cnt;
4369 }
4370
4371 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4372                              struct qedr_cq *cq, int num_entries,
4373                              struct ib_wc *wc, struct rdma_cqe_responder *resp,
4374                              int *update)
4375 {
4376         int cnt;
4377
4378         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4379                 cnt = process_resp_flush(qp, cq, num_entries, wc,
4380                                          resp->rq_cons_or_srq_id);
4381                 try_consume_resp_cqe(cq, qp, resp, update);
4382         } else {
4383                 cnt = process_resp_one(dev, qp, cq, wc, resp);
4384                 consume_cqe(cq);
4385                 *update |= 1;
4386         }
4387
4388         return cnt;
4389 }
4390
4391 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4392                                 struct rdma_cqe_requester *req, int *update)
4393 {
4394         if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4395                 consume_cqe(cq);
4396                 *update |= 1;
4397         }
4398 }
4399
4400 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4401 {
4402         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4403         struct qedr_cq *cq = get_qedr_cq(ibcq);
4404         union rdma_cqe *cqe;
4405         u32 old_cons, new_cons;
4406         unsigned long flags;
4407         int update = 0;
4408         int done = 0;
4409
4410         if (cq->destroyed) {
4411                 DP_ERR(dev,
4412                        "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4413                        cq, cq->icid);
4414                 return 0;
4415         }
4416
4417         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4418                 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4419
4420         spin_lock_irqsave(&cq->cq_lock, flags);
4421         cqe = cq->latest_cqe;
4422         old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4423         while (num_entries && is_valid_cqe(cq, cqe)) {
4424                 struct qedr_qp *qp;
4425                 int cnt = 0;
4426
4427                 /* prevent speculative reads of any field of CQE */
4428                 rmb();
4429
4430                 qp = cqe_get_qp(cqe);
4431                 if (!qp) {
4432                         WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4433                         break;
4434                 }
4435
4436                 wc->qp = &qp->ibqp;
4437
4438                 switch (cqe_get_type(cqe)) {
4439                 case RDMA_CQE_TYPE_REQUESTER:
4440                         cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4441                                                &cqe->req);
4442                         try_consume_req_cqe(cq, qp, &cqe->req, &update);
4443                         break;
4444                 case RDMA_CQE_TYPE_RESPONDER_RQ:
4445                         cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4446                                                 &cqe->resp, &update);
4447                         break;
4448                 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4449                         cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4450                                                     wc, &cqe->resp);
4451                         update = 1;
4452                         break;
4453                 case RDMA_CQE_TYPE_INVALID:
4454                 default:
4455                         DP_ERR(dev, "Error: invalid CQE type = %d\n",
4456                                cqe_get_type(cqe));
4457                 }
4458                 num_entries -= cnt;
4459                 wc += cnt;
4460                 done += cnt;
4461
4462                 cqe = get_cqe(cq);
4463         }
4464         new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4465
4466         cq->cq_cons += new_cons - old_cons;
4467
4468         if (update)
4469                 /* doorbell notifies abount latest VALID entry,
4470                  * but chain already point to the next INVALID one
4471                  */
4472                 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4473
4474         spin_unlock_irqrestore(&cq->cq_lock, flags);
4475         return done;
4476 }
4477
4478 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4479                      u32 port_num, const struct ib_wc *in_wc,
4480                      const struct ib_grh *in_grh, const struct ib_mad *in,
4481                      struct ib_mad *out_mad, size_t *out_mad_size,
4482                      u16 *out_mad_pkey_index)
4483 {
4484         return IB_MAD_RESULT_SUCCESS;
4485 }