drivers/infiniband/hw/qedr/verbs.c

   1 /* QLogic qedr NIC Driver
   2  * Copyright (c) 2015-2016  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/dma-mapping.h>
  33 #include <linux/crc32.h>
  34 #include <net/ip.h>
  35 #include <net/ipv6.h>
  36 #include <net/udp.h>
  37 #include <linux/iommu.h>
  38
  39 #include <rdma/ib_verbs.h>
  40 #include <rdma/ib_user_verbs.h>
  41 #include <rdma/iw_cm.h>
  42 #include <rdma/ib_umem.h>
  43 #include <rdma/ib_addr.h>
  44 #include <rdma/ib_cache.h>
  45
  46 #include <linux/qed/common_hsi.h>
  47 #include "qedr_hsi_rdma.h"
  48 #include <linux/qed/qed_if.h>
  49 #include "qedr.h"
  50 #include "verbs.h"
  51 #include <rdma/qedr-abi.h>
  52 #include "qedr_cm.h"
  53
  54 #define DB_ADDR_SHIFT(addr)             ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
  55
  56 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
  57                                         size_t len)
  58 {
  59         size_t min_len = min_t(size_t, len, udata->outlen);
  60
  61         return ib_copy_to_udata(udata, src, min_len);
  62 }
  63
  64 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
  65 {
  66         if (index > QEDR_ROCE_PKEY_TABLE_LEN)
  67                 return -EINVAL;
  68
  69         *pkey = QEDR_ROCE_PKEY_DEFAULT;
  70         return 0;
  71 }
  72
  73 int qedr_query_gid(struct ib_device *ibdev, u8 port, int index,
  74                    union ib_gid *sgid)
  75 {
  76         struct qedr_dev *dev = get_qedr_dev(ibdev);
  77         int rc = 0;
  78
  79         if (!rdma_cap_roce_gid_table(ibdev, port))
  80                 return -ENODEV;
  81
  82         rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
  83         if (rc == -EAGAIN) {
  84                 memcpy(sgid, &zgid, sizeof(*sgid));
  85                 return 0;
  86         }
  87
  88         DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index,
  89                  sgid->global.interface_id, sgid->global.subnet_prefix);
  90
  91         return rc;
  92 }
  93
  94 int qedr_add_gid(struct ib_device *device, u8 port_num,
  95                  unsigned int index, const union ib_gid *gid,
  96                  const struct ib_gid_attr *attr, void **context)
  97 {
  98         if (!rdma_cap_roce_gid_table(device, port_num))
  99                 return -EINVAL;
 100
 101         if (port_num > QEDR_MAX_PORT)
 102                 return -EINVAL;
 103
 104         if (!context)
 105                 return -EINVAL;
 106
 107         return 0;
 108 }
 109
 110 int qedr_del_gid(struct ib_device *device, u8 port_num,
 111                  unsigned int index, void **context)
 112 {
 113         if (!rdma_cap_roce_gid_table(device, port_num))
 114                 return -EINVAL;
 115
 116         if (port_num > QEDR_MAX_PORT)
 117                 return -EINVAL;
 118
 119         if (!context)
 120                 return -EINVAL;
 121
 122         return 0;
 123 }
 124
 125 int qedr_query_device(struct ib_device *ibdev,
 126                       struct ib_device_attr *attr, struct ib_udata *udata)
 127 {
 128         struct qedr_dev *dev = get_qedr_dev(ibdev);
 129         struct qedr_device_attr *qattr = &dev->attr;
 130
 131         if (!dev->rdma_ctx) {
 132                 DP_ERR(dev,
 133                        "qedr_query_device called with invalid params rdma_ctx=%p\n",
 134                        dev->rdma_ctx);
 135                 return -EINVAL;
 136         }
 137
 138         memset(attr, 0, sizeof(*attr));
 139
 140         attr->fw_ver = qattr->fw_ver;
 141         attr->sys_image_guid = qattr->sys_image_guid;
 142         attr->max_mr_size = qattr->max_mr_size;
 143         attr->page_size_cap = qattr->page_size_caps;
 144         attr->vendor_id = qattr->vendor_id;
 145         attr->vendor_part_id = qattr->vendor_part_id;
 146         attr->hw_ver = qattr->hw_ver;
 147         attr->max_qp = qattr->max_qp;
 148         attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
 149         attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
 150             IB_DEVICE_RC_RNR_NAK_GEN |
 151             IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
 152
 153         attr->max_sge = qattr->max_sge;
 154         attr->max_sge_rd = qattr->max_sge;
 155         attr->max_cq = qattr->max_cq;
 156         attr->max_cqe = qattr->max_cqe;
 157         attr->max_mr = qattr->max_mr;
 158         attr->max_mw = qattr->max_mw;
 159         attr->max_pd = qattr->max_pd;
 160         attr->atomic_cap = dev->atomic_cap;
 161         attr->max_fmr = qattr->max_fmr;
 162         attr->max_map_per_fmr = 16;
 163         attr->max_qp_init_rd_atom =
 164             1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
 165         attr->max_qp_rd_atom =
 166             min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
 167                 attr->max_qp_init_rd_atom);
 168
 169         attr->max_srq = qattr->max_srq;
 170         attr->max_srq_sge = qattr->max_srq_sge;
 171         attr->max_srq_wr = qattr->max_srq_wr;
 172
 173         attr->local_ca_ack_delay = qattr->dev_ack_delay;
 174         attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
 175         attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
 176         attr->max_ah = qattr->max_ah;
 177
 178         return 0;
 179 }
 180
 181 #define QEDR_SPEED_SDR          (1)
 182 #define QEDR_SPEED_DDR          (2)
 183 #define QEDR_SPEED_QDR          (4)
 184 #define QEDR_SPEED_FDR10        (8)
 185 #define QEDR_SPEED_FDR          (16)
 186 #define QEDR_SPEED_EDR          (32)
 187
 188 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
 189                                             u8 *ib_width)
 190 {
 191         switch (speed) {
 192         case 1000:
 193                 *ib_speed = QEDR_SPEED_SDR;
 194                 *ib_width = IB_WIDTH_1X;
 195                 break;
 196         case 10000:
 197                 *ib_speed = QEDR_SPEED_QDR;
 198                 *ib_width = IB_WIDTH_1X;
 199                 break;
 200
 201         case 20000:
 202                 *ib_speed = QEDR_SPEED_DDR;
 203                 *ib_width = IB_WIDTH_4X;
 204                 break;
 205
 206         case 25000:
 207                 *ib_speed = QEDR_SPEED_EDR;
 208                 *ib_width = IB_WIDTH_1X;
 209                 break;
 210
 211         case 40000:
 212                 *ib_speed = QEDR_SPEED_QDR;
 213                 *ib_width = IB_WIDTH_4X;
 214                 break;
 215
 216         case 50000:
 217                 *ib_speed = QEDR_SPEED_QDR;
 218                 *ib_width = IB_WIDTH_4X;
 219                 break;
 220
 221         case 100000:
 222                 *ib_speed = QEDR_SPEED_EDR;
 223                 *ib_width = IB_WIDTH_4X;
 224                 break;
 225
 226         default:
 227                 /* Unsupported */
 228                 *ib_speed = QEDR_SPEED_SDR;
 229                 *ib_width = IB_WIDTH_1X;
 230         }
 231 }
 232
 233 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
 234 {
 235         struct qedr_dev *dev;
 236         struct qed_rdma_port *rdma_port;
 237
 238         dev = get_qedr_dev(ibdev);
 239         if (port > 1) {
 240                 DP_ERR(dev, "invalid_port=0x%x\n", port);
 241                 return -EINVAL;
 242         }
 243
 244         if (!dev->rdma_ctx) {
 245                 DP_ERR(dev, "rdma_ctx is NULL\n");
 246                 return -EINVAL;
 247         }
 248
 249         rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
 250
 251         /* *attr being zeroed by the caller, avoid zeroing it here */
 252         if (rdma_port->port_state == QED_RDMA_PORT_UP) {
 253                 attr->state = IB_PORT_ACTIVE;
 254                 attr->phys_state = 5;
 255         } else {
 256                 attr->state = IB_PORT_DOWN;
 257                 attr->phys_state = 3;
 258         }
 259         attr->max_mtu = IB_MTU_4096;
 260         attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
 261         attr->lid = 0;
 262         attr->lmc = 0;
 263         attr->sm_lid = 0;
 264         attr->sm_sl = 0;
 265         attr->port_cap_flags = IB_PORT_IP_BASED_GIDS;
 266         attr->gid_tbl_len = QEDR_MAX_SGID;
 267         attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
 268         attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
 269         attr->qkey_viol_cntr = 0;
 270         get_link_speed_and_width(rdma_port->link_speed,
 271                                  &attr->active_speed, &attr->active_width);
 272         attr->max_msg_sz = rdma_port->max_msg_size;
 273         attr->max_vl_num = 4;
 274
 275         return 0;
 276 }
 277
 278 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
 279                      struct ib_port_modify *props)
 280 {
 281         struct qedr_dev *dev;
 282
 283         dev = get_qedr_dev(ibdev);
 284         if (port > 1) {
 285                 DP_ERR(dev, "invalid_port=0x%x\n", port);
 286                 return -EINVAL;
 287         }
 288
 289         return 0;
 290 }
 291
 292 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
 293                          unsigned long len)
 294 {
 295         struct qedr_mm *mm;
 296
 297         mm = kzalloc(sizeof(*mm), GFP_KERNEL);
 298         if (!mm)
 299                 return -ENOMEM;
 300
 301         mm->key.phy_addr = phy_addr;
 302         /* This function might be called with a length which is not a multiple
 303          * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
 304          * forces this granularity by increasing the requested size if needed.
 305          * When qedr_mmap is called, it will search the list with the updated
 306          * length as a key. To prevent search failures, the length is rounded up
 307          * in advance to PAGE_SIZE.
 308          */
 309         mm->key.len = roundup(len, PAGE_SIZE);
 310         INIT_LIST_HEAD(&mm->entry);
 311
 312         mutex_lock(&uctx->mm_list_lock);
 313         list_add(&mm->entry, &uctx->mm_head);
 314         mutex_unlock(&uctx->mm_list_lock);
 315
 316         DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
 317                  "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
 318                  (unsigned long long)mm->key.phy_addr,
 319                  (unsigned long)mm->key.len, uctx);
 320
 321         return 0;
 322 }
 323
 324 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
 325                              unsigned long len)
 326 {
 327         bool found = false;
 328         struct qedr_mm *mm;
 329
 330         mutex_lock(&uctx->mm_list_lock);
 331         list_for_each_entry(mm, &uctx->mm_head, entry) {
 332                 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
 333                         continue;
 334
 335                 found = true;
 336                 break;
 337         }
 338         mutex_unlock(&uctx->mm_list_lock);
 339         DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
 340                  "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
 341                  mm->key.phy_addr, mm->key.len, uctx, found);
 342
 343         return found;
 344 }
 345
 346 struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
 347                                         struct ib_udata *udata)
 348 {
 349         int rc;
 350         struct qedr_ucontext *ctx;
 351         struct qedr_alloc_ucontext_resp uresp;
 352         struct qedr_dev *dev = get_qedr_dev(ibdev);
 353         struct qed_rdma_add_user_out_params oparams;
 354
 355         if (!udata)
 356                 return ERR_PTR(-EFAULT);
 357
 358         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 359         if (!ctx)
 360                 return ERR_PTR(-ENOMEM);
 361
 362         rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
 363         if (rc) {
 364                 DP_ERR(dev,
 365                        "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
 366                        rc);
 367                 goto err;
 368         }
 369
 370         ctx->dpi = oparams.dpi;
 371         ctx->dpi_addr = oparams.dpi_addr;
 372         ctx->dpi_phys_addr = oparams.dpi_phys_addr;
 373         ctx->dpi_size = oparams.dpi_size;
 374         INIT_LIST_HEAD(&ctx->mm_head);
 375         mutex_init(&ctx->mm_list_lock);
 376
 377         memset(&uresp, 0, sizeof(uresp));
 378
 379         uresp.db_pa = ctx->dpi_phys_addr;
 380         uresp.db_size = ctx->dpi_size;
 381         uresp.max_send_wr = dev->attr.max_sqe;
 382         uresp.max_recv_wr = dev->attr.max_rqe;
 383         uresp.max_srq_wr = dev->attr.max_srq_wr;
 384         uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
 385         uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
 386         uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
 387         uresp.max_cqes = QEDR_MAX_CQES;
 388
 389         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 390         if (rc)
 391                 goto err;
 392
 393         ctx->dev = dev;
 394
 395         rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
 396         if (rc)
 397                 goto err;
 398
 399         DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
 400                  &ctx->ibucontext);
 401         return &ctx->ibucontext;
 402
 403 err:
 404         kfree(ctx);
 405         return ERR_PTR(rc);
 406 }
 407
 408 int qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
 409 {
 410         struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
 411         struct qedr_mm *mm, *tmp;
 412         int status = 0;
 413
 414         DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
 415                  uctx);
 416         uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
 417
 418         list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
 419                 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
 420                          "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
 421                          mm->key.phy_addr, mm->key.len, uctx);
 422                 list_del(&mm->entry);
 423                 kfree(mm);
 424         }
 425
 426         kfree(uctx);
 427         return status;
 428 }
 429
 430 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 431 {
 432         struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
 433         struct qedr_dev *dev = get_qedr_dev(context->device);
 434         unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
 435         u64 unmapped_db = dev->db_phys_addr;
 436         unsigned long len = (vma->vm_end - vma->vm_start);
 437         int rc = 0;
 438         bool found;
 439
 440         DP_DEBUG(dev, QEDR_MSG_INIT,
 441                  "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
 442                  vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len);
 443         if (vma->vm_start & (PAGE_SIZE - 1)) {
 444                 DP_ERR(dev, "Vma_start not page aligned = %ld\n",
 445                        vma->vm_start);
 446                 return -EINVAL;
 447         }
 448
 449         found = qedr_search_mmap(ucontext, vm_page, len);
 450         if (!found) {
 451                 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n",
 452                        vma->vm_pgoff);
 453                 return -EINVAL;
 454         }
 455
 456         DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
 457
 458         if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
 459                                                      dev->db_size))) {
 460                 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
 461                 if (vma->vm_flags & VM_READ) {
 462                         DP_ERR(dev, "Trying to map doorbell bar for read\n");
 463                         return -EPERM;
 464                 }
 465
 466                 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 467
 468                 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
 469                                         PAGE_SIZE, vma->vm_page_prot);
 470         } else {
 471                 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n");
 472                 rc = remap_pfn_range(vma, vma->vm_start,
 473                                      vma->vm_pgoff, len, vma->vm_page_prot);
 474         }
 475         DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc);
 476         return rc;
 477 }
 478
 479 struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
 480                             struct ib_ucontext *context, struct ib_udata *udata)
 481 {
 482         struct qedr_dev *dev = get_qedr_dev(ibdev);
 483         struct qedr_pd *pd;
 484         u16 pd_id;
 485         int rc;
 486
 487         DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
 488                  (udata && context) ? "User Lib" : "Kernel");
 489
 490         if (!dev->rdma_ctx) {
 491                 DP_ERR(dev, "invlaid RDMA context\n");
 492                 return ERR_PTR(-EINVAL);
 493         }
 494
 495         pd = kzalloc(sizeof(*pd), GFP_KERNEL);
 496         if (!pd)
 497                 return ERR_PTR(-ENOMEM);
 498
 499         rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
 500         if (rc)
 501                 goto err;
 502
 503         pd->pd_id = pd_id;
 504
 505         if (udata && context) {
 506                 struct qedr_alloc_pd_uresp uresp;
 507
 508                 uresp.pd_id = pd_id;
 509
 510                 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 511                 if (rc) {
 512                         DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
 513                         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
 514                         goto err;
 515                 }
 516
 517                 pd->uctx = get_qedr_ucontext(context);
 518                 pd->uctx->pd = pd;
 519         }
 520
 521         return &pd->ibpd;
 522
 523 err:
 524         kfree(pd);
 525         return ERR_PTR(rc);
 526 }
 527
 528 int qedr_dealloc_pd(struct ib_pd *ibpd)
 529 {
 530         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
 531         struct qedr_pd *pd = get_qedr_pd(ibpd);
 532
 533         if (!pd) {
 534                 pr_err("Invalid PD received in dealloc_pd\n");
 535                 return -EINVAL;
 536         }
 537
 538         DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
 539         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
 540
 541         kfree(pd);
 542
 543         return 0;
 544 }
 545
 546 static void qedr_free_pbl(struct qedr_dev *dev,
 547                           struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
 548 {
 549         struct pci_dev *pdev = dev->pdev;
 550         int i;
 551
 552         for (i = 0; i < pbl_info->num_pbls; i++) {
 553                 if (!pbl[i].va)
 554                         continue;
 555                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 556                                   pbl[i].va, pbl[i].pa);
 557         }
 558
 559         kfree(pbl);
 560 }
 561
 562 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
 563 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
 564
 565 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
 566 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
 567 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
 568
 569 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
 570                                            struct qedr_pbl_info *pbl_info,
 571                                            gfp_t flags)
 572 {
 573         struct pci_dev *pdev = dev->pdev;
 574         struct qedr_pbl *pbl_table;
 575         dma_addr_t *pbl_main_tbl;
 576         dma_addr_t pa;
 577         void *va;
 578         int i;
 579
 580         pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
 581         if (!pbl_table)
 582                 return ERR_PTR(-ENOMEM);
 583
 584         for (i = 0; i < pbl_info->num_pbls; i++) {
 585                 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size,
 586                                         &pa, flags);
 587                 if (!va)
 588                         goto err;
 589
 590                 memset(va, 0, pbl_info->pbl_size);
 591                 pbl_table[i].va = va;
 592                 pbl_table[i].pa = pa;
 593         }
 594
 595         /* Two-Layer PBLs, if we have more than one pbl we need to initialize
 596          * the first one with physical pointers to all of the rest
 597          */
 598         pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
 599         for (i = 0; i < pbl_info->num_pbls - 1; i++)
 600                 pbl_main_tbl[i] = pbl_table[i + 1].pa;
 601
 602         return pbl_table;
 603
 604 err:
 605         for (i--; i >= 0; i--)
 606                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 607                                   pbl_table[i].va, pbl_table[i].pa);
 608
 609         qedr_free_pbl(dev, pbl_info, pbl_table);
 610
 611         return ERR_PTR(-ENOMEM);
 612 }
 613
 614 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
 615                                 struct qedr_pbl_info *pbl_info,
 616                                 u32 num_pbes, int two_layer_capable)
 617 {
 618         u32 pbl_capacity;
 619         u32 pbl_size;
 620         u32 num_pbls;
 621
 622         if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
 623                 if (num_pbes > MAX_PBES_TWO_LAYER) {
 624                         DP_ERR(dev, "prepare pbl table: too many pages %d\n",
 625                                num_pbes);
 626                         return -EINVAL;
 627                 }
 628
 629                 /* calculate required pbl page size */
 630                 pbl_size = MIN_FW_PBL_PAGE_SIZE;
 631                 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
 632                                NUM_PBES_ON_PAGE(pbl_size);
 633
 634                 while (pbl_capacity < num_pbes) {
 635                         pbl_size *= 2;
 636                         pbl_capacity = pbl_size / sizeof(u64);
 637                         pbl_capacity = pbl_capacity * pbl_capacity;
 638                 }
 639
 640                 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
 641                 num_pbls++;     /* One for the layer0 ( points to the pbls) */
 642                 pbl_info->two_layered = true;
 643         } else {
 644                 /* One layered PBL */
 645                 num_pbls = 1;
 646                 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
 647                                  roundup_pow_of_two((num_pbes * sizeof(u64))));
 648                 pbl_info->two_layered = false;
 649         }
 650
 651         pbl_info->num_pbls = num_pbls;
 652         pbl_info->pbl_size = pbl_size;
 653         pbl_info->num_pbes = num_pbes;
 654
 655         DP_DEBUG(dev, QEDR_MSG_MR,
 656                  "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
 657                  pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
 658
 659         return 0;
 660 }
 661
 662 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
 663                                struct qedr_pbl *pbl,
 664                                struct qedr_pbl_info *pbl_info, u32 pg_shift)
 665 {
 666         int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
 667         u32 fw_pg_cnt, fw_pg_per_umem_pg;
 668         struct qedr_pbl *pbl_tbl;
 669         struct scatterlist *sg;
 670         struct regpair *pbe;
 671         u64 pg_addr;
 672         int entry;
 673
 674         if (!pbl_info->num_pbes)
 675                 return;
 676
 677         /* If we have a two layered pbl, the first pbl points to the rest
 678          * of the pbls and the first entry lays on the second pbl in the table
 679          */
 680         if (pbl_info->two_layered)
 681                 pbl_tbl = &pbl[1];
 682         else
 683                 pbl_tbl = pbl;
 684
 685         pbe = (struct regpair *)pbl_tbl->va;
 686         if (!pbe) {
 687                 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
 688                 return;
 689         }
 690
 691         pbe_cnt = 0;
 692
 693         shift = umem->page_shift;
 694
 695         fw_pg_per_umem_pg = BIT(umem->page_shift - pg_shift);
 696
 697         for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
 698                 pages = sg_dma_len(sg) >> shift;
 699                 pg_addr = sg_dma_address(sg);
 700                 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
 701                         for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
 702                                 pbe->lo = cpu_to_le32(pg_addr);
 703                                 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
 704
 705                                 pg_addr += BIT(pg_shift);
 706                                 pbe_cnt++;
 707                                 total_num_pbes++;
 708                                 pbe++;
 709
 710                                 if (total_num_pbes == pbl_info->num_pbes)
 711                                         return;
 712
 713                                 /* If the given pbl is full storing the pbes,
 714                                  * move to next pbl.
 715                                  */
 716                                 if (pbe_cnt ==
 717                                     (pbl_info->pbl_size / sizeof(u64))) {
 718                                         pbl_tbl++;
 719                                         pbe = (struct regpair *)pbl_tbl->va;
 720                                         pbe_cnt = 0;
 721                                 }
 722
 723                                 fw_pg_cnt++;
 724                         }
 725                 }
 726         }
 727 }
 728
 729 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
 730                               struct qedr_cq *cq, struct ib_udata *udata)
 731 {
 732         struct qedr_create_cq_uresp uresp;
 733         int rc;
 734
 735         memset(&uresp, 0, sizeof(uresp));
 736
 737         uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 738         uresp.icid = cq->icid;
 739
 740         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 741         if (rc)
 742                 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
 743
 744         return rc;
 745 }
 746
 747 static void consume_cqe(struct qedr_cq *cq)
 748 {
 749         if (cq->latest_cqe == cq->toggle_cqe)
 750                 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 751
 752         cq->latest_cqe = qed_chain_consume(&cq->pbl);
 753 }
 754
 755 static inline int qedr_align_cq_entries(int entries)
 756 {
 757         u64 size, aligned_size;
 758
 759         /* We allocate an extra entry that we don't report to the FW. */
 760         size = (entries + 1) * QEDR_CQE_SIZE;
 761         aligned_size = ALIGN(size, PAGE_SIZE);
 762
 763         return aligned_size / QEDR_CQE_SIZE;
 764 }
 765
 766 static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
 767                                        struct qedr_dev *dev,
 768                                        struct qedr_userq *q,
 769                                        u64 buf_addr, size_t buf_len,
 770                                        int access, int dmasync)
 771 {
 772         u32 fw_pages;
 773         int rc;
 774
 775         q->buf_addr = buf_addr;
 776         q->buf_len = buf_len;
 777         q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
 778         if (IS_ERR(q->umem)) {
 779                 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
 780                        PTR_ERR(q->umem));
 781                 return PTR_ERR(q->umem);
 782         }
 783
 784         fw_pages = ib_umem_page_count(q->umem) <<
 785             (q->umem->page_shift - FW_PAGE_SHIFT);
 786
 787         rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
 788         if (rc)
 789                 goto err0;
 790
 791         q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
 792         if (IS_ERR(q->pbl_tbl)) {
 793                 rc = PTR_ERR(q->pbl_tbl);
 794                 goto err0;
 795         }
 796
 797                 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
 798                                    FW_PAGE_SHIFT);
 799
 800         return 0;
 801
 802 err0:
 803         ib_umem_release(q->umem);
 804
 805         return rc;
 806 }
 807
 808 static inline void qedr_init_cq_params(struct qedr_cq *cq,
 809                                        struct qedr_ucontext *ctx,
 810                                        struct qedr_dev *dev, int vector,
 811                                        int chain_entries, int page_cnt,
 812                                        u64 pbl_ptr,
 813                                        struct qed_rdma_create_cq_in_params
 814                                        *params)
 815 {
 816         memset(params, 0, sizeof(*params));
 817         params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
 818         params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
 819         params->cnq_id = vector;
 820         params->cq_size = chain_entries - 1;
 821         params->dpi = (ctx) ? ctx->dpi : dev->dpi;
 822         params->pbl_num_pages = page_cnt;
 823         params->pbl_ptr = pbl_ptr;
 824         params->pbl_two_level = 0;
 825 }
 826
 827 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
 828 {
 829         /* Flush data before signalling doorbell */
 830         wmb();
 831         cq->db.data.agg_flags = flags;
 832         cq->db.data.value = cpu_to_le32(cons);
 833         writeq(cq->db.raw, cq->db_addr);
 834
 835         /* Make sure write would stick */
 836         mmiowb();
 837 }
 838
 839 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 840 {
 841         struct qedr_cq *cq = get_qedr_cq(ibcq);
 842         unsigned long sflags;
 843         struct qedr_dev *dev;
 844
 845         dev = get_qedr_dev(ibcq->device);
 846
 847         if (cq->destroyed) {
 848                 DP_ERR(dev,
 849                        "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
 850                        cq, cq->icid);
 851                 return -EINVAL;
 852         }
 853
 854
 855         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
 856                 return 0;
 857
 858         spin_lock_irqsave(&cq->cq_lock, sflags);
 859
 860         cq->arm_flags = 0;
 861
 862         if (flags & IB_CQ_SOLICITED)
 863                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
 864
 865         if (flags & IB_CQ_NEXT_COMP)
 866                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
 867
 868         doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
 869
 870         spin_unlock_irqrestore(&cq->cq_lock, sflags);
 871
 872         return 0;
 873 }
 874
 875 struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
 876                              const struct ib_cq_init_attr *attr,
 877                              struct ib_ucontext *ib_ctx, struct ib_udata *udata)
 878 {
 879         struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
 880         struct qed_rdma_destroy_cq_out_params destroy_oparams;
 881         struct qed_rdma_destroy_cq_in_params destroy_iparams;
 882         struct qedr_dev *dev = get_qedr_dev(ibdev);
 883         struct qed_rdma_create_cq_in_params params;
 884         struct qedr_create_cq_ureq ureq;
 885         int vector = attr->comp_vector;
 886         int entries = attr->cqe;
 887         struct qedr_cq *cq;
 888         int chain_entries;
 889         int page_cnt;
 890         u64 pbl_ptr;
 891         u16 icid;
 892         int rc;
 893
 894         DP_DEBUG(dev, QEDR_MSG_INIT,
 895                  "create_cq: called from %s. entries=%d, vector=%d\n",
 896                  udata ? "User Lib" : "Kernel", entries, vector);
 897
 898         if (entries > QEDR_MAX_CQES) {
 899                 DP_ERR(dev,
 900                        "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
 901                        entries, QEDR_MAX_CQES);
 902                 return ERR_PTR(-EINVAL);
 903         }
 904
 905         chain_entries = qedr_align_cq_entries(entries);
 906         chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
 907
 908         cq = kzalloc(sizeof(*cq), GFP_KERNEL);
 909         if (!cq)
 910                 return ERR_PTR(-ENOMEM);
 911
 912         if (udata) {
 913                 memset(&ureq, 0, sizeof(ureq));
 914                 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
 915                         DP_ERR(dev,
 916                                "create cq: problem copying data from user space\n");
 917                         goto err0;
 918                 }
 919
 920                 if (!ureq.len) {
 921                         DP_ERR(dev,
 922                                "create cq: cannot create a cq with 0 entries\n");
 923                         goto err0;
 924                 }
 925
 926                 cq->cq_type = QEDR_CQ_TYPE_USER;
 927
 928                 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
 929                                           ureq.len, IB_ACCESS_LOCAL_WRITE, 1);
 930                 if (rc)
 931                         goto err0;
 932
 933                 pbl_ptr = cq->q.pbl_tbl->pa;
 934                 page_cnt = cq->q.pbl_info.num_pbes;
 935
 936                 cq->ibcq.cqe = chain_entries;
 937         } else {
 938                 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
 939
 940                 rc = dev->ops->common->chain_alloc(dev->cdev,
 941                                                    QED_CHAIN_USE_TO_CONSUME,
 942                                                    QED_CHAIN_MODE_PBL,
 943                                                    QED_CHAIN_CNT_TYPE_U32,
 944                                                    chain_entries,
 945                                                    sizeof(union rdma_cqe),
 946                                                    &cq->pbl, NULL);
 947                 if (rc)
 948                         goto err1;
 949
 950                 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
 951                 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
 952                 cq->ibcq.cqe = cq->pbl.capacity;
 953         }
 954
 955         qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
 956                             pbl_ptr, &params);
 957
 958         rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
 959         if (rc)
 960                 goto err2;
 961
 962         cq->icid = icid;
 963         cq->sig = QEDR_CQ_MAGIC_NUMBER;
 964         spin_lock_init(&cq->cq_lock);
 965
 966         if (ib_ctx) {
 967                 rc = qedr_copy_cq_uresp(dev, cq, udata);
 968                 if (rc)
 969                         goto err3;
 970         } else {
 971                 /* Generate doorbell address. */
 972                 cq->db_addr = dev->db_addr +
 973                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 974                 cq->db.data.icid = cq->icid;
 975                 cq->db.data.params = DB_AGG_CMD_SET <<
 976                     RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
 977
 978                 /* point to the very last element, passing it we will toggle */
 979                 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
 980                 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 981                 cq->latest_cqe = NULL;
 982                 consume_cqe(cq);
 983                 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
 984         }
 985
 986         DP_DEBUG(dev, QEDR_MSG_CQ,
 987                  "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
 988                  cq->icid, cq, params.cq_size);
 989
 990         return &cq->ibcq;
 991
 992 err3:
 993         destroy_iparams.icid = cq->icid;
 994         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
 995                                   &destroy_oparams);
 996 err2:
 997         if (udata)
 998                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
 999         else
1000                 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1001 err1:
1002         if (udata)
1003                 ib_umem_release(cq->q.umem);
1004 err0:
1005         kfree(cq);
1006         return ERR_PTR(-EINVAL);
1007 }
1008
1009 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
1010 {
1011         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1012         struct qedr_cq *cq = get_qedr_cq(ibcq);
1013
1014         DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1015
1016         return 0;
1017 }
1018
1019 #define QEDR_DESTROY_CQ_MAX_ITERATIONS          (10)
1020 #define QEDR_DESTROY_CQ_ITER_DURATION           (10)
1021
1022 int qedr_destroy_cq(struct ib_cq *ibcq)
1023 {
1024         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1025         struct qed_rdma_destroy_cq_out_params oparams;
1026         struct qed_rdma_destroy_cq_in_params iparams;
1027         struct qedr_cq *cq = get_qedr_cq(ibcq);
1028         int iter;
1029         int rc;
1030
1031         DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1032
1033         cq->destroyed = 1;
1034
1035         /* GSIs CQs are handled by driver, so they don't exist in the FW */
1036         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
1037                 goto done;
1038
1039         iparams.icid = cq->icid;
1040         rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1041         if (rc)
1042                 return rc;
1043
1044         dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1045
1046         if (ibcq->uobject && ibcq->uobject->context) {
1047                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1048                 ib_umem_release(cq->q.umem);
1049         }
1050
1051         /* We don't want the IRQ handler to handle a non-existing CQ so we
1052          * wait until all CNQ interrupts, if any, are received. This will always
1053          * happen and will always happen very fast. If not, then a serious error
1054          * has occured. That is why we can use a long delay.
1055          * We spin for a short time so we don’t lose time on context switching
1056          * in case all the completions are handled in that span. Otherwise
1057          * we sleep for a while and check again. Since the CNQ may be
1058          * associated with (only) the current CPU we use msleep to allow the
1059          * current CPU to be freed.
1060          * The CNQ notification is increased in qedr_irq_handler().
1061          */
1062         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1063         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1064                 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1065                 iter--;
1066         }
1067
1068         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1069         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1070                 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1071                 iter--;
1072         }
1073
1074         if (oparams.num_cq_notif != cq->cnq_notif)
1075                 goto err;
1076
1077         /* Note that we don't need to have explicit code to wait for the
1078          * completion of the event handler because it is invoked from the EQ.
1079          * Since the destroy CQ ramrod has also been received on the EQ we can
1080          * be certain that there's no event handler in process.
1081          */
1082 done:
1083         cq->sig = ~cq->sig;
1084
1085         kfree(cq);
1086
1087         return 0;
1088
1089 err:
1090         DP_ERR(dev,
1091                "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n",
1092                cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif);
1093
1094         return -EINVAL;
1095 }
1096
1097 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1098                                           struct ib_qp_attr *attr,
1099                                           int attr_mask,
1100                                           struct qed_rdma_modify_qp_in_params
1101                                           *qp_params)
1102 {
1103         enum rdma_network_type nw_type;
1104         struct ib_gid_attr gid_attr;
1105         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1106         union ib_gid gid;
1107         u32 ipv4_addr;
1108         int rc = 0;
1109         int i;
1110
1111         rc = ib_get_cached_gid(ibqp->device,
1112                                rdma_ah_get_port_num(&attr->ah_attr),
1113                                grh->sgid_index, &gid, &gid_attr);
1114         if (rc)
1115                 return rc;
1116
1117         if (!memcmp(&gid, &zgid, sizeof(gid)))
1118                 return -ENOENT;
1119
1120         if (gid_attr.ndev) {
1121                 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev);
1122
1123                 dev_put(gid_attr.ndev);
1124                 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid);
1125                 switch (nw_type) {
1126                 case RDMA_NETWORK_IPV6:
1127                         memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1128                                sizeof(qp_params->sgid));
1129                         memcpy(&qp_params->dgid.bytes[0],
1130                                &grh->dgid,
1131                                sizeof(qp_params->dgid));
1132                         qp_params->roce_mode = ROCE_V2_IPV6;
1133                         SET_FIELD(qp_params->modify_flags,
1134                                   QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1135                         break;
1136                 case RDMA_NETWORK_IB:
1137                         memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1138                                sizeof(qp_params->sgid));
1139                         memcpy(&qp_params->dgid.bytes[0],
1140                                &grh->dgid,
1141                                sizeof(qp_params->dgid));
1142                         qp_params->roce_mode = ROCE_V1;
1143                         break;
1144                 case RDMA_NETWORK_IPV4:
1145                         memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1146                         memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1147                         ipv4_addr = qedr_get_ipv4_from_gid(gid.raw);
1148                         qp_params->sgid.ipv4_addr = ipv4_addr;
1149                         ipv4_addr =
1150                             qedr_get_ipv4_from_gid(grh->dgid.raw);
1151                         qp_params->dgid.ipv4_addr = ipv4_addr;
1152                         SET_FIELD(qp_params->modify_flags,
1153                                   QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1154                         qp_params->roce_mode = ROCE_V2_IPV4;
1155                         break;
1156                 }
1157         }
1158
1159         for (i = 0; i < 4; i++) {
1160                 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1161                 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1162         }
1163
1164         if (qp_params->vlan_id >= VLAN_CFI_MASK)
1165                 qp_params->vlan_id = 0;
1166
1167         return 0;
1168 }
1169
1170 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1171                                struct ib_qp_init_attr *attrs)
1172 {
1173         struct qedr_device_attr *qattr = &dev->attr;
1174
1175         /* QP0... attrs->qp_type == IB_QPT_GSI */
1176         if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1177                 DP_DEBUG(dev, QEDR_MSG_QP,
1178                          "create qp: unsupported qp type=0x%x requested\n",
1179                          attrs->qp_type);
1180                 return -EINVAL;
1181         }
1182
1183         if (attrs->cap.max_send_wr > qattr->max_sqe) {
1184                 DP_ERR(dev,
1185                        "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1186                        attrs->cap.max_send_wr, qattr->max_sqe);
1187                 return -EINVAL;
1188         }
1189
1190         if (attrs->cap.max_inline_data > qattr->max_inline) {
1191                 DP_ERR(dev,
1192                        "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1193                        attrs->cap.max_inline_data, qattr->max_inline);
1194                 return -EINVAL;
1195         }
1196
1197         if (attrs->cap.max_send_sge > qattr->max_sge) {
1198                 DP_ERR(dev,
1199                        "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1200                        attrs->cap.max_send_sge, qattr->max_sge);
1201                 return -EINVAL;
1202         }
1203
1204         if (attrs->cap.max_recv_sge > qattr->max_sge) {
1205                 DP_ERR(dev,
1206                        "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1207                        attrs->cap.max_recv_sge, qattr->max_sge);
1208                 return -EINVAL;
1209         }
1210
1211         /* Unprivileged user space cannot create special QP */
1212         if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1213                 DP_ERR(dev,
1214                        "create qp: userspace can't create special QPs of type=0x%x\n",
1215                        attrs->qp_type);
1216                 return -EINVAL;
1217         }
1218
1219         return 0;
1220 }
1221
1222 static void qedr_copy_rq_uresp(struct qedr_create_qp_uresp *uresp,
1223                                struct qedr_qp *qp)
1224 {
1225         uresp->rq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1226         uresp->rq_icid = qp->icid;
1227 }
1228
1229 static void qedr_copy_sq_uresp(struct qedr_create_qp_uresp *uresp,
1230                                struct qedr_qp *qp)
1231 {
1232         uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1233         uresp->sq_icid = qp->icid + 1;
1234 }
1235
1236 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1237                               struct qedr_qp *qp, struct ib_udata *udata)
1238 {
1239         struct qedr_create_qp_uresp uresp;
1240         int rc;
1241
1242         memset(&uresp, 0, sizeof(uresp));
1243         qedr_copy_sq_uresp(&uresp, qp);
1244         qedr_copy_rq_uresp(&uresp, qp);
1245
1246         uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1247         uresp.qp_id = qp->qp_id;
1248
1249         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1250         if (rc)
1251                 DP_ERR(dev,
1252                        "create qp: failed a copy to user space with qp icid=0x%x.\n",
1253                        qp->icid);
1254
1255         return rc;
1256 }
1257
1258 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1259                                       struct qedr_qp *qp,
1260                                       struct qedr_pd *pd,
1261                                       struct ib_qp_init_attr *attrs)
1262 {
1263         spin_lock_init(&qp->q_lock);
1264         qp->pd = pd;
1265         qp->qp_type = attrs->qp_type;
1266         qp->max_inline_data = attrs->cap.max_inline_data;
1267         qp->sq.max_sges = attrs->cap.max_send_sge;
1268         qp->state = QED_ROCE_QP_STATE_RESET;
1269         qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1270         qp->sq_cq = get_qedr_cq(attrs->send_cq);
1271         qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1272         qp->dev = dev;
1273         qp->rq.max_sges = attrs->cap.max_recv_sge;
1274
1275         DP_DEBUG(dev, QEDR_MSG_QP,
1276                  "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1277                  qp->rq.max_sges, qp->rq_cq->icid);
1278         DP_DEBUG(dev, QEDR_MSG_QP,
1279                  "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1280                  pd->pd_id, qp->qp_type, qp->max_inline_data,
1281                  qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1282         DP_DEBUG(dev, QEDR_MSG_QP,
1283                  "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1284                  qp->sq.max_sges, qp->sq_cq->icid);
1285 }
1286
1287 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1288 {
1289         qp->sq.db = dev->db_addr +
1290                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1291         qp->sq.db_data.data.icid = qp->icid + 1;
1292         qp->rq.db = dev->db_addr +
1293                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1294         qp->rq.db_data.data.icid = qp->icid;
1295 }
1296
1297 static inline void
1298 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1299                               struct qedr_pd *pd,
1300                               struct qedr_qp *qp,
1301                               struct ib_qp_init_attr *attrs,
1302                               bool fmr_and_reserved_lkey,
1303                               struct qed_rdma_create_qp_in_params *params)
1304 {
1305         /* QP handle to be written in an async event */
1306         params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1307         params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1308
1309         params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1310         params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1311         params->pd = pd->pd_id;
1312         params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1313         params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1314         params->stats_queue = 0;
1315         params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1316         params->srq_id = 0;
1317         params->use_srq = false;
1318 }
1319
1320 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1321 {
1322         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1323                  "qp=%p. "
1324                  "sq_addr=0x%llx, "
1325                  "sq_len=%zd, "
1326                  "rq_addr=0x%llx, "
1327                  "rq_len=%zd"
1328                  "\n",
1329                  qp,
1330                  qp->usq.buf_addr,
1331                  qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1332 }
1333
1334 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1335 {
1336         if (qp->usq.umem)
1337                 ib_umem_release(qp->usq.umem);
1338         qp->usq.umem = NULL;
1339
1340         if (qp->urq.umem)
1341                 ib_umem_release(qp->urq.umem);
1342         qp->urq.umem = NULL;
1343 }
1344
1345 static int qedr_create_user_qp(struct qedr_dev *dev,
1346                                struct qedr_qp *qp,
1347                                struct ib_pd *ibpd,
1348                                struct ib_udata *udata,
1349                                struct ib_qp_init_attr *attrs)
1350 {
1351         struct qed_rdma_create_qp_in_params in_params;
1352         struct qed_rdma_create_qp_out_params out_params;
1353         struct qedr_pd *pd = get_qedr_pd(ibpd);
1354         struct ib_ucontext *ib_ctx = NULL;
1355         struct qedr_ucontext *ctx = NULL;
1356         struct qedr_create_qp_ureq ureq;
1357         int rc = -EINVAL;
1358
1359         ib_ctx = ibpd->uobject->context;
1360         ctx = get_qedr_ucontext(ib_ctx);
1361
1362         memset(&ureq, 0, sizeof(ureq));
1363         rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1364         if (rc) {
1365                 DP_ERR(dev, "Problem copying data from user space\n");
1366                 return rc;
1367         }
1368
1369         /* SQ - read access only (0), dma sync not required (0) */
1370         rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
1371                                   ureq.sq_len, 0, 0);
1372         if (rc)
1373                 return rc;
1374
1375         /* RQ - read access only (0), dma sync not required (0) */
1376         rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
1377                                   ureq.rq_len, 0, 0);
1378
1379         if (rc)
1380                 return rc;
1381
1382         memset(&in_params, 0, sizeof(in_params));
1383         qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1384         in_params.qp_handle_lo = ureq.qp_handle_lo;
1385         in_params.qp_handle_hi = ureq.qp_handle_hi;
1386         in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1387         in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1388         in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1389         in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1390
1391         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1392                                               &in_params, &out_params);
1393
1394         if (!qp->qed_qp) {
1395                 rc = -ENOMEM;
1396                 goto err1;
1397         }
1398
1399         qp->qp_id = out_params.qp_id;
1400         qp->icid = out_params.icid;
1401
1402         rc = qedr_copy_qp_uresp(dev, qp, udata);
1403         if (rc)
1404                 goto err;
1405
1406         qedr_qp_user_print(dev, qp);
1407
1408         return 0;
1409 err:
1410         rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1411         if (rc)
1412                 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1413
1414 err1:
1415         qedr_cleanup_user(dev, qp);
1416         return rc;
1417 }
1418
1419 static int
1420 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1421                            struct qedr_qp *qp,
1422                            struct qed_rdma_create_qp_in_params *in_params,
1423                            u32 n_sq_elems, u32 n_rq_elems)
1424 {
1425         struct qed_rdma_create_qp_out_params out_params;
1426         int rc;
1427
1428         rc = dev->ops->common->chain_alloc(dev->cdev,
1429                                            QED_CHAIN_USE_TO_PRODUCE,
1430                                            QED_CHAIN_MODE_PBL,
1431                                            QED_CHAIN_CNT_TYPE_U32,
1432                                            n_sq_elems,
1433                                            QEDR_SQE_ELEMENT_SIZE,
1434                                            &qp->sq.pbl, NULL);
1435
1436         if (rc)
1437                 return rc;
1438
1439         in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1440         in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1441
1442         rc = dev->ops->common->chain_alloc(dev->cdev,
1443                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1444                                            QED_CHAIN_MODE_PBL,
1445                                            QED_CHAIN_CNT_TYPE_U32,
1446                                            n_rq_elems,
1447                                            QEDR_RQE_ELEMENT_SIZE,
1448                                            &qp->rq.pbl, NULL);
1449         if (rc)
1450                 return rc;
1451
1452         in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1453         in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1454
1455         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1456                                               in_params, &out_params);
1457
1458         if (!qp->qed_qp)
1459                 return -EINVAL;
1460
1461         qp->qp_id = out_params.qp_id;
1462         qp->icid = out_params.icid;
1463
1464         qedr_set_roce_db_info(dev, qp);
1465
1466         return 0;
1467 }
1468
1469 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
1470 {
1471         dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1472         kfree(qp->wqe_wr_id);
1473
1474         dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1475         kfree(qp->rqe_wr_id);
1476 }
1477
1478 static int qedr_create_kernel_qp(struct qedr_dev *dev,
1479                                  struct qedr_qp *qp,
1480                                  struct ib_pd *ibpd,
1481                                  struct ib_qp_init_attr *attrs)
1482 {
1483         struct qed_rdma_create_qp_in_params in_params;
1484         struct qedr_pd *pd = get_qedr_pd(ibpd);
1485         int rc = -EINVAL;
1486         u32 n_rq_elems;
1487         u32 n_sq_elems;
1488         u32 n_sq_entries;
1489
1490         memset(&in_params, 0, sizeof(in_params));
1491
1492         /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1493          * the ring. The ring should allow at least a single WR, even if the
1494          * user requested none, due to allocation issues.
1495          * We should add an extra WR since the prod and cons indices of
1496          * wqe_wr_id are managed in such a way that the WQ is considered full
1497          * when (prod+1)%max_wr==cons. We currently don't do that because we
1498          * double the number of entries due an iSER issue that pushes far more
1499          * WRs than indicated. If we decline its ib_post_send() then we get
1500          * error prints in the dmesg we'd like to avoid.
1501          */
1502         qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1503                               dev->attr.max_sqe);
1504
1505         qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id),
1506                                 GFP_KERNEL);
1507         if (!qp->wqe_wr_id) {
1508                 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1509                 return -ENOMEM;
1510         }
1511
1512         /* QP handle to be written in CQE */
1513         in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1514         in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
1515
1516         /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1517          * the ring. There ring should allow at least a single WR, even if the
1518          * user requested none, due to allocation issues.
1519          */
1520         qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
1521
1522         /* Allocate driver internal RQ array */
1523         qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id),
1524                                 GFP_KERNEL);
1525         if (!qp->rqe_wr_id) {
1526                 DP_ERR(dev,
1527                        "create qp: failed RQ shadow memory allocation\n");
1528                 kfree(qp->wqe_wr_id);
1529                 return -ENOMEM;
1530         }
1531
1532         qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
1533
1534         n_sq_entries = attrs->cap.max_send_wr;
1535         n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1536         n_sq_entries = max_t(u32, n_sq_entries, 1);
1537         n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1538
1539         n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1540
1541         rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1542                                         n_sq_elems, n_rq_elems);
1543         if (rc)
1544                 qedr_cleanup_kernel(dev, qp);
1545
1546         return rc;
1547 }
1548
1549 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1550                              struct ib_qp_init_attr *attrs,
1551                              struct ib_udata *udata)
1552 {
1553         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1554         struct qedr_pd *pd = get_qedr_pd(ibpd);
1555         struct qedr_qp *qp;
1556         struct ib_qp *ibqp;
1557         int rc = 0;
1558
1559         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1560                  udata ? "user library" : "kernel", pd);
1561
1562         rc = qedr_check_qp_attrs(ibpd, dev, attrs);
1563         if (rc)
1564                 return ERR_PTR(rc);
1565
1566         if (attrs->srq)
1567                 return ERR_PTR(-EINVAL);
1568
1569         DP_DEBUG(dev, QEDR_MSG_QP,
1570                  "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1571                  udata ? "user library" : "kernel", attrs->event_handler, pd,
1572                  get_qedr_cq(attrs->send_cq),
1573                  get_qedr_cq(attrs->send_cq)->icid,
1574                  get_qedr_cq(attrs->recv_cq),
1575                  get_qedr_cq(attrs->recv_cq)->icid);
1576
1577         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1578         if (!qp) {
1579                 DP_ERR(dev, "create qp: failed allocating memory\n");
1580                 return ERR_PTR(-ENOMEM);
1581         }
1582
1583         qedr_set_common_qp_params(dev, qp, pd, attrs);
1584
1585         if (attrs->qp_type == IB_QPT_GSI) {
1586                 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1587                 if (IS_ERR(ibqp))
1588                         kfree(qp);
1589                 return ibqp;
1590         }
1591
1592         if (udata)
1593                 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
1594         else
1595                 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
1596
1597         if (rc)
1598                 goto err;
1599
1600         qp->ibqp.qp_num = qp->qp_id;
1601
1602         return &qp->ibqp;
1603
1604 err:
1605         kfree(qp);
1606
1607         return ERR_PTR(-EFAULT);
1608 }
1609
1610 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
1611 {
1612         switch (qp_state) {
1613         case QED_ROCE_QP_STATE_RESET:
1614                 return IB_QPS_RESET;
1615         case QED_ROCE_QP_STATE_INIT:
1616                 return IB_QPS_INIT;
1617         case QED_ROCE_QP_STATE_RTR:
1618                 return IB_QPS_RTR;
1619         case QED_ROCE_QP_STATE_RTS:
1620                 return IB_QPS_RTS;
1621         case QED_ROCE_QP_STATE_SQD:
1622                 return IB_QPS_SQD;
1623         case QED_ROCE_QP_STATE_ERR:
1624                 return IB_QPS_ERR;
1625         case QED_ROCE_QP_STATE_SQE:
1626                 return IB_QPS_SQE;
1627         }
1628         return IB_QPS_ERR;
1629 }
1630
1631 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
1632                                         enum ib_qp_state qp_state)
1633 {
1634         switch (qp_state) {
1635         case IB_QPS_RESET:
1636                 return QED_ROCE_QP_STATE_RESET;
1637         case IB_QPS_INIT:
1638                 return QED_ROCE_QP_STATE_INIT;
1639         case IB_QPS_RTR:
1640                 return QED_ROCE_QP_STATE_RTR;
1641         case IB_QPS_RTS:
1642                 return QED_ROCE_QP_STATE_RTS;
1643         case IB_QPS_SQD:
1644                 return QED_ROCE_QP_STATE_SQD;
1645         case IB_QPS_ERR:
1646                 return QED_ROCE_QP_STATE_ERR;
1647         default:
1648                 return QED_ROCE_QP_STATE_ERR;
1649         }
1650 }
1651
1652 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1653 {
1654         qed_chain_reset(&qph->pbl);
1655         qph->prod = 0;
1656         qph->cons = 0;
1657         qph->wqe_cons = 0;
1658         qph->db_data.data.value = cpu_to_le16(0);
1659 }
1660
1661 static int qedr_update_qp_state(struct qedr_dev *dev,
1662                                 struct qedr_qp *qp,
1663                                 enum qed_roce_qp_state new_state)
1664 {
1665         int status = 0;
1666
1667         if (new_state == qp->state)
1668                 return 0;
1669
1670         switch (qp->state) {
1671         case QED_ROCE_QP_STATE_RESET:
1672                 switch (new_state) {
1673                 case QED_ROCE_QP_STATE_INIT:
1674                         qp->prev_wqe_size = 0;
1675                         qedr_reset_qp_hwq_info(&qp->sq);
1676                         qedr_reset_qp_hwq_info(&qp->rq);
1677                         break;
1678                 default:
1679                         status = -EINVAL;
1680                         break;
1681                 };
1682                 break;
1683         case QED_ROCE_QP_STATE_INIT:
1684                 switch (new_state) {
1685                 case QED_ROCE_QP_STATE_RTR:
1686                         /* Update doorbell (in case post_recv was
1687                          * done before move to RTR)
1688                          */
1689                         wmb();
1690                         writel(qp->rq.db_data.raw, qp->rq.db);
1691                         /* Make sure write takes effect */
1692                         mmiowb();
1693                         break;
1694                 case QED_ROCE_QP_STATE_ERR:
1695                         break;
1696                 default:
1697                         /* Invalid state change. */
1698                         status = -EINVAL;
1699                         break;
1700                 };
1701                 break;
1702         case QED_ROCE_QP_STATE_RTR:
1703                 /* RTR->XXX */
1704                 switch (new_state) {
1705                 case QED_ROCE_QP_STATE_RTS:
1706                         break;
1707                 case QED_ROCE_QP_STATE_ERR:
1708                         break;
1709                 default:
1710                         /* Invalid state change. */
1711                         status = -EINVAL;
1712                         break;
1713                 };
1714                 break;
1715         case QED_ROCE_QP_STATE_RTS:
1716                 /* RTS->XXX */
1717                 switch (new_state) {
1718                 case QED_ROCE_QP_STATE_SQD:
1719                         break;
1720                 case QED_ROCE_QP_STATE_ERR:
1721                         break;
1722                 default:
1723                         /* Invalid state change. */
1724                         status = -EINVAL;
1725                         break;
1726                 };
1727                 break;
1728         case QED_ROCE_QP_STATE_SQD:
1729                 /* SQD->XXX */
1730                 switch (new_state) {
1731                 case QED_ROCE_QP_STATE_RTS:
1732                 case QED_ROCE_QP_STATE_ERR:
1733                         break;
1734                 default:
1735                         /* Invalid state change. */
1736                         status = -EINVAL;
1737                         break;
1738                 };
1739                 break;
1740         case QED_ROCE_QP_STATE_ERR:
1741                 /* ERR->XXX */
1742                 switch (new_state) {
1743                 case QED_ROCE_QP_STATE_RESET:
1744                         if ((qp->rq.prod != qp->rq.cons) ||
1745                             (qp->sq.prod != qp->sq.cons)) {
1746                                 DP_NOTICE(dev,
1747                                           "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
1748                                           qp->rq.prod, qp->rq.cons, qp->sq.prod,
1749                                           qp->sq.cons);
1750                                 status = -EINVAL;
1751                         }
1752                         break;
1753                 default:
1754                         status = -EINVAL;
1755                         break;
1756                 };
1757                 break;
1758         default:
1759                 status = -EINVAL;
1760                 break;
1761         };
1762
1763         return status;
1764 }
1765
1766 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1767                    int attr_mask, struct ib_udata *udata)
1768 {
1769         struct qedr_qp *qp = get_qedr_qp(ibqp);
1770         struct qed_rdma_modify_qp_in_params qp_params = { 0 };
1771         struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
1772         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1773         enum ib_qp_state old_qp_state, new_qp_state;
1774         int rc = 0;
1775
1776         DP_DEBUG(dev, QEDR_MSG_QP,
1777                  "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
1778                  attr->qp_state);
1779
1780         old_qp_state = qedr_get_ibqp_state(qp->state);
1781         if (attr_mask & IB_QP_STATE)
1782                 new_qp_state = attr->qp_state;
1783         else
1784                 new_qp_state = old_qp_state;
1785
1786         if (!ib_modify_qp_is_ok
1787             (old_qp_state, new_qp_state, ibqp->qp_type, attr_mask,
1788              IB_LINK_LAYER_ETHERNET)) {
1789                 DP_ERR(dev,
1790                        "modify qp: invalid attribute mask=0x%x specified for\n"
1791                        "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
1792                        attr_mask, qp->qp_id, ibqp->qp_type, old_qp_state,
1793                        new_qp_state);
1794                 rc = -EINVAL;
1795                 goto err;
1796         }
1797
1798         /* Translate the masks... */
1799         if (attr_mask & IB_QP_STATE) {
1800                 SET_FIELD(qp_params.modify_flags,
1801                           QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
1802                 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
1803         }
1804
1805         if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1806                 qp_params.sqd_async = true;
1807
1808         if (attr_mask & IB_QP_PKEY_INDEX) {
1809                 SET_FIELD(qp_params.modify_flags,
1810                           QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
1811                 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
1812                         rc = -EINVAL;
1813                         goto err;
1814                 }
1815
1816                 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
1817         }
1818
1819         if (attr_mask & IB_QP_QKEY)
1820                 qp->qkey = attr->qkey;
1821
1822         if (attr_mask & IB_QP_ACCESS_FLAGS) {
1823                 SET_FIELD(qp_params.modify_flags,
1824                           QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
1825                 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
1826                                                   IB_ACCESS_REMOTE_READ;
1827                 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
1828                                                    IB_ACCESS_REMOTE_WRITE;
1829                 qp_params.incoming_atomic_en = attr->qp_access_flags &
1830                                                IB_ACCESS_REMOTE_ATOMIC;
1831         }
1832
1833         if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
1834                 if (attr_mask & IB_QP_PATH_MTU) {
1835                         if (attr->path_mtu < IB_MTU_256 ||
1836                             attr->path_mtu > IB_MTU_4096) {
1837                                 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
1838                                 rc = -EINVAL;
1839                                 goto err;
1840                         }
1841                         qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
1842                                       ib_mtu_enum_to_int(iboe_get_mtu
1843                                                          (dev->ndev->mtu)));
1844                 }
1845
1846                 if (!qp->mtu) {
1847                         qp->mtu =
1848                         ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1849                         pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
1850                 }
1851
1852                 SET_FIELD(qp_params.modify_flags,
1853                           QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
1854
1855                 qp_params.traffic_class_tos = grh->traffic_class;
1856                 qp_params.flow_label = grh->flow_label;
1857                 qp_params.hop_limit_ttl = grh->hop_limit;
1858
1859                 qp->sgid_idx = grh->sgid_index;
1860
1861                 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
1862                 if (rc) {
1863                         DP_ERR(dev,
1864                                "modify qp: problems with GID index %d (rc=%d)\n",
1865                                grh->sgid_index, rc);
1866                         return rc;
1867                 }
1868
1869                 rc = qedr_get_dmac(dev, &attr->ah_attr,
1870                                    qp_params.remote_mac_addr);
1871                 if (rc)
1872                         return rc;
1873
1874                 qp_params.use_local_mac = true;
1875                 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
1876
1877                 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
1878                          qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
1879                          qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
1880                 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
1881                          qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
1882                          qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
1883                 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
1884                          qp_params.remote_mac_addr);
1885
1886                 qp_params.mtu = qp->mtu;
1887                 qp_params.lb_indication = false;
1888         }
1889
1890         if (!qp_params.mtu) {
1891                 /* Stay with current MTU */
1892                 if (qp->mtu)
1893                         qp_params.mtu = qp->mtu;
1894                 else
1895                         qp_params.mtu =
1896                             ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1897         }
1898
1899         if (attr_mask & IB_QP_TIMEOUT) {
1900                 SET_FIELD(qp_params.modify_flags,
1901                           QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
1902
1903                 qp_params.ack_timeout = attr->timeout;
1904                 if (attr->timeout) {
1905                         u32 temp;
1906
1907                         temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
1908                         /* FW requires [msec] */
1909                         qp_params.ack_timeout = temp;
1910                 } else {
1911                         /* Infinite */
1912                         qp_params.ack_timeout = 0;
1913                 }
1914         }
1915         if (attr_mask & IB_QP_RETRY_CNT) {
1916                 SET_FIELD(qp_params.modify_flags,
1917                           QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
1918                 qp_params.retry_cnt = attr->retry_cnt;
1919         }
1920
1921         if (attr_mask & IB_QP_RNR_RETRY) {
1922                 SET_FIELD(qp_params.modify_flags,
1923                           QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
1924                 qp_params.rnr_retry_cnt = attr->rnr_retry;
1925         }
1926
1927         if (attr_mask & IB_QP_RQ_PSN) {
1928                 SET_FIELD(qp_params.modify_flags,
1929                           QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
1930                 qp_params.rq_psn = attr->rq_psn;
1931                 qp->rq_psn = attr->rq_psn;
1932         }
1933
1934         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1935                 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
1936                         rc = -EINVAL;
1937                         DP_ERR(dev,
1938                                "unsupported max_rd_atomic=%d, supported=%d\n",
1939                                attr->max_rd_atomic,
1940                                dev->attr.max_qp_req_rd_atomic_resc);
1941                         goto err;
1942                 }
1943
1944                 SET_FIELD(qp_params.modify_flags,
1945                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
1946                 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
1947         }
1948
1949         if (attr_mask & IB_QP_MIN_RNR_TIMER) {
1950                 SET_FIELD(qp_params.modify_flags,
1951                           QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
1952                 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
1953         }
1954
1955         if (attr_mask & IB_QP_SQ_PSN) {
1956                 SET_FIELD(qp_params.modify_flags,
1957                           QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
1958                 qp_params.sq_psn = attr->sq_psn;
1959                 qp->sq_psn = attr->sq_psn;
1960         }
1961
1962         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1963                 if (attr->max_dest_rd_atomic >
1964                     dev->attr.max_qp_resp_rd_atomic_resc) {
1965                         DP_ERR(dev,
1966                                "unsupported max_dest_rd_atomic=%d, supported=%d\n",
1967                                attr->max_dest_rd_atomic,
1968                                dev->attr.max_qp_resp_rd_atomic_resc);
1969
1970                         rc = -EINVAL;
1971                         goto err;
1972                 }
1973
1974                 SET_FIELD(qp_params.modify_flags,
1975                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
1976                 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
1977         }
1978
1979         if (attr_mask & IB_QP_DEST_QPN) {
1980                 SET_FIELD(qp_params.modify_flags,
1981                           QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
1982
1983                 qp_params.dest_qp = attr->dest_qp_num;
1984                 qp->dest_qp_num = attr->dest_qp_num;
1985         }
1986
1987         if (qp->qp_type != IB_QPT_GSI)
1988                 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
1989                                               qp->qed_qp, &qp_params);
1990
1991         if (attr_mask & IB_QP_STATE) {
1992                 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
1993                         rc = qedr_update_qp_state(dev, qp, qp_params.new_state);
1994                 qp->state = qp_params.new_state;
1995         }
1996
1997 err:
1998         return rc;
1999 }
2000
2001 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2002 {
2003         int ib_qp_acc_flags = 0;
2004
2005         if (params->incoming_rdma_write_en)
2006                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2007         if (params->incoming_rdma_read_en)
2008                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2009         if (params->incoming_atomic_en)
2010                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2011         ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2012         return ib_qp_acc_flags;
2013 }
2014
2015 int qedr_query_qp(struct ib_qp *ibqp,
2016                   struct ib_qp_attr *qp_attr,
2017                   int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2018 {
2019         struct qed_rdma_query_qp_out_params params;
2020         struct qedr_qp *qp = get_qedr_qp(ibqp);
2021         struct qedr_dev *dev = qp->dev;
2022         int rc = 0;
2023
2024         memset(&params, 0, sizeof(params));
2025
2026         rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2027         if (rc)
2028                 goto err;
2029
2030         memset(qp_attr, 0, sizeof(*qp_attr));
2031         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2032
2033         qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2034         qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2035         qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2036         qp_attr->path_mig_state = IB_MIG_MIGRATED;
2037         qp_attr->rq_psn = params.rq_psn;
2038         qp_attr->sq_psn = params.sq_psn;
2039         qp_attr->dest_qp_num = params.dest_qp;
2040
2041         qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2042
2043         qp_attr->cap.max_send_wr = qp->sq.max_wr;
2044         qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2045         qp_attr->cap.max_send_sge = qp->sq.max_sges;
2046         qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2047         qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2048         qp_init_attr->cap = qp_attr->cap;
2049
2050         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2051         rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2052                         params.flow_label, qp->sgid_idx,
2053                         params.hop_limit_ttl, params.traffic_class_tos);
2054         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2055         rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2056         rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2057         qp_attr->timeout = params.timeout;
2058         qp_attr->rnr_retry = params.rnr_retry;
2059         qp_attr->retry_cnt = params.retry_cnt;
2060         qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2061         qp_attr->pkey_index = params.pkey_index;
2062         qp_attr->port_num = 1;
2063         rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2064         rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2065         qp_attr->alt_pkey_index = 0;
2066         qp_attr->alt_port_num = 0;
2067         qp_attr->alt_timeout = 0;
2068         memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2069
2070         qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2071         qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2072         qp_attr->max_rd_atomic = params.max_rd_atomic;
2073         qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2074
2075         DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2076                  qp_attr->cap.max_inline_data);
2077
2078 err:
2079         return rc;
2080 }
2081
2082 int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp)
2083 {
2084         int rc = 0;
2085
2086         if (qp->qp_type != IB_QPT_GSI) {
2087                 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2088                 if (rc)
2089                         return rc;
2090         }
2091
2092         if (qp->ibqp.uobject && qp->ibqp.uobject->context)
2093                 qedr_cleanup_user(dev, qp);
2094         else
2095                 qedr_cleanup_kernel(dev, qp);
2096
2097         return 0;
2098 }
2099
2100 int qedr_destroy_qp(struct ib_qp *ibqp)
2101 {
2102         struct qedr_qp *qp = get_qedr_qp(ibqp);
2103         struct qedr_dev *dev = qp->dev;
2104         struct ib_qp_attr attr;
2105         int attr_mask = 0;
2106         int rc = 0;
2107
2108         DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2109                  qp, qp->qp_type);
2110
2111         if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2112             (qp->state != QED_ROCE_QP_STATE_ERR) &&
2113             (qp->state != QED_ROCE_QP_STATE_INIT)) {
2114
2115                 attr.qp_state = IB_QPS_ERR;
2116                 attr_mask |= IB_QP_STATE;
2117
2118                 /* Change the QP state to ERROR */
2119                 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2120         }
2121
2122         if (qp->qp_type == IB_QPT_GSI)
2123                 qedr_destroy_gsi_qp(dev);
2124
2125         qedr_free_qp_resources(dev, qp);
2126
2127         kfree(qp);
2128
2129         return rc;
2130 }
2131
2132 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr,
2133                              struct ib_udata *udata)
2134 {
2135         struct qedr_ah *ah;
2136
2137         ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2138         if (!ah)
2139                 return ERR_PTR(-ENOMEM);
2140
2141         ah->attr = *attr;
2142
2143         return &ah->ibah;
2144 }
2145
2146 int qedr_destroy_ah(struct ib_ah *ibah)
2147 {
2148         struct qedr_ah *ah = get_qedr_ah(ibah);
2149
2150         kfree(ah);
2151         return 0;
2152 }
2153
2154 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2155 {
2156         struct qedr_pbl *pbl, *tmp;
2157
2158         if (info->pbl_table)
2159                 list_add_tail(&info->pbl_table->list_entry,
2160                               &info->free_pbl_list);
2161
2162         if (!list_empty(&info->inuse_pbl_list))
2163                 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2164
2165         list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2166                 list_del(&pbl->list_entry);
2167                 qedr_free_pbl(dev, &info->pbl_info, pbl);
2168         }
2169 }
2170
2171 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2172                         size_t page_list_len, bool two_layered)
2173 {
2174         struct qedr_pbl *tmp;
2175         int rc;
2176
2177         INIT_LIST_HEAD(&info->free_pbl_list);
2178         INIT_LIST_HEAD(&info->inuse_pbl_list);
2179
2180         rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2181                                   page_list_len, two_layered);
2182         if (rc)
2183                 goto done;
2184
2185         info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2186         if (IS_ERR(info->pbl_table)) {
2187                 rc = PTR_ERR(info->pbl_table);
2188                 goto done;
2189         }
2190
2191         DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2192                  &info->pbl_table->pa);
2193
2194         /* in usual case we use 2 PBLs, so we add one to free
2195          * list and allocating another one
2196          */
2197         tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2198         if (IS_ERR(tmp)) {
2199                 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2200                 goto done;
2201         }
2202
2203         list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2204
2205         DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2206
2207 done:
2208         if (rc)
2209                 free_mr_info(dev, info);
2210
2211         return rc;
2212 }
2213
2214 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2215                                u64 usr_addr, int acc, struct ib_udata *udata)
2216 {
2217         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2218         struct qedr_mr *mr;
2219         struct qedr_pd *pd;
2220         int rc = -ENOMEM;
2221
2222         pd = get_qedr_pd(ibpd);
2223         DP_DEBUG(dev, QEDR_MSG_MR,
2224                  "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2225                  pd->pd_id, start, len, usr_addr, acc);
2226
2227         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2228                 return ERR_PTR(-EINVAL);
2229
2230         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2231         if (!mr)
2232                 return ERR_PTR(rc);
2233
2234         mr->type = QEDR_MR_USER;
2235
2236         mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
2237         if (IS_ERR(mr->umem)) {
2238                 rc = -EFAULT;
2239                 goto err0;
2240         }
2241
2242         rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2243         if (rc)
2244                 goto err1;
2245
2246         qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2247                            &mr->info.pbl_info, mr->umem->page_shift);
2248
2249         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2250         if (rc) {
2251                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2252                 goto err1;
2253         }
2254
2255         /* Index only, 18 bit long, lkey = itid << 8 | key */
2256         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2257         mr->hw_mr.key = 0;
2258         mr->hw_mr.pd = pd->pd_id;
2259         mr->hw_mr.local_read = 1;
2260         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2261         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2262         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2263         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2264         mr->hw_mr.mw_bind = false;
2265         mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2266         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2267         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2268         mr->hw_mr.page_size_log = mr->umem->page_shift;
2269         mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2270         mr->hw_mr.length = len;
2271         mr->hw_mr.vaddr = usr_addr;
2272         mr->hw_mr.zbva = false;
2273         mr->hw_mr.phy_mr = false;
2274         mr->hw_mr.dma_mr = false;
2275
2276         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2277         if (rc) {
2278                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2279                 goto err2;
2280         }
2281
2282         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2283         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2284             mr->hw_mr.remote_atomic)
2285                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2286
2287         DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2288                  mr->ibmr.lkey);
2289         return &mr->ibmr;
2290
2291 err2:
2292         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2293 err1:
2294         qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2295 err0:
2296         kfree(mr);
2297         return ERR_PTR(rc);
2298 }
2299
2300 int qedr_dereg_mr(struct ib_mr *ib_mr)
2301 {
2302         struct qedr_mr *mr = get_qedr_mr(ib_mr);
2303         struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2304         int rc = 0;
2305
2306         rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2307         if (rc)
2308                 return rc;
2309
2310         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2311
2312         if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2313                 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2314
2315         /* it could be user registered memory. */
2316         if (mr->umem)
2317                 ib_umem_release(mr->umem);
2318
2319         kfree(mr);
2320
2321         return rc;
2322 }
2323
2324 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2325                                        int max_page_list_len)
2326 {
2327         struct qedr_pd *pd = get_qedr_pd(ibpd);
2328         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2329         struct qedr_mr *mr;
2330         int rc = -ENOMEM;
2331
2332         DP_DEBUG(dev, QEDR_MSG_MR,
2333                  "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2334                  max_page_list_len);
2335
2336         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2337         if (!mr)
2338                 return ERR_PTR(rc);
2339
2340         mr->dev = dev;
2341         mr->type = QEDR_MR_FRMR;
2342
2343         rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2344         if (rc)
2345                 goto err0;
2346
2347         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2348         if (rc) {
2349                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2350                 goto err0;
2351         }
2352
2353         /* Index only, 18 bit long, lkey = itid << 8 | key */
2354         mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2355         mr->hw_mr.key = 0;
2356         mr->hw_mr.pd = pd->pd_id;
2357         mr->hw_mr.local_read = 1;
2358         mr->hw_mr.local_write = 0;
2359         mr->hw_mr.remote_read = 0;
2360         mr->hw_mr.remote_write = 0;
2361         mr->hw_mr.remote_atomic = 0;
2362         mr->hw_mr.mw_bind = false;
2363         mr->hw_mr.pbl_ptr = 0;
2364         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2365         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2366         mr->hw_mr.fbo = 0;
2367         mr->hw_mr.length = 0;
2368         mr->hw_mr.vaddr = 0;
2369         mr->hw_mr.zbva = false;
2370         mr->hw_mr.phy_mr = true;
2371         mr->hw_mr.dma_mr = false;
2372
2373         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2374         if (rc) {
2375                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2376                 goto err1;
2377         }
2378
2379         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2380         mr->ibmr.rkey = mr->ibmr.lkey;
2381
2382         DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2383         return mr;
2384
2385 err1:
2386         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2387 err0:
2388         kfree(mr);
2389         return ERR_PTR(rc);
2390 }
2391
2392 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
2393                             enum ib_mr_type mr_type, u32 max_num_sg)
2394 {
2395         struct qedr_dev *dev;
2396         struct qedr_mr *mr;
2397
2398         if (mr_type != IB_MR_TYPE_MEM_REG)
2399                 return ERR_PTR(-EINVAL);
2400
2401         mr = __qedr_alloc_mr(ibpd, max_num_sg);
2402
2403         if (IS_ERR(mr))
2404                 return ERR_PTR(-EINVAL);
2405
2406         dev = mr->dev;
2407
2408         return &mr->ibmr;
2409 }
2410
2411 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2412 {
2413         struct qedr_mr *mr = get_qedr_mr(ibmr);
2414         struct qedr_pbl *pbl_table;
2415         struct regpair *pbe;
2416         u32 pbes_in_page;
2417
2418         if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2419                 DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages);
2420                 return -ENOMEM;
2421         }
2422
2423         DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2424                  mr->npages, addr);
2425
2426         pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2427         pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2428         pbe = (struct regpair *)pbl_table->va;
2429         pbe +=  mr->npages % pbes_in_page;
2430         pbe->lo = cpu_to_le32((u32)addr);
2431         pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2432
2433         mr->npages++;
2434
2435         return 0;
2436 }
2437
2438 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2439 {
2440         int work = info->completed - info->completed_handled - 1;
2441
2442         DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2443         while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2444                 struct qedr_pbl *pbl;
2445
2446                 /* Free all the page list that are possible to be freed
2447                  * (all the ones that were invalidated), under the assumption
2448                  * that if an FMR was completed successfully that means that
2449                  * if there was an invalidate operation before it also ended
2450                  */
2451                 pbl = list_first_entry(&info->inuse_pbl_list,
2452                                        struct qedr_pbl, list_entry);
2453                 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
2454                 info->completed_handled++;
2455         }
2456 }
2457
2458 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2459                    int sg_nents, unsigned int *sg_offset)
2460 {
2461         struct qedr_mr *mr = get_qedr_mr(ibmr);
2462
2463         mr->npages = 0;
2464
2465         handle_completed_mrs(mr->dev, &mr->info);
2466         return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2467 }
2468
2469 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2470 {
2471         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2472         struct qedr_pd *pd = get_qedr_pd(ibpd);
2473         struct qedr_mr *mr;
2474         int rc;
2475
2476         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2477         if (!mr)
2478                 return ERR_PTR(-ENOMEM);
2479
2480         mr->type = QEDR_MR_DMA;
2481
2482         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2483         if (rc) {
2484                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2485                 goto err1;
2486         }
2487
2488         /* index only, 18 bit long, lkey = itid << 8 | key */
2489         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2490         mr->hw_mr.pd = pd->pd_id;
2491         mr->hw_mr.local_read = 1;
2492         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2493         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2494         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2495         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2496         mr->hw_mr.dma_mr = true;
2497
2498         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2499         if (rc) {
2500                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2501                 goto err2;
2502         }
2503
2504         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2505         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2506             mr->hw_mr.remote_atomic)
2507                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2508
2509         DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2510         return &mr->ibmr;
2511
2512 err2:
2513         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2514 err1:
2515         kfree(mr);
2516         return ERR_PTR(rc);
2517 }
2518
2519 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2520 {
2521         return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2522 }
2523
2524 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2525 {
2526         int i, len = 0;
2527
2528         for (i = 0; i < num_sge; i++)
2529                 len += sg_list[i].length;
2530
2531         return len;
2532 }
2533
2534 static void swap_wqe_data64(u64 *p)
2535 {
2536         int i;
2537
2538         for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2539                 *p = cpu_to_be64(cpu_to_le64(*p));
2540 }
2541
2542 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2543                                        struct qedr_qp *qp, u8 *wqe_size,
2544                                        struct ib_send_wr *wr,
2545                                        struct ib_send_wr **bad_wr, u8 *bits,
2546                                        u8 bit)
2547 {
2548         u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2549         char *seg_prt, *wqe;
2550         int i, seg_siz;
2551
2552         if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2553                 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2554                 *bad_wr = wr;
2555                 return 0;
2556         }
2557
2558         if (!data_size)
2559                 return data_size;
2560
2561         *bits |= bit;
2562
2563         seg_prt = NULL;
2564         wqe = NULL;
2565         seg_siz = 0;
2566
2567         /* Copy data inline */
2568         for (i = 0; i < wr->num_sge; i++) {
2569                 u32 len = wr->sg_list[i].length;
2570                 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2571
2572                 while (len > 0) {
2573                         u32 cur;
2574
2575                         /* New segment required */
2576                         if (!seg_siz) {
2577                                 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2578                                 seg_prt = wqe;
2579                                 seg_siz = sizeof(struct rdma_sq_common_wqe);
2580                                 (*wqe_size)++;
2581                         }
2582
2583                         /* Calculate currently allowed length */
2584                         cur = min_t(u32, len, seg_siz);
2585                         memcpy(seg_prt, src, cur);
2586
2587                         /* Update segment variables */
2588                         seg_prt += cur;
2589                         seg_siz -= cur;
2590
2591                         /* Update sge variables */
2592                         src += cur;
2593                         len -= cur;
2594
2595                         /* Swap fully-completed segments */
2596                         if (!seg_siz)
2597                                 swap_wqe_data64((u64 *)wqe);
2598                 }
2599         }
2600
2601         /* swap last not completed segment */
2602         if (seg_siz)
2603                 swap_wqe_data64((u64 *)wqe);
2604
2605         return data_size;
2606 }
2607
2608 #define RQ_SGE_SET(sge, vaddr, vlength, vflags)                 \
2609         do {                                                    \
2610                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
2611                 (sge)->length = cpu_to_le32(vlength);           \
2612                 (sge)->flags = cpu_to_le32(vflags);             \
2613         } while (0)
2614
2615 #define SRQ_HDR_SET(hdr, vwr_id, num_sge)                       \
2616         do {                                                    \
2617                 DMA_REGPAIR_LE(hdr->wr_id, vwr_id);             \
2618                 (hdr)->num_sges = num_sge;                      \
2619         } while (0)
2620
2621 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)                 \
2622         do {                                                    \
2623                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
2624                 (sge)->length = cpu_to_le32(vlength);           \
2625                 (sge)->l_key = cpu_to_le32(vlkey);              \
2626         } while (0)
2627
2628 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
2629                                 struct ib_send_wr *wr)
2630 {
2631         u32 data_size = 0;
2632         int i;
2633
2634         for (i = 0; i < wr->num_sge; i++) {
2635                 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
2636
2637                 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
2638                 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
2639                 sge->length = cpu_to_le32(wr->sg_list[i].length);
2640                 data_size += wr->sg_list[i].length;
2641         }
2642
2643         if (wqe_size)
2644                 *wqe_size += wr->num_sge;
2645
2646         return data_size;
2647 }
2648
2649 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
2650                                      struct qedr_qp *qp,
2651                                      struct rdma_sq_rdma_wqe_1st *rwqe,
2652                                      struct rdma_sq_rdma_wqe_2nd *rwqe2,
2653                                      struct ib_send_wr *wr,
2654                                      struct ib_send_wr **bad_wr)
2655 {
2656         rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
2657         DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
2658
2659         if (wr->send_flags & IB_SEND_INLINE &&
2660             (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2661              wr->opcode == IB_WR_RDMA_WRITE)) {
2662                 u8 flags = 0;
2663
2664                 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
2665                 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
2666                                                    bad_wr, &rwqe->flags, flags);
2667         }
2668
2669         return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
2670 }
2671
2672 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
2673                                      struct qedr_qp *qp,
2674                                      struct rdma_sq_send_wqe_1st *swqe,
2675                                      struct rdma_sq_send_wqe_2st *swqe2,
2676                                      struct ib_send_wr *wr,
2677                                      struct ib_send_wr **bad_wr)
2678 {
2679         memset(swqe2, 0, sizeof(*swqe2));
2680         if (wr->send_flags & IB_SEND_INLINE) {
2681                 u8 flags = 0;
2682
2683                 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
2684                 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
2685                                                    bad_wr, &swqe->flags, flags);
2686         }
2687
2688         return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
2689 }
2690
2691 static int qedr_prepare_reg(struct qedr_qp *qp,
2692                             struct rdma_sq_fmr_wqe_1st *fwqe1,
2693                             struct ib_reg_wr *wr)
2694 {
2695         struct qedr_mr *mr = get_qedr_mr(wr->mr);
2696         struct rdma_sq_fmr_wqe_2nd *fwqe2;
2697
2698         fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
2699         fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
2700         fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
2701         fwqe1->l_key = wr->key;
2702
2703         fwqe2->access_ctrl = 0;
2704
2705         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
2706                    !!(wr->access & IB_ACCESS_REMOTE_READ));
2707         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
2708                    !!(wr->access & IB_ACCESS_REMOTE_WRITE));
2709         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
2710                    !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
2711         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
2712         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
2713                    !!(wr->access & IB_ACCESS_LOCAL_WRITE));
2714         fwqe2->fmr_ctrl = 0;
2715
2716         SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
2717                    ilog2(mr->ibmr.page_size) - 12);
2718
2719         fwqe2->length_hi = 0;
2720         fwqe2->length_lo = mr->ibmr.length;
2721         fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
2722         fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
2723
2724         qp->wqe_wr_id[qp->sq.prod].mr = mr;
2725
2726         return 0;
2727 }
2728
2729 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
2730 {
2731         switch (opcode) {
2732         case IB_WR_RDMA_WRITE:
2733         case IB_WR_RDMA_WRITE_WITH_IMM:
2734                 return IB_WC_RDMA_WRITE;
2735         case IB_WR_SEND_WITH_IMM:
2736         case IB_WR_SEND:
2737         case IB_WR_SEND_WITH_INV:
2738                 return IB_WC_SEND;
2739         case IB_WR_RDMA_READ:
2740                 return IB_WC_RDMA_READ;
2741         case IB_WR_ATOMIC_CMP_AND_SWP:
2742                 return IB_WC_COMP_SWAP;
2743         case IB_WR_ATOMIC_FETCH_AND_ADD:
2744                 return IB_WC_FETCH_ADD;
2745         case IB_WR_REG_MR:
2746                 return IB_WC_REG_MR;
2747         case IB_WR_LOCAL_INV:
2748                 return IB_WC_LOCAL_INV;
2749         default:
2750                 return IB_WC_SEND;
2751         }
2752 }
2753
2754 static inline bool qedr_can_post_send(struct qedr_qp *qp, struct ib_send_wr *wr)
2755 {
2756         int wq_is_full, err_wr, pbl_is_full;
2757         struct qedr_dev *dev = qp->dev;
2758
2759         /* prevent SQ overflow and/or processing of a bad WR */
2760         err_wr = wr->num_sge > qp->sq.max_sges;
2761         wq_is_full = qedr_wq_is_full(&qp->sq);
2762         pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
2763                       QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2764         if (wq_is_full || err_wr || pbl_is_full) {
2765                 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
2766                         DP_ERR(dev,
2767                                "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
2768                                qp);
2769                         qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
2770                 }
2771
2772                 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
2773                         DP_ERR(dev,
2774                                "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
2775                                qp);
2776                         qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
2777                 }
2778
2779                 if (pbl_is_full &&
2780                     !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
2781                         DP_ERR(dev,
2782                                "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
2783                                qp);
2784                         qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
2785                 }
2786                 return false;
2787         }
2788         return true;
2789 }
2790
2791 static int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2792                      struct ib_send_wr **bad_wr)
2793 {
2794         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2795         struct qedr_qp *qp = get_qedr_qp(ibqp);
2796         struct rdma_sq_atomic_wqe_1st *awqe1;
2797         struct rdma_sq_atomic_wqe_2nd *awqe2;
2798         struct rdma_sq_atomic_wqe_3rd *awqe3;
2799         struct rdma_sq_send_wqe_2st *swqe2;
2800         struct rdma_sq_local_inv_wqe *iwqe;
2801         struct rdma_sq_rdma_wqe_2nd *rwqe2;
2802         struct rdma_sq_send_wqe_1st *swqe;
2803         struct rdma_sq_rdma_wqe_1st *rwqe;
2804         struct rdma_sq_fmr_wqe_1st *fwqe1;
2805         struct rdma_sq_common_wqe *wqe;
2806         u32 length;
2807         int rc = 0;
2808         bool comp;
2809
2810         if (!qedr_can_post_send(qp, wr)) {
2811                 *bad_wr = wr;
2812                 return -ENOMEM;
2813         }
2814
2815         wqe = qed_chain_produce(&qp->sq.pbl);
2816         qp->wqe_wr_id[qp->sq.prod].signaled =
2817                 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
2818
2819         wqe->flags = 0;
2820         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
2821                    !!(wr->send_flags & IB_SEND_SOLICITED));
2822         comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
2823         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
2824         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
2825                    !!(wr->send_flags & IB_SEND_FENCE));
2826         wqe->prev_wqe_size = qp->prev_wqe_size;
2827
2828         qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
2829
2830         switch (wr->opcode) {
2831         case IB_WR_SEND_WITH_IMM:
2832                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
2833                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2834                 swqe->wqe_size = 2;
2835                 swqe2 = qed_chain_produce(&qp->sq.pbl);
2836
2837                 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.imm_data);
2838                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2839                                                    wr, bad_wr);
2840                 swqe->length = cpu_to_le32(length);
2841                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2842                 qp->prev_wqe_size = swqe->wqe_size;
2843                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2844                 break;
2845         case IB_WR_SEND:
2846                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
2847                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2848
2849                 swqe->wqe_size = 2;
2850                 swqe2 = qed_chain_produce(&qp->sq.pbl);
2851                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2852                                                    wr, bad_wr);
2853                 swqe->length = cpu_to_le32(length);
2854                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2855                 qp->prev_wqe_size = swqe->wqe_size;
2856                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2857                 break;
2858         case IB_WR_SEND_WITH_INV:
2859                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
2860                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2861                 swqe2 = qed_chain_produce(&qp->sq.pbl);
2862                 swqe->wqe_size = 2;
2863                 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
2864                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2865                                                    wr, bad_wr);
2866                 swqe->length = cpu_to_le32(length);
2867                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2868                 qp->prev_wqe_size = swqe->wqe_size;
2869                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2870                 break;
2871
2872         case IB_WR_RDMA_WRITE_WITH_IMM:
2873                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
2874                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2875
2876                 rwqe->wqe_size = 2;
2877                 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
2878                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2879                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2880                                                    wr, bad_wr);
2881                 rwqe->length = cpu_to_le32(length);
2882                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2883                 qp->prev_wqe_size = rwqe->wqe_size;
2884                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2885                 break;
2886         case IB_WR_RDMA_WRITE:
2887                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
2888                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2889
2890                 rwqe->wqe_size = 2;
2891                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2892                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2893                                                    wr, bad_wr);
2894                 rwqe->length = cpu_to_le32(length);
2895                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2896                 qp->prev_wqe_size = rwqe->wqe_size;
2897                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2898                 break;
2899         case IB_WR_RDMA_READ_WITH_INV:
2900                 DP_ERR(dev,
2901                        "RDMA READ WITH INVALIDATE not supported\n");
2902                 *bad_wr = wr;
2903                 rc = -EINVAL;
2904                 break;
2905
2906         case IB_WR_RDMA_READ:
2907                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
2908                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2909
2910                 rwqe->wqe_size = 2;
2911                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2912                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2913                                                    wr, bad_wr);
2914                 rwqe->length = cpu_to_le32(length);
2915                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2916                 qp->prev_wqe_size = rwqe->wqe_size;
2917                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2918                 break;
2919
2920         case IB_WR_ATOMIC_CMP_AND_SWP:
2921         case IB_WR_ATOMIC_FETCH_AND_ADD:
2922                 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
2923                 awqe1->wqe_size = 4;
2924
2925                 awqe2 = qed_chain_produce(&qp->sq.pbl);
2926                 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
2927                 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
2928
2929                 awqe3 = qed_chain_produce(&qp->sq.pbl);
2930
2931                 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2932                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
2933                         DMA_REGPAIR_LE(awqe3->swap_data,
2934                                        atomic_wr(wr)->compare_add);
2935                 } else {
2936                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
2937                         DMA_REGPAIR_LE(awqe3->swap_data,
2938                                        atomic_wr(wr)->swap);
2939                         DMA_REGPAIR_LE(awqe3->cmp_data,
2940                                        atomic_wr(wr)->compare_add);
2941                 }
2942
2943                 qedr_prepare_sq_sges(qp, NULL, wr);
2944
2945                 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
2946                 qp->prev_wqe_size = awqe1->wqe_size;
2947                 break;
2948
2949         case IB_WR_LOCAL_INV:
2950                 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
2951                 iwqe->wqe_size = 1;
2952
2953                 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
2954                 iwqe->inv_l_key = wr->ex.invalidate_rkey;
2955                 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
2956                 qp->prev_wqe_size = iwqe->wqe_size;
2957                 break;
2958         case IB_WR_REG_MR:
2959                 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
2960                 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
2961                 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
2962                 fwqe1->wqe_size = 2;
2963
2964                 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
2965                 if (rc) {
2966                         DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
2967                         *bad_wr = wr;
2968                         break;
2969                 }
2970
2971                 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
2972                 qp->prev_wqe_size = fwqe1->wqe_size;
2973                 break;
2974         default:
2975                 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
2976                 rc = -EINVAL;
2977                 *bad_wr = wr;
2978                 break;
2979         }
2980
2981         if (*bad_wr) {
2982                 u16 value;
2983
2984                 /* Restore prod to its position before
2985                  * this WR was processed
2986                  */
2987                 value = le16_to_cpu(qp->sq.db_data.data.value);
2988                 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
2989
2990                 /* Restore prev_wqe_size */
2991                 qp->prev_wqe_size = wqe->prev_wqe_size;
2992                 rc = -EINVAL;
2993                 DP_ERR(dev, "POST SEND FAILED\n");
2994         }
2995
2996         return rc;
2997 }
2998
2999 int qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
3000                    struct ib_send_wr **bad_wr)
3001 {
3002         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3003         struct qedr_qp *qp = get_qedr_qp(ibqp);
3004         unsigned long flags;
3005         int rc = 0;
3006
3007         *bad_wr = NULL;
3008
3009         if (qp->qp_type == IB_QPT_GSI)
3010                 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3011
3012         spin_lock_irqsave(&qp->q_lock, flags);
3013
3014         if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3015             (qp->state != QED_ROCE_QP_STATE_ERR) &&
3016             (qp->state != QED_ROCE_QP_STATE_SQD)) {
3017                 spin_unlock_irqrestore(&qp->q_lock, flags);
3018                 *bad_wr = wr;
3019                 DP_DEBUG(dev, QEDR_MSG_CQ,
3020                          "QP in wrong state! QP icid=0x%x state %d\n",
3021                          qp->icid, qp->state);
3022                 return -EINVAL;
3023         }
3024
3025         while (wr) {
3026                 rc = __qedr_post_send(ibqp, wr, bad_wr);
3027                 if (rc)
3028                         break;
3029
3030                 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3031
3032                 qedr_inc_sw_prod(&qp->sq);
3033
3034                 qp->sq.db_data.data.value++;
3035
3036                 wr = wr->next;
3037         }
3038
3039         /* Trigger doorbell
3040          * If there was a failure in the first WR then it will be triggered in
3041          * vane. However this is not harmful (as long as the producer value is
3042          * unchanged). For performance reasons we avoid checking for this
3043          * redundant doorbell.
3044          */
3045         wmb();
3046         writel(qp->sq.db_data.raw, qp->sq.db);
3047
3048         /* Make sure write sticks */
3049         mmiowb();
3050
3051         spin_unlock_irqrestore(&qp->q_lock, flags);
3052
3053         return rc;
3054 }
3055
3056 int qedr_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
3057                    struct ib_recv_wr **bad_wr)
3058 {
3059         struct qedr_qp *qp = get_qedr_qp(ibqp);
3060         struct qedr_dev *dev = qp->dev;
3061         unsigned long flags;
3062         int status = 0;
3063
3064         if (qp->qp_type == IB_QPT_GSI)
3065                 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3066
3067         spin_lock_irqsave(&qp->q_lock, flags);
3068
3069         if (qp->state == QED_ROCE_QP_STATE_RESET) {
3070                 spin_unlock_irqrestore(&qp->q_lock, flags);
3071                 *bad_wr = wr;
3072                 return -EINVAL;
3073         }
3074
3075         while (wr) {
3076                 int i;
3077
3078                 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3079                     QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3080                     wr->num_sge > qp->rq.max_sges) {
3081                         DP_ERR(dev, "Can't post WR  (%d < %d) || (%d > %d)\n",
3082                                qed_chain_get_elem_left_u32(&qp->rq.pbl),
3083                                QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3084                                qp->rq.max_sges);
3085                         status = -ENOMEM;
3086                         *bad_wr = wr;
3087                         break;
3088                 }
3089                 for (i = 0; i < wr->num_sge; i++) {
3090                         u32 flags = 0;
3091                         struct rdma_rq_sge *rqe =
3092                             qed_chain_produce(&qp->rq.pbl);
3093
3094                         /* First one must include the number
3095                          * of SGE in the list
3096                          */
3097                         if (!i)
3098                                 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3099                                           wr->num_sge);
3100
3101                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY,
3102                                   wr->sg_list[i].lkey);
3103
3104                         RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3105                                    wr->sg_list[i].length, flags);
3106                 }
3107
3108                 /* Special case of no sges. FW requires between 1-4 sges...
3109                  * in this case we need to post 1 sge with length zero. this is
3110                  * because rdma write with immediate consumes an RQ.
3111                  */
3112                 if (!wr->num_sge) {
3113                         u32 flags = 0;
3114                         struct rdma_rq_sge *rqe =
3115                             qed_chain_produce(&qp->rq.pbl);
3116
3117                         /* First one must include the number
3118                          * of SGE in the list
3119                          */
3120                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
3121                         SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3122
3123                         RQ_SGE_SET(rqe, 0, 0, flags);
3124                         i = 1;
3125                 }
3126
3127                 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3128                 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3129
3130                 qedr_inc_sw_prod(&qp->rq);
3131
3132                 /* Flush all the writes before signalling doorbell */
3133                 wmb();
3134
3135                 qp->rq.db_data.data.value++;
3136
3137                 writel(qp->rq.db_data.raw, qp->rq.db);
3138
3139                 /* Make sure write sticks */
3140                 mmiowb();
3141
3142                 wr = wr->next;
3143         }
3144
3145         spin_unlock_irqrestore(&qp->q_lock, flags);
3146
3147         return status;
3148 }
3149
3150 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3151 {
3152         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3153
3154         return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3155                 cq->pbl_toggle;
3156 }
3157
3158 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3159 {
3160         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3161         struct qedr_qp *qp;
3162
3163         qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3164                                                    resp_cqe->qp_handle.lo,
3165                                                    u64);
3166         return qp;
3167 }
3168
3169 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3170 {
3171         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3172
3173         return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3174 }
3175
3176 /* Return latest CQE (needs processing) */
3177 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3178 {
3179         return cq->latest_cqe;
3180 }
3181
3182 /* In fmr we need to increase the number of fmr completed counter for the fmr
3183  * algorithm determining whether we can free a pbl or not.
3184  * we need to perform this whether the work request was signaled or not. for
3185  * this purpose we call this function from the condition that checks if a wr
3186  * should be skipped, to make sure we don't miss it ( possibly this fmr
3187  * operation was not signalted)
3188  */
3189 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3190 {
3191         if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3192                 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3193 }
3194
3195 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3196                        struct qedr_cq *cq, int num_entries,
3197                        struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3198                        int force)
3199 {
3200         u16 cnt = 0;
3201
3202         while (num_entries && qp->sq.wqe_cons != hw_cons) {
3203                 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3204                         qedr_chk_if_fmr(qp);
3205                         /* skip WC */
3206                         goto next_cqe;
3207                 }
3208
3209                 /* fill WC */
3210                 wc->status = status;
3211                 wc->vendor_err = 0;
3212                 wc->wc_flags = 0;
3213                 wc->src_qp = qp->id;
3214                 wc->qp = &qp->ibqp;
3215
3216                 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3217                 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3218
3219                 switch (wc->opcode) {
3220                 case IB_WC_RDMA_WRITE:
3221                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3222                         break;
3223                 case IB_WC_COMP_SWAP:
3224                 case IB_WC_FETCH_ADD:
3225                         wc->byte_len = 8;
3226                         break;
3227                 case IB_WC_REG_MR:
3228                         qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3229                         break;
3230                 case IB_WC_RDMA_READ:
3231                 case IB_WC_SEND:
3232                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3233                         break;
3234                 default:
3235                         break;
3236                 }
3237
3238                 num_entries--;
3239                 wc++;
3240                 cnt++;
3241 next_cqe:
3242                 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3243                         qed_chain_consume(&qp->sq.pbl);
3244                 qedr_inc_sw_cons(&qp->sq);
3245         }
3246
3247         return cnt;
3248 }
3249
3250 static int qedr_poll_cq_req(struct qedr_dev *dev,
3251                             struct qedr_qp *qp, struct qedr_cq *cq,
3252                             int num_entries, struct ib_wc *wc,
3253                             struct rdma_cqe_requester *req)
3254 {
3255         int cnt = 0;
3256
3257         switch (req->status) {
3258         case RDMA_CQE_REQ_STS_OK:
3259                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3260                                   IB_WC_SUCCESS, 0);
3261                 break;
3262         case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3263                 if (qp->state != QED_ROCE_QP_STATE_ERR)
3264                         DP_ERR(dev,
3265                                "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3266                                cq->icid, qp->icid);
3267                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3268                                   IB_WC_WR_FLUSH_ERR, 1);
3269                 break;
3270         default:
3271                 /* process all WQE before the cosumer */
3272                 qp->state = QED_ROCE_QP_STATE_ERR;
3273                 cnt = process_req(dev, qp, cq, num_entries, wc,
3274                                   req->sq_cons - 1, IB_WC_SUCCESS, 0);
3275                 wc += cnt;
3276                 /* if we have extra WC fill it with actual error info */
3277                 if (cnt < num_entries) {
3278                         enum ib_wc_status wc_status;
3279
3280                         switch (req->status) {
3281                         case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3282                                 DP_ERR(dev,
3283                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3284                                        cq->icid, qp->icid);
3285                                 wc_status = IB_WC_BAD_RESP_ERR;
3286                                 break;
3287                         case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3288                                 DP_ERR(dev,
3289                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3290                                        cq->icid, qp->icid);
3291                                 wc_status = IB_WC_LOC_LEN_ERR;
3292                                 break;
3293                         case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3294                                 DP_ERR(dev,
3295                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3296                                        cq->icid, qp->icid);
3297                                 wc_status = IB_WC_LOC_QP_OP_ERR;
3298                                 break;
3299                         case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3300                                 DP_ERR(dev,
3301                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3302                                        cq->icid, qp->icid);
3303                                 wc_status = IB_WC_LOC_PROT_ERR;
3304                                 break;
3305                         case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3306                                 DP_ERR(dev,
3307                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3308                                        cq->icid, qp->icid);
3309                                 wc_status = IB_WC_MW_BIND_ERR;
3310                                 break;
3311                         case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3312                                 DP_ERR(dev,
3313                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3314                                        cq->icid, qp->icid);
3315                                 wc_status = IB_WC_REM_INV_REQ_ERR;
3316                                 break;
3317                         case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3318                                 DP_ERR(dev,
3319                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3320                                        cq->icid, qp->icid);
3321                                 wc_status = IB_WC_REM_ACCESS_ERR;
3322                                 break;
3323                         case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3324                                 DP_ERR(dev,
3325                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3326                                        cq->icid, qp->icid);
3327                                 wc_status = IB_WC_REM_OP_ERR;
3328                                 break;
3329                         case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3330                                 DP_ERR(dev,
3331                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3332                                        cq->icid, qp->icid);
3333                                 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3334                                 break;
3335                         case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3336                                 DP_ERR(dev,
3337                                        "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3338                                        cq->icid, qp->icid);
3339                                 wc_status = IB_WC_RETRY_EXC_ERR;
3340                                 break;
3341                         default:
3342                                 DP_ERR(dev,
3343                                        "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3344                                        cq->icid, qp->icid);
3345                                 wc_status = IB_WC_GENERAL_ERR;
3346                         }
3347                         cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3348                                            wc_status, 1);
3349                 }
3350         }
3351
3352         return cnt;
3353 }
3354
3355 static inline int qedr_cqe_resp_status_to_ib(u8 status)
3356 {
3357         switch (status) {
3358         case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3359                 return IB_WC_LOC_ACCESS_ERR;
3360         case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3361                 return IB_WC_LOC_LEN_ERR;
3362         case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3363                 return IB_WC_LOC_QP_OP_ERR;
3364         case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3365                 return IB_WC_LOC_PROT_ERR;
3366         case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3367                 return IB_WC_MW_BIND_ERR;
3368         case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3369                 return IB_WC_REM_INV_RD_REQ_ERR;
3370         case RDMA_CQE_RESP_STS_OK:
3371                 return IB_WC_SUCCESS;
3372         default:
3373                 return IB_WC_GENERAL_ERR;
3374         }
3375 }
3376
3377 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
3378                                           struct ib_wc *wc)
3379 {
3380         wc->status = IB_WC_SUCCESS;
3381         wc->byte_len = le32_to_cpu(resp->length);
3382
3383         if (resp->flags & QEDR_RESP_IMM) {
3384                 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3385                 wc->wc_flags |= IB_WC_WITH_IMM;
3386
3387                 if (resp->flags & QEDR_RESP_RDMA)
3388                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3389
3390                 if (resp->flags & QEDR_RESP_INV)
3391                         return -EINVAL;
3392
3393         } else if (resp->flags & QEDR_RESP_INV) {
3394                 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3395                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3396
3397                 if (resp->flags & QEDR_RESP_RDMA)
3398                         return -EINVAL;
3399
3400         } else if (resp->flags & QEDR_RESP_RDMA) {
3401                 return -EINVAL;
3402         }
3403
3404         return 0;
3405 }
3406
3407 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3408                                struct qedr_cq *cq, struct ib_wc *wc,
3409                                struct rdma_cqe_responder *resp, u64 wr_id)
3410 {
3411         /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
3412         wc->opcode = IB_WC_RECV;
3413         wc->wc_flags = 0;
3414
3415         if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
3416                 if (qedr_set_ok_cqe_resp_wc(resp, wc))
3417                         DP_ERR(dev,
3418                                "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
3419                                cq, cq->icid, resp->flags);
3420
3421         } else {
3422                 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
3423                 if (wc->status == IB_WC_GENERAL_ERR)
3424                         DP_ERR(dev,
3425                                "CQ %p (icid=%d) contains an invalid CQE status %d\n",
3426                                cq, cq->icid, resp->status);
3427         }
3428
3429         /* Fill the rest of the WC */
3430         wc->vendor_err = 0;
3431         wc->src_qp = qp->id;
3432         wc->qp = &qp->ibqp;
3433         wc->wr_id = wr_id;
3434 }
3435
3436 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3437                             struct qedr_cq *cq, struct ib_wc *wc,
3438                             struct rdma_cqe_responder *resp)
3439 {
3440         u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3441
3442         __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3443
3444         while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3445                 qed_chain_consume(&qp->rq.pbl);
3446         qedr_inc_sw_cons(&qp->rq);
3447
3448         return 1;
3449 }
3450
3451 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
3452                               int num_entries, struct ib_wc *wc, u16 hw_cons)
3453 {
3454         u16 cnt = 0;
3455
3456         while (num_entries && qp->rq.wqe_cons != hw_cons) {
3457                 /* fill WC */
3458                 wc->status = IB_WC_WR_FLUSH_ERR;
3459                 wc->vendor_err = 0;
3460                 wc->wc_flags = 0;
3461                 wc->src_qp = qp->id;
3462                 wc->byte_len = 0;
3463                 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3464                 wc->qp = &qp->ibqp;
3465                 num_entries--;
3466                 wc++;
3467                 cnt++;
3468                 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3469                         qed_chain_consume(&qp->rq.pbl);
3470                 qedr_inc_sw_cons(&qp->rq);
3471         }
3472
3473         return cnt;
3474 }
3475
3476 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3477                                  struct rdma_cqe_responder *resp, int *update)
3478 {
3479         if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
3480                 consume_cqe(cq);
3481                 *update |= 1;
3482         }
3483 }
3484
3485 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
3486                              struct qedr_cq *cq, int num_entries,
3487                              struct ib_wc *wc, struct rdma_cqe_responder *resp,
3488                              int *update)
3489 {
3490         int cnt;
3491
3492         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3493                 cnt = process_resp_flush(qp, cq, num_entries, wc,
3494                                          resp->rq_cons);
3495                 try_consume_resp_cqe(cq, qp, resp, update);
3496         } else {
3497                 cnt = process_resp_one(dev, qp, cq, wc, resp);
3498                 consume_cqe(cq);
3499                 *update |= 1;
3500         }
3501
3502         return cnt;
3503 }
3504
3505 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3506                                 struct rdma_cqe_requester *req, int *update)
3507 {
3508         if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
3509                 consume_cqe(cq);
3510                 *update |= 1;
3511         }
3512 }
3513
3514 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
3515 {
3516         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
3517         struct qedr_cq *cq = get_qedr_cq(ibcq);
3518         union rdma_cqe *cqe = cq->latest_cqe;
3519         u32 old_cons, new_cons;
3520         unsigned long flags;
3521         int update = 0;
3522         int done = 0;
3523
3524         if (cq->destroyed) {
3525                 DP_ERR(dev,
3526                        "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
3527                        cq, cq->icid);
3528                 return 0;
3529         }
3530
3531         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
3532                 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
3533
3534         spin_lock_irqsave(&cq->cq_lock, flags);
3535         old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3536         while (num_entries && is_valid_cqe(cq, cqe)) {
3537                 struct qedr_qp *qp;
3538                 int cnt = 0;
3539
3540                 /* prevent speculative reads of any field of CQE */
3541                 rmb();
3542
3543                 qp = cqe_get_qp(cqe);
3544                 if (!qp) {
3545                         WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
3546                         break;
3547                 }
3548
3549                 wc->qp = &qp->ibqp;
3550
3551                 switch (cqe_get_type(cqe)) {
3552                 case RDMA_CQE_TYPE_REQUESTER:
3553                         cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
3554                                                &cqe->req);
3555                         try_consume_req_cqe(cq, qp, &cqe->req, &update);
3556                         break;
3557                 case RDMA_CQE_TYPE_RESPONDER_RQ:
3558                         cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
3559                                                 &cqe->resp, &update);
3560                         break;
3561                 case RDMA_CQE_TYPE_INVALID:
3562                 default:
3563                         DP_ERR(dev, "Error: invalid CQE type = %d\n",
3564                                cqe_get_type(cqe));
3565                 }
3566                 num_entries -= cnt;
3567                 wc += cnt;
3568                 done += cnt;
3569
3570                 cqe = get_cqe(cq);
3571         }
3572         new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3573
3574         cq->cq_cons += new_cons - old_cons;
3575
3576         if (update)
3577                 /* doorbell notifies abount latest VALID entry,
3578                  * but chain already point to the next INVALID one
3579                  */
3580                 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
3581
3582         spin_unlock_irqrestore(&cq->cq_lock, flags);
3583         return done;
3584 }
3585
3586 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
3587                      u8 port_num,
3588                      const struct ib_wc *in_wc,
3589                      const struct ib_grh *in_grh,
3590                      const struct ib_mad_hdr *mad_hdr,
3591                      size_t in_mad_size, struct ib_mad_hdr *out_mad,
3592                      size_t *out_mad_size, u16 *out_mad_pkey_index)
3593 {
3594         struct qedr_dev *dev = get_qedr_dev(ibdev);
3595
3596         DP_DEBUG(dev, QEDR_MSG_GSI,
3597                  "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
3598                  mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
3599                  mad_hdr->class_specific, mad_hdr->class_version,
3600                  mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
3601         return IB_MAD_RESULT_SUCCESS;
3602 }
3603
3604 int qedr_port_immutable(struct ib_device *ibdev, u8 port_num,
3605                         struct ib_port_immutable *immutable)
3606 {
3607         struct ib_port_attr attr;
3608         int err;
3609
3610         immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE |
3611                                     RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
3612
3613         err = ib_query_port(ibdev, port_num, &attr);
3614         if (err)
3615                 return err;
3616
3617         immutable->pkey_tbl_len = attr.pkey_tbl_len;
3618         immutable->gid_tbl_len = attr.gid_tbl_len;
3619         immutable->max_mad_size = IB_MGMT_MAD_SIZE;
3620
3621         return 0;
3622 }