drivers/infiniband/hw/ocrdma/ocrdma_verbs.c

   1 /* This file is part of the Emulex RoCE Device Driver for
   2  * RoCE (RDMA over Converged Ethernet) adapters.
   3  * Copyright (C) 2012-2015 Emulex. All rights reserved.
   4  * EMULEX and SLI are trademarks of Emulex.
   5  * www.emulex.com
   6  *
   7  * This software is available to you under a choice of one of two licenses.
   8  * You may choose to be licensed under the terms of the GNU General Public
   9  * License (GPL) Version 2, available from the file COPYING in the main
  10  * directory of this source tree, or the BSD license below:
  11  *
  12  * Redistribution and use in source and binary forms, with or without
  13  * modification, are permitted provided that the following conditions
  14  * are met:
  15  *
  16  * - Redistributions of source code must retain the above copyright notice,
  17  *   this list of conditions and the following disclaimer.
  18  *
  19  * - Redistributions in binary form must reproduce the above copyright
  20  *   notice, this list of conditions and the following disclaimer in
  21  *   the documentation and/or other materials provided with the distribution.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  24  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,THE
  25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  26  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  27  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  30  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  31  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  32  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  33  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34  *
  35  * Contact Information:
  36  * linux-drivers@emulex.com
  37  *
  38  * Emulex
  39  * 3333 Susan Street
  40  * Costa Mesa, CA 92626
  41  */
  42
  43 #include <linux/dma-mapping.h>
  44 #include <rdma/ib_verbs.h>
  45 #include <rdma/ib_user_verbs.h>
  46 #include <rdma/iw_cm.h>
  47 #include <rdma/ib_umem.h>
  48 #include <rdma/ib_addr.h>
  49 #include <rdma/ib_cache.h>
  50 #include <rdma/uverbs_ioctl.h>
  51
  52 #include "ocrdma.h"
  53 #include "ocrdma_hw.h"
  54 #include "ocrdma_verbs.h"
  55 #include <rdma/ocrdma-abi.h>
  56
  57 int ocrdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
  58 {
  59         if (index > 0)
  60                 return -EINVAL;
  61
  62         *pkey = 0xffff;
  63         return 0;
  64 }
  65
  66 int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
  67                         struct ib_udata *uhw)
  68 {
  69         struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
  70
  71         if (uhw->inlen || uhw->outlen)
  72                 return -EINVAL;
  73
  74         memset(attr, 0, sizeof *attr);
  75         memcpy(&attr->fw_ver, &dev->attr.fw_ver[0],
  76                min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver)));
  77         ocrdma_get_guid(dev, (u8 *)&attr->sys_image_guid);
  78         attr->max_mr_size = dev->attr.max_mr_size;
  79         attr->page_size_cap = 0xffff000;
  80         attr->vendor_id = dev->nic_info.pdev->vendor;
  81         attr->vendor_part_id = dev->nic_info.pdev->device;
  82         attr->hw_ver = dev->asic_id;
  83         attr->max_qp = dev->attr.max_qp;
  84         attr->max_ah = OCRDMA_MAX_AH;
  85         attr->max_qp_wr = dev->attr.max_wqe;
  86
  87         attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
  88                                         IB_DEVICE_RC_RNR_NAK_GEN |
  89                                         IB_DEVICE_SHUTDOWN_PORT |
  90                                         IB_DEVICE_SYS_IMAGE_GUID |
  91                                         IB_DEVICE_LOCAL_DMA_LKEY |
  92                                         IB_DEVICE_MEM_MGT_EXTENSIONS;
  93         attr->max_send_sge = dev->attr.max_send_sge;
  94         attr->max_recv_sge = dev->attr.max_recv_sge;
  95         attr->max_sge_rd = dev->attr.max_rdma_sge;
  96         attr->max_cq = dev->attr.max_cq;
  97         attr->max_cqe = dev->attr.max_cqe;
  98         attr->max_mr = dev->attr.max_mr;
  99         attr->max_mw = dev->attr.max_mw;
 100         attr->max_pd = dev->attr.max_pd;
 101         attr->atomic_cap = 0;
 102         attr->max_fmr = 0;
 103         attr->max_map_per_fmr = 0;
 104         attr->max_qp_rd_atom =
 105             min(dev->attr.max_ord_per_qp, dev->attr.max_ird_per_qp);
 106         attr->max_qp_init_rd_atom = dev->attr.max_ord_per_qp;
 107         attr->max_srq = dev->attr.max_srq;
 108         attr->max_srq_sge = dev->attr.max_srq_sge;
 109         attr->max_srq_wr = dev->attr.max_rqe;
 110         attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay;
 111         attr->max_fast_reg_page_list_len = dev->attr.max_pages_per_frmr;
 112         attr->max_pkeys = 1;
 113         return 0;
 114 }
 115
 116 static inline void get_link_speed_and_width(struct ocrdma_dev *dev,
 117                                             u8 *ib_speed, u8 *ib_width)
 118 {
 119         int status;
 120         u8 speed;
 121
 122         status = ocrdma_mbx_get_link_speed(dev, &speed, NULL);
 123         if (status)
 124                 speed = OCRDMA_PHYS_LINK_SPEED_ZERO;
 125
 126         switch (speed) {
 127         case OCRDMA_PHYS_LINK_SPEED_1GBPS:
 128                 *ib_speed = IB_SPEED_SDR;
 129                 *ib_width = IB_WIDTH_1X;
 130                 break;
 131
 132         case OCRDMA_PHYS_LINK_SPEED_10GBPS:
 133                 *ib_speed = IB_SPEED_QDR;
 134                 *ib_width = IB_WIDTH_1X;
 135                 break;
 136
 137         case OCRDMA_PHYS_LINK_SPEED_20GBPS:
 138                 *ib_speed = IB_SPEED_DDR;
 139                 *ib_width = IB_WIDTH_4X;
 140                 break;
 141
 142         case OCRDMA_PHYS_LINK_SPEED_40GBPS:
 143                 *ib_speed = IB_SPEED_QDR;
 144                 *ib_width = IB_WIDTH_4X;
 145                 break;
 146
 147         default:
 148                 /* Unsupported */
 149                 *ib_speed = IB_SPEED_SDR;
 150                 *ib_width = IB_WIDTH_1X;
 151         }
 152 }
 153
 154 int ocrdma_query_port(struct ib_device *ibdev,
 155                       u8 port, struct ib_port_attr *props)
 156 {
 157         enum ib_port_state port_state;
 158         struct ocrdma_dev *dev;
 159         struct net_device *netdev;
 160
 161         /* props being zeroed by the caller, avoid zeroing it here */
 162         dev = get_ocrdma_dev(ibdev);
 163         netdev = dev->nic_info.netdev;
 164         if (netif_running(netdev) && netif_oper_up(netdev)) {
 165                 port_state = IB_PORT_ACTIVE;
 166                 props->phys_state = 5;
 167         } else {
 168                 port_state = IB_PORT_DOWN;
 169                 props->phys_state = 3;
 170         }
 171         props->max_mtu = IB_MTU_4096;
 172         props->active_mtu = iboe_get_mtu(netdev->mtu);
 173         props->lid = 0;
 174         props->lmc = 0;
 175         props->sm_lid = 0;
 176         props->sm_sl = 0;
 177         props->state = port_state;
 178         props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
 179                                 IB_PORT_DEVICE_MGMT_SUP |
 180                                 IB_PORT_VENDOR_CLASS_SUP;
 181         props->ip_gids = true;
 182         props->gid_tbl_len = OCRDMA_MAX_SGID;
 183         props->pkey_tbl_len = 1;
 184         props->bad_pkey_cntr = 0;
 185         props->qkey_viol_cntr = 0;
 186         get_link_speed_and_width(dev, &props->active_speed,
 187                                  &props->active_width);
 188         props->max_msg_sz = 0x80000000;
 189         props->max_vl_num = 4;
 190         return 0;
 191 }
 192
 193 int ocrdma_modify_port(struct ib_device *ibdev, u8 port, int mask,
 194                        struct ib_port_modify *props)
 195 {
 196         return 0;
 197 }
 198
 199 static int ocrdma_add_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
 200                            unsigned long len)
 201 {
 202         struct ocrdma_mm *mm;
 203
 204         mm = kzalloc(sizeof(*mm), GFP_KERNEL);
 205         if (mm == NULL)
 206                 return -ENOMEM;
 207         mm->key.phy_addr = phy_addr;
 208         mm->key.len = len;
 209         INIT_LIST_HEAD(&mm->entry);
 210
 211         mutex_lock(&uctx->mm_list_lock);
 212         list_add_tail(&mm->entry, &uctx->mm_head);
 213         mutex_unlock(&uctx->mm_list_lock);
 214         return 0;
 215 }
 216
 217 static void ocrdma_del_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
 218                             unsigned long len)
 219 {
 220         struct ocrdma_mm *mm, *tmp;
 221
 222         mutex_lock(&uctx->mm_list_lock);
 223         list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
 224                 if (len != mm->key.len && phy_addr != mm->key.phy_addr)
 225                         continue;
 226
 227                 list_del(&mm->entry);
 228                 kfree(mm);
 229                 break;
 230         }
 231         mutex_unlock(&uctx->mm_list_lock);
 232 }
 233
 234 static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
 235                               unsigned long len)
 236 {
 237         bool found = false;
 238         struct ocrdma_mm *mm;
 239
 240         mutex_lock(&uctx->mm_list_lock);
 241         list_for_each_entry(mm, &uctx->mm_head, entry) {
 242                 if (len != mm->key.len && phy_addr != mm->key.phy_addr)
 243                         continue;
 244
 245                 found = true;
 246                 break;
 247         }
 248         mutex_unlock(&uctx->mm_list_lock);
 249         return found;
 250 }
 251
 252
 253 static u16 _ocrdma_pd_mgr_get_bitmap(struct ocrdma_dev *dev, bool dpp_pool)
 254 {
 255         u16 pd_bitmap_idx = 0;
 256         const unsigned long *pd_bitmap;
 257
 258         if (dpp_pool) {
 259                 pd_bitmap = dev->pd_mgr->pd_dpp_bitmap;
 260                 pd_bitmap_idx = find_first_zero_bit(pd_bitmap,
 261                                                     dev->pd_mgr->max_dpp_pd);
 262                 __set_bit(pd_bitmap_idx, dev->pd_mgr->pd_dpp_bitmap);
 263                 dev->pd_mgr->pd_dpp_count++;
 264                 if (dev->pd_mgr->pd_dpp_count > dev->pd_mgr->pd_dpp_thrsh)
 265                         dev->pd_mgr->pd_dpp_thrsh = dev->pd_mgr->pd_dpp_count;
 266         } else {
 267                 pd_bitmap = dev->pd_mgr->pd_norm_bitmap;
 268                 pd_bitmap_idx = find_first_zero_bit(pd_bitmap,
 269                                                     dev->pd_mgr->max_normal_pd);
 270                 __set_bit(pd_bitmap_idx, dev->pd_mgr->pd_norm_bitmap);
 271                 dev->pd_mgr->pd_norm_count++;
 272                 if (dev->pd_mgr->pd_norm_count > dev->pd_mgr->pd_norm_thrsh)
 273                         dev->pd_mgr->pd_norm_thrsh = dev->pd_mgr->pd_norm_count;
 274         }
 275         return pd_bitmap_idx;
 276 }
 277
 278 static int _ocrdma_pd_mgr_put_bitmap(struct ocrdma_dev *dev, u16 pd_id,
 279                                         bool dpp_pool)
 280 {
 281         u16 pd_count;
 282         u16 pd_bit_index;
 283
 284         pd_count = dpp_pool ? dev->pd_mgr->pd_dpp_count :
 285                               dev->pd_mgr->pd_norm_count;
 286         if (pd_count == 0)
 287                 return -EINVAL;
 288
 289         if (dpp_pool) {
 290                 pd_bit_index = pd_id - dev->pd_mgr->pd_dpp_start;
 291                 if (pd_bit_index >= dev->pd_mgr->max_dpp_pd) {
 292                         return -EINVAL;
 293                 } else {
 294                         __clear_bit(pd_bit_index, dev->pd_mgr->pd_dpp_bitmap);
 295                         dev->pd_mgr->pd_dpp_count--;
 296                 }
 297         } else {
 298                 pd_bit_index = pd_id - dev->pd_mgr->pd_norm_start;
 299                 if (pd_bit_index >= dev->pd_mgr->max_normal_pd) {
 300                         return -EINVAL;
 301                 } else {
 302                         __clear_bit(pd_bit_index, dev->pd_mgr->pd_norm_bitmap);
 303                         dev->pd_mgr->pd_norm_count--;
 304                 }
 305         }
 306
 307         return 0;
 308 }
 309
 310 static int ocrdma_put_pd_num(struct ocrdma_dev *dev, u16 pd_id,
 311                                    bool dpp_pool)
 312 {
 313         int status;
 314
 315         mutex_lock(&dev->dev_lock);
 316         status = _ocrdma_pd_mgr_put_bitmap(dev, pd_id, dpp_pool);
 317         mutex_unlock(&dev->dev_lock);
 318         return status;
 319 }
 320
 321 static int ocrdma_get_pd_num(struct ocrdma_dev *dev, struct ocrdma_pd *pd)
 322 {
 323         u16 pd_idx = 0;
 324         int status = 0;
 325
 326         mutex_lock(&dev->dev_lock);
 327         if (pd->dpp_enabled) {
 328                 /* try allocating DPP PD, if not available then normal PD */
 329                 if (dev->pd_mgr->pd_dpp_count < dev->pd_mgr->max_dpp_pd) {
 330                         pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, true);
 331                         pd->id = dev->pd_mgr->pd_dpp_start + pd_idx;
 332                         pd->dpp_page = dev->pd_mgr->dpp_page_index + pd_idx;
 333                 } else if (dev->pd_mgr->pd_norm_count <
 334                            dev->pd_mgr->max_normal_pd) {
 335                         pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false);
 336                         pd->id = dev->pd_mgr->pd_norm_start + pd_idx;
 337                         pd->dpp_enabled = false;
 338                 } else {
 339                         status = -EINVAL;
 340                 }
 341         } else {
 342                 if (dev->pd_mgr->pd_norm_count < dev->pd_mgr->max_normal_pd) {
 343                         pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false);
 344                         pd->id = dev->pd_mgr->pd_norm_start + pd_idx;
 345                 } else {
 346                         status = -EINVAL;
 347                 }
 348         }
 349         mutex_unlock(&dev->dev_lock);
 350         return status;
 351 }
 352
 353 /*
 354  * NOTE:
 355  *
 356  * ocrdma_ucontext must be used here because this function is also
 357  * called from ocrdma_alloc_ucontext where ib_udata does not have
 358  * valid ib_ucontext pointer. ib_uverbs_get_context does not call
 359  * uobj_{alloc|get_xxx} helpers which are used to store the
 360  * ib_ucontext in uverbs_attr_bundle wrapping the ib_udata. so
 361  * ib_udata does NOT imply valid ib_ucontext here!
 362  */
 363 static int _ocrdma_alloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd,
 364                             struct ocrdma_ucontext *uctx,
 365                             struct ib_udata *udata)
 366 {
 367         int status;
 368
 369         if (udata && uctx && dev->attr.max_dpp_pds) {
 370                 pd->dpp_enabled =
 371                         ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R;
 372                 pd->num_dpp_qp =
 373                         pd->dpp_enabled ? (dev->nic_info.db_page_size /
 374                                            dev->attr.wqe_size) : 0;
 375         }
 376
 377         if (dev->pd_mgr->pd_prealloc_valid)
 378                 return ocrdma_get_pd_num(dev, pd);
 379
 380 retry:
 381         status = ocrdma_mbx_alloc_pd(dev, pd);
 382         if (status) {
 383                 if (pd->dpp_enabled) {
 384                         pd->dpp_enabled = false;
 385                         pd->num_dpp_qp = 0;
 386                         goto retry;
 387                 }
 388                 return status;
 389         }
 390
 391         return 0;
 392 }
 393
 394 static inline int is_ucontext_pd(struct ocrdma_ucontext *uctx,
 395                                  struct ocrdma_pd *pd)
 396 {
 397         return (uctx->cntxt_pd == pd);
 398 }
 399
 400 static void _ocrdma_dealloc_pd(struct ocrdma_dev *dev,
 401                               struct ocrdma_pd *pd)
 402 {
 403         if (dev->pd_mgr->pd_prealloc_valid)
 404                 ocrdma_put_pd_num(dev, pd->id, pd->dpp_enabled);
 405         else
 406                 ocrdma_mbx_dealloc_pd(dev, pd);
 407 }
 408
 409 static int ocrdma_alloc_ucontext_pd(struct ocrdma_dev *dev,
 410                                     struct ocrdma_ucontext *uctx,
 411                                     struct ib_udata *udata)
 412 {
 413         struct ib_device *ibdev = &dev->ibdev;
 414         struct ib_pd *pd;
 415         int status;
 416
 417         pd = rdma_zalloc_drv_obj(ibdev, ib_pd);
 418         if (!pd)
 419                 return -ENOMEM;
 420
 421         pd->device  = ibdev;
 422         uctx->cntxt_pd = get_ocrdma_pd(pd);
 423
 424         status = _ocrdma_alloc_pd(dev, uctx->cntxt_pd, uctx, udata);
 425         if (status) {
 426                 kfree(uctx->cntxt_pd);
 427                 goto err;
 428         }
 429
 430         uctx->cntxt_pd->uctx = uctx;
 431         uctx->cntxt_pd->ibpd.device = &dev->ibdev;
 432 err:
 433         return status;
 434 }
 435
 436 static void ocrdma_dealloc_ucontext_pd(struct ocrdma_ucontext *uctx)
 437 {
 438         struct ocrdma_pd *pd = uctx->cntxt_pd;
 439         struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device);
 440
 441         if (uctx->pd_in_use) {
 442                 pr_err("%s(%d) Freeing in use pdid=0x%x.\n",
 443                        __func__, dev->id, pd->id);
 444         }
 445         kfree(uctx->cntxt_pd);
 446         uctx->cntxt_pd = NULL;
 447         _ocrdma_dealloc_pd(dev, pd);
 448 }
 449
 450 static struct ocrdma_pd *ocrdma_get_ucontext_pd(struct ocrdma_ucontext *uctx)
 451 {
 452         struct ocrdma_pd *pd = NULL;
 453
 454         mutex_lock(&uctx->mm_list_lock);
 455         if (!uctx->pd_in_use) {
 456                 uctx->pd_in_use = true;
 457                 pd = uctx->cntxt_pd;
 458         }
 459         mutex_unlock(&uctx->mm_list_lock);
 460
 461         return pd;
 462 }
 463
 464 static void ocrdma_release_ucontext_pd(struct ocrdma_ucontext *uctx)
 465 {
 466         mutex_lock(&uctx->mm_list_lock);
 467         uctx->pd_in_use = false;
 468         mutex_unlock(&uctx->mm_list_lock);
 469 }
 470
 471 int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
 472 {
 473         struct ib_device *ibdev = uctx->device;
 474         int status;
 475         struct ocrdma_ucontext *ctx = get_ocrdma_ucontext(uctx);
 476         struct ocrdma_alloc_ucontext_resp resp = {};
 477         struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
 478         struct pci_dev *pdev = dev->nic_info.pdev;
 479         u32 map_len = roundup(sizeof(u32) * 2048, PAGE_SIZE);
 480
 481         if (!udata)
 482                 return -EFAULT;
 483         INIT_LIST_HEAD(&ctx->mm_head);
 484         mutex_init(&ctx->mm_list_lock);
 485
 486         ctx->ah_tbl.va = dma_alloc_coherent(&pdev->dev, map_len,
 487                                             &ctx->ah_tbl.pa, GFP_KERNEL);
 488         if (!ctx->ah_tbl.va)
 489                 return -ENOMEM;
 490
 491         ctx->ah_tbl.len = map_len;
 492
 493         resp.ah_tbl_len = ctx->ah_tbl.len;
 494         resp.ah_tbl_page = virt_to_phys(ctx->ah_tbl.va);
 495
 496         status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len);
 497         if (status)
 498                 goto map_err;
 499
 500         status = ocrdma_alloc_ucontext_pd(dev, ctx, udata);
 501         if (status)
 502                 goto pd_err;
 503
 504         resp.dev_id = dev->id;
 505         resp.max_inline_data = dev->attr.max_inline_data;
 506         resp.wqe_size = dev->attr.wqe_size;
 507         resp.rqe_size = dev->attr.rqe_size;
 508         resp.dpp_wqe_size = dev->attr.wqe_size;
 509
 510         memcpy(resp.fw_ver, dev->attr.fw_ver, sizeof(resp.fw_ver));
 511         status = ib_copy_to_udata(udata, &resp, sizeof(resp));
 512         if (status)
 513                 goto cpy_err;
 514         return 0;
 515
 516 cpy_err:
 517         ocrdma_dealloc_ucontext_pd(ctx);
 518 pd_err:
 519         ocrdma_del_mmap(ctx, ctx->ah_tbl.pa, ctx->ah_tbl.len);
 520 map_err:
 521         dma_free_coherent(&pdev->dev, ctx->ah_tbl.len, ctx->ah_tbl.va,
 522                           ctx->ah_tbl.pa);
 523         return status;
 524 }
 525
 526 void ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx)
 527 {
 528         struct ocrdma_mm *mm, *tmp;
 529         struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ibctx);
 530         struct ocrdma_dev *dev = get_ocrdma_dev(ibctx->device);
 531         struct pci_dev *pdev = dev->nic_info.pdev;
 532
 533         ocrdma_dealloc_ucontext_pd(uctx);
 534
 535         ocrdma_del_mmap(uctx, uctx->ah_tbl.pa, uctx->ah_tbl.len);
 536         dma_free_coherent(&pdev->dev, uctx->ah_tbl.len, uctx->ah_tbl.va,
 537                           uctx->ah_tbl.pa);
 538
 539         list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
 540                 list_del(&mm->entry);
 541                 kfree(mm);
 542         }
 543 }
 544
 545 int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 546 {
 547         struct ocrdma_ucontext *ucontext = get_ocrdma_ucontext(context);
 548         struct ocrdma_dev *dev = get_ocrdma_dev(context->device);
 549         unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
 550         u64 unmapped_db = (u64) dev->nic_info.unmapped_db;
 551         unsigned long len = (vma->vm_end - vma->vm_start);
 552         int status;
 553         bool found;
 554
 555         if (vma->vm_start & (PAGE_SIZE - 1))
 556                 return -EINVAL;
 557         found = ocrdma_search_mmap(ucontext, vma->vm_pgoff << PAGE_SHIFT, len);
 558         if (!found)
 559                 return -EINVAL;
 560
 561         if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
 562                 dev->nic_info.db_total_size)) &&
 563                 (len <= dev->nic_info.db_page_size)) {
 564                 if (vma->vm_flags & VM_READ)
 565                         return -EPERM;
 566
 567                 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 568                 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
 569                                             len, vma->vm_page_prot);
 570         } else if (dev->nic_info.dpp_unmapped_len &&
 571                 (vm_page >= (u64) dev->nic_info.dpp_unmapped_addr) &&
 572                 (vm_page <= (u64) (dev->nic_info.dpp_unmapped_addr +
 573                         dev->nic_info.dpp_unmapped_len)) &&
 574                 (len <= dev->nic_info.dpp_unmapped_len)) {
 575                 if (vma->vm_flags & VM_READ)
 576                         return -EPERM;
 577
 578                 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 579                 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
 580                                             len, vma->vm_page_prot);
 581         } else {
 582                 status = remap_pfn_range(vma, vma->vm_start,
 583                                          vma->vm_pgoff, len, vma->vm_page_prot);
 584         }
 585         return status;
 586 }
 587
 588 static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd,
 589                                 struct ib_udata *udata)
 590 {
 591         int status;
 592         u64 db_page_addr;
 593         u64 dpp_page_addr = 0;
 594         u32 db_page_size;
 595         struct ocrdma_alloc_pd_uresp rsp;
 596         struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
 597                 udata, struct ocrdma_ucontext, ibucontext);
 598
 599         memset(&rsp, 0, sizeof(rsp));
 600         rsp.id = pd->id;
 601         rsp.dpp_enabled = pd->dpp_enabled;
 602         db_page_addr = ocrdma_get_db_addr(dev, pd->id);
 603         db_page_size = dev->nic_info.db_page_size;
 604
 605         status = ocrdma_add_mmap(uctx, db_page_addr, db_page_size);
 606         if (status)
 607                 return status;
 608
 609         if (pd->dpp_enabled) {
 610                 dpp_page_addr = dev->nic_info.dpp_unmapped_addr +
 611                                 (pd->id * PAGE_SIZE);
 612                 status = ocrdma_add_mmap(uctx, dpp_page_addr,
 613                                  PAGE_SIZE);
 614                 if (status)
 615                         goto dpp_map_err;
 616                 rsp.dpp_page_addr_hi = upper_32_bits(dpp_page_addr);
 617                 rsp.dpp_page_addr_lo = dpp_page_addr;
 618         }
 619
 620         status = ib_copy_to_udata(udata, &rsp, sizeof(rsp));
 621         if (status)
 622                 goto ucopy_err;
 623
 624         pd->uctx = uctx;
 625         return 0;
 626
 627 ucopy_err:
 628         if (pd->dpp_enabled)
 629                 ocrdma_del_mmap(pd->uctx, dpp_page_addr, PAGE_SIZE);
 630 dpp_map_err:
 631         ocrdma_del_mmap(pd->uctx, db_page_addr, db_page_size);
 632         return status;
 633 }
 634
 635 int ocrdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 636 {
 637         struct ib_device *ibdev = ibpd->device;
 638         struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
 639         struct ocrdma_pd *pd;
 640         int status;
 641         u8 is_uctx_pd = false;
 642         struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
 643                 udata, struct ocrdma_ucontext, ibucontext);
 644
 645         if (udata) {
 646                 pd = ocrdma_get_ucontext_pd(uctx);
 647                 if (pd) {
 648                         is_uctx_pd = true;
 649                         goto pd_mapping;
 650                 }
 651         }
 652
 653         pd = get_ocrdma_pd(ibpd);
 654         status = _ocrdma_alloc_pd(dev, pd, uctx, udata);
 655         if (status)
 656                 goto exit;
 657
 658 pd_mapping:
 659         if (udata) {
 660                 status = ocrdma_copy_pd_uresp(dev, pd, udata);
 661                 if (status)
 662                         goto err;
 663         }
 664         return 0;
 665
 666 err:
 667         if (is_uctx_pd)
 668                 ocrdma_release_ucontext_pd(uctx);
 669         else
 670                 _ocrdma_dealloc_pd(dev, pd);
 671 exit:
 672         return status;
 673 }
 674
 675 void ocrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 676 {
 677         struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
 678         struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
 679         struct ocrdma_ucontext *uctx = NULL;
 680         u64 usr_db;
 681
 682         uctx = pd->uctx;
 683         if (uctx) {
 684                 u64 dpp_db = dev->nic_info.dpp_unmapped_addr +
 685                         (pd->id * PAGE_SIZE);
 686                 if (pd->dpp_enabled)
 687                         ocrdma_del_mmap(pd->uctx, dpp_db, PAGE_SIZE);
 688                 usr_db = ocrdma_get_db_addr(dev, pd->id);
 689                 ocrdma_del_mmap(pd->uctx, usr_db, dev->nic_info.db_page_size);
 690
 691                 if (is_ucontext_pd(uctx, pd)) {
 692                         ocrdma_release_ucontext_pd(uctx);
 693                         return;
 694                 }
 695         }
 696         _ocrdma_dealloc_pd(dev, pd);
 697 }
 698
 699 static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
 700                             u32 pdid, int acc, u32 num_pbls, u32 addr_check)
 701 {
 702         int status;
 703
 704         mr->hwmr.fr_mr = 0;
 705         mr->hwmr.local_rd = 1;
 706         mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
 707         mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
 708         mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
 709         mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0;
 710         mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
 711         mr->hwmr.num_pbls = num_pbls;
 712
 713         status = ocrdma_mbx_alloc_lkey(dev, &mr->hwmr, pdid, addr_check);
 714         if (status)
 715                 return status;
 716
 717         mr->ibmr.lkey = mr->hwmr.lkey;
 718         if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
 719                 mr->ibmr.rkey = mr->hwmr.lkey;
 720         return 0;
 721 }
 722
 723 struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *ibpd, int acc)
 724 {
 725         int status;
 726         struct ocrdma_mr *mr;
 727         struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
 728         struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
 729
 730         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
 731                 pr_err("%s err, invalid access rights\n", __func__);
 732                 return ERR_PTR(-EINVAL);
 733         }
 734
 735         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 736         if (!mr)
 737                 return ERR_PTR(-ENOMEM);
 738
 739         status = ocrdma_alloc_lkey(dev, mr, pd->id, acc, 0,
 740                                    OCRDMA_ADDR_CHECK_DISABLE);
 741         if (status) {
 742                 kfree(mr);
 743                 return ERR_PTR(status);
 744         }
 745
 746         return &mr->ibmr;
 747 }
 748
 749 static void ocrdma_free_mr_pbl_tbl(struct ocrdma_dev *dev,
 750                                    struct ocrdma_hw_mr *mr)
 751 {
 752         struct pci_dev *pdev = dev->nic_info.pdev;
 753         int i = 0;
 754
 755         if (mr->pbl_table) {
 756                 for (i = 0; i < mr->num_pbls; i++) {
 757                         if (!mr->pbl_table[i].va)
 758                                 continue;
 759                         dma_free_coherent(&pdev->dev, mr->pbl_size,
 760                                           mr->pbl_table[i].va,
 761                                           mr->pbl_table[i].pa);
 762                 }
 763                 kfree(mr->pbl_table);
 764                 mr->pbl_table = NULL;
 765         }
 766 }
 767
 768 static int ocrdma_get_pbl_info(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
 769                               u32 num_pbes)
 770 {
 771         u32 num_pbls = 0;
 772         u32 idx = 0;
 773         int status = 0;
 774         u32 pbl_size;
 775
 776         do {
 777                 pbl_size = OCRDMA_MIN_HPAGE_SIZE * (1 << idx);
 778                 if (pbl_size > MAX_OCRDMA_PBL_SIZE) {
 779                         status = -EFAULT;
 780                         break;
 781                 }
 782                 num_pbls = roundup(num_pbes, (pbl_size / sizeof(u64)));
 783                 num_pbls = num_pbls / (pbl_size / sizeof(u64));
 784                 idx++;
 785         } while (num_pbls >= dev->attr.max_num_mr_pbl);
 786
 787         mr->hwmr.num_pbes = num_pbes;
 788         mr->hwmr.num_pbls = num_pbls;
 789         mr->hwmr.pbl_size = pbl_size;
 790         return status;
 791 }
 792
 793 static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr)
 794 {
 795         int status = 0;
 796         int i;
 797         u32 dma_len = mr->pbl_size;
 798         struct pci_dev *pdev = dev->nic_info.pdev;
 799         void *va;
 800         dma_addr_t pa;
 801
 802         mr->pbl_table = kcalloc(mr->num_pbls, sizeof(struct ocrdma_pbl),
 803                                 GFP_KERNEL);
 804
 805         if (!mr->pbl_table)
 806                 return -ENOMEM;
 807
 808         for (i = 0; i < mr->num_pbls; i++) {
 809                 va = dma_alloc_coherent(&pdev->dev, dma_len, &pa, GFP_KERNEL);
 810                 if (!va) {
 811                         ocrdma_free_mr_pbl_tbl(dev, mr);
 812                         status = -ENOMEM;
 813                         break;
 814                 }
 815                 mr->pbl_table[i].va = va;
 816                 mr->pbl_table[i].pa = pa;
 817         }
 818         return status;
 819 }
 820
 821 static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
 822                             u32 num_pbes)
 823 {
 824         struct ocrdma_pbe *pbe;
 825         struct sg_dma_page_iter sg_iter;
 826         struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;
 827         struct ib_umem *umem = mr->umem;
 828         int pbe_cnt, total_num_pbes = 0;
 829         u64 pg_addr;
 830
 831         if (!mr->hwmr.num_pbes)
 832                 return;
 833
 834         pbe = (struct ocrdma_pbe *)pbl_tbl->va;
 835         pbe_cnt = 0;
 836
 837         for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
 838                 /* store the page address in pbe */
 839                 pg_addr = sg_page_iter_dma_address(&sg_iter);
 840                 pbe->pa_lo = cpu_to_le32(pg_addr);
 841                 pbe->pa_hi = cpu_to_le32(upper_32_bits(pg_addr));
 842                 pbe_cnt += 1;
 843                 total_num_pbes += 1;
 844                 pbe++;
 845
 846                 /* if done building pbes, issue the mbx cmd. */
 847                 if (total_num_pbes == num_pbes)
 848                         return;
 849
 850                 /* if the given pbl is full storing the pbes,
 851                  * move to next pbl.
 852                  */
 853                 if (pbe_cnt == (mr->hwmr.pbl_size / sizeof(u64))) {
 854                         pbl_tbl++;
 855                         pbe = (struct ocrdma_pbe *)pbl_tbl->va;
 856                         pbe_cnt = 0;
 857                 }
 858         }
 859 }
 860
 861 struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 862                                  u64 usr_addr, int acc, struct ib_udata *udata)
 863 {
 864         int status = -ENOMEM;
 865         struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
 866         struct ocrdma_mr *mr;
 867         struct ocrdma_pd *pd;
 868         u32 num_pbes;
 869
 870         pd = get_ocrdma_pd(ibpd);
 871
 872         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
 873                 return ERR_PTR(-EINVAL);
 874
 875         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 876         if (!mr)
 877                 return ERR_PTR(status);
 878         mr->umem = ib_umem_get(udata, start, len, acc, 0);
 879         if (IS_ERR(mr->umem)) {
 880                 status = -EFAULT;
 881                 goto umem_err;
 882         }
 883         num_pbes = ib_umem_page_count(mr->umem);
 884         status = ocrdma_get_pbl_info(dev, mr, num_pbes);
 885         if (status)
 886                 goto umem_err;
 887
 888         mr->hwmr.pbe_size = PAGE_SIZE;
 889         mr->hwmr.fbo = ib_umem_offset(mr->umem);
 890         mr->hwmr.va = usr_addr;
 891         mr->hwmr.len = len;
 892         mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
 893         mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
 894         mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
 895         mr->hwmr.local_rd = 1;
 896         mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
 897         status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);
 898         if (status)
 899                 goto umem_err;
 900         build_user_pbes(dev, mr, num_pbes);
 901         status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc);
 902         if (status)
 903                 goto mbx_err;
 904         mr->ibmr.lkey = mr->hwmr.lkey;
 905         if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
 906                 mr->ibmr.rkey = mr->hwmr.lkey;
 907
 908         return &mr->ibmr;
 909
 910 mbx_err:
 911         ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
 912 umem_err:
 913         kfree(mr);
 914         return ERR_PTR(status);
 915 }
 916
 917 int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
 918 {
 919         struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr);
 920         struct ocrdma_dev *dev = get_ocrdma_dev(ib_mr->device);
 921
 922         (void) ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey);
 923
 924         kfree(mr->pages);
 925         ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
 926
 927         /* it could be user registered memory. */
 928         ib_umem_release(mr->umem);
 929         kfree(mr);
 930
 931         /* Don't stop cleanup, in case FW is unresponsive */
 932         if (dev->mqe_ctx.fw_error_state) {
 933                 pr_err("%s(%d) fw not responding.\n",
 934                        __func__, dev->id);
 935         }
 936         return 0;
 937 }
 938
 939 static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
 940                                 struct ib_udata *udata)
 941 {
 942         int status;
 943         struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
 944                 udata, struct ocrdma_ucontext, ibucontext);
 945         struct ocrdma_create_cq_uresp uresp;
 946
 947         /* this must be user flow! */
 948         if (!udata)
 949                 return -EINVAL;
 950
 951         memset(&uresp, 0, sizeof(uresp));
 952         uresp.cq_id = cq->id;
 953         uresp.page_size = PAGE_ALIGN(cq->len);
 954         uresp.num_pages = 1;
 955         uresp.max_hw_cqe = cq->max_hw_cqe;
 956         uresp.page_addr[0] = virt_to_phys(cq->va);
 957         uresp.db_page_addr =  ocrdma_get_db_addr(dev, uctx->cntxt_pd->id);
 958         uresp.db_page_size = dev->nic_info.db_page_size;
 959         uresp.phase_change = cq->phase_change ? 1 : 0;
 960         status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 961         if (status) {
 962                 pr_err("%s(%d) copy error cqid=0x%x.\n",
 963                        __func__, dev->id, cq->id);
 964                 goto err;
 965         }
 966         status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size);
 967         if (status)
 968                 goto err;
 969         status = ocrdma_add_mmap(uctx, uresp.page_addr[0], uresp.page_size);
 970         if (status) {
 971                 ocrdma_del_mmap(uctx, uresp.db_page_addr, uresp.db_page_size);
 972                 goto err;
 973         }
 974         cq->ucontext = uctx;
 975 err:
 976         return status;
 977 }
 978
 979 int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 980                      struct ib_udata *udata)
 981 {
 982         struct ib_device *ibdev = ibcq->device;
 983         int entries = attr->cqe;
 984         struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
 985         struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
 986         struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
 987                 udata, struct ocrdma_ucontext, ibucontext);
 988         u16 pd_id = 0;
 989         int status;
 990         struct ocrdma_create_cq_ureq ureq;
 991
 992         if (attr->flags)
 993                 return -EINVAL;
 994
 995         if (udata) {
 996                 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
 997                         return -EFAULT;
 998         } else
 999                 ureq.dpp_cq = 0;
1000
1001         spin_lock_init(&cq->cq_lock);
1002         spin_lock_init(&cq->comp_handler_lock);
1003         INIT_LIST_HEAD(&cq->sq_head);
1004         INIT_LIST_HEAD(&cq->rq_head);
1005
1006         if (udata)
1007                 pd_id = uctx->cntxt_pd->id;
1008
1009         status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq, pd_id);
1010         if (status)
1011                 return status;
1012
1013         if (udata) {
1014                 status = ocrdma_copy_cq_uresp(dev, cq, udata);
1015                 if (status)
1016                         goto ctx_err;
1017         }
1018         cq->phase = OCRDMA_CQE_VALID;
1019         dev->cq_tbl[cq->id] = cq;
1020         return 0;
1021
1022 ctx_err:
1023         ocrdma_mbx_destroy_cq(dev, cq);
1024         return status;
1025 }
1026
1027 int ocrdma_resize_cq(struct ib_cq *ibcq, int new_cnt,
1028                      struct ib_udata *udata)
1029 {
1030         int status = 0;
1031         struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
1032
1033         if (new_cnt < 1 || new_cnt > cq->max_hw_cqe) {
1034                 status = -EINVAL;
1035                 return status;
1036         }
1037         ibcq->cqe = new_cnt;
1038         return status;
1039 }
1040
1041 static void ocrdma_flush_cq(struct ocrdma_cq *cq)
1042 {
1043         int cqe_cnt;
1044         int valid_count = 0;
1045         unsigned long flags;
1046
1047         struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device);
1048         struct ocrdma_cqe *cqe = NULL;
1049
1050         cqe = cq->va;
1051         cqe_cnt = cq->cqe_cnt;
1052
1053         /* Last irq might have scheduled a polling thread
1054          * sync-up with it before hard flushing.
1055          */
1056         spin_lock_irqsave(&cq->cq_lock, flags);
1057         while (cqe_cnt) {
1058                 if (is_cqe_valid(cq, cqe))
1059                         valid_count++;
1060                 cqe++;
1061                 cqe_cnt--;
1062         }
1063         ocrdma_ring_cq_db(dev, cq->id, false, false, valid_count);
1064         spin_unlock_irqrestore(&cq->cq_lock, flags);
1065 }
1066
1067 void ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1068 {
1069         struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
1070         struct ocrdma_eq *eq = NULL;
1071         struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
1072         int pdid = 0;
1073         u32 irq, indx;
1074
1075         dev->cq_tbl[cq->id] = NULL;
1076         indx = ocrdma_get_eq_table_index(dev, cq->eqn);
1077
1078         eq = &dev->eq_tbl[indx];
1079         irq = ocrdma_get_irq(dev, eq);
1080         synchronize_irq(irq);
1081         ocrdma_flush_cq(cq);
1082
1083         ocrdma_mbx_destroy_cq(dev, cq);
1084         if (cq->ucontext) {
1085                 pdid = cq->ucontext->cntxt_pd->id;
1086                 ocrdma_del_mmap(cq->ucontext, (u64) cq->pa,
1087                                 PAGE_ALIGN(cq->len));
1088                 ocrdma_del_mmap(cq->ucontext,
1089                                 ocrdma_get_db_addr(dev, pdid),
1090                                 dev->nic_info.db_page_size);
1091         }
1092 }
1093
1094 static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
1095 {
1096         int status = -EINVAL;
1097
1098         if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) {
1099                 dev->qp_tbl[qp->id] = qp;
1100                 status = 0;
1101         }
1102         return status;
1103 }
1104
1105 static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
1106 {
1107         dev->qp_tbl[qp->id] = NULL;
1108 }
1109
1110 static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev,
1111                                   struct ib_qp_init_attr *attrs,
1112                                   struct ib_udata *udata)
1113 {
1114         if ((attrs->qp_type != IB_QPT_GSI) &&
1115             (attrs->qp_type != IB_QPT_RC) &&
1116             (attrs->qp_type != IB_QPT_UC) &&
1117             (attrs->qp_type != IB_QPT_UD)) {
1118                 pr_err("%s(%d) unsupported qp type=0x%x requested\n",
1119                        __func__, dev->id, attrs->qp_type);
1120                 return -EINVAL;
1121         }
1122         /* Skip the check for QP1 to support CM size of 128 */
1123         if ((attrs->qp_type != IB_QPT_GSI) &&
1124             (attrs->cap.max_send_wr > dev->attr.max_wqe)) {
1125                 pr_err("%s(%d) unsupported send_wr=0x%x requested\n",
1126                        __func__, dev->id, attrs->cap.max_send_wr);
1127                 pr_err("%s(%d) supported send_wr=0x%x\n",
1128                        __func__, dev->id, dev->attr.max_wqe);
1129                 return -EINVAL;
1130         }
1131         if (!attrs->srq && (attrs->cap.max_recv_wr > dev->attr.max_rqe)) {
1132                 pr_err("%s(%d) unsupported recv_wr=0x%x requested\n",
1133                        __func__, dev->id, attrs->cap.max_recv_wr);
1134                 pr_err("%s(%d) supported recv_wr=0x%x\n",
1135                        __func__, dev->id, dev->attr.max_rqe);
1136                 return -EINVAL;
1137         }
1138         if (attrs->cap.max_inline_data > dev->attr.max_inline_data) {
1139                 pr_err("%s(%d) unsupported inline data size=0x%x requested\n",
1140                        __func__, dev->id, attrs->cap.max_inline_data);
1141                 pr_err("%s(%d) supported inline data size=0x%x\n",
1142                        __func__, dev->id, dev->attr.max_inline_data);
1143                 return -EINVAL;
1144         }
1145         if (attrs->cap.max_send_sge > dev->attr.max_send_sge) {
1146                 pr_err("%s(%d) unsupported send_sge=0x%x requested\n",
1147                        __func__, dev->id, attrs->cap.max_send_sge);
1148                 pr_err("%s(%d) supported send_sge=0x%x\n",
1149                        __func__, dev->id, dev->attr.max_send_sge);
1150                 return -EINVAL;
1151         }
1152         if (attrs->cap.max_recv_sge > dev->attr.max_recv_sge) {
1153                 pr_err("%s(%d) unsupported recv_sge=0x%x requested\n",
1154                        __func__, dev->id, attrs->cap.max_recv_sge);
1155                 pr_err("%s(%d) supported recv_sge=0x%x\n",
1156                        __func__, dev->id, dev->attr.max_recv_sge);
1157                 return -EINVAL;
1158         }
1159         /* unprivileged user space cannot create special QP */
1160         if (udata && attrs->qp_type == IB_QPT_GSI) {
1161                 pr_err
1162                     ("%s(%d) Userspace can't create special QPs of type=0x%x\n",
1163                      __func__, dev->id, attrs->qp_type);
1164                 return -EINVAL;
1165         }
1166         /* allow creating only one GSI type of QP */
1167         if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) {
1168                 pr_err("%s(%d) GSI special QPs already created.\n",
1169                        __func__, dev->id);
1170                 return -EINVAL;
1171         }
1172         /* verify consumer QPs are not trying to use GSI QP's CQ */
1173         if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) {
1174                 if ((dev->gsi_sqcq == get_ocrdma_cq(attrs->send_cq)) ||
1175                         (dev->gsi_rqcq == get_ocrdma_cq(attrs->recv_cq))) {
1176                         pr_err("%s(%d) Consumer QP cannot use GSI CQs.\n",
1177                                 __func__, dev->id);
1178                         return -EINVAL;
1179                 }
1180         }
1181         return 0;
1182 }
1183
1184 static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp,
1185                                 struct ib_udata *udata, int dpp_offset,
1186                                 int dpp_credit_lmt, int srq)
1187 {
1188         int status;
1189         u64 usr_db;
1190         struct ocrdma_create_qp_uresp uresp;
1191         struct ocrdma_pd *pd = qp->pd;
1192         struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device);
1193
1194         memset(&uresp, 0, sizeof(uresp));
1195         usr_db = dev->nic_info.unmapped_db +
1196                         (pd->id * dev->nic_info.db_page_size);
1197         uresp.qp_id = qp->id;
1198         uresp.sq_dbid = qp->sq.dbid;
1199         uresp.num_sq_pages = 1;
1200         uresp.sq_page_size = PAGE_ALIGN(qp->sq.len);
1201         uresp.sq_page_addr[0] = virt_to_phys(qp->sq.va);
1202         uresp.num_wqe_allocated = qp->sq.max_cnt;
1203         if (!srq) {
1204                 uresp.rq_dbid = qp->rq.dbid;
1205                 uresp.num_rq_pages = 1;
1206                 uresp.rq_page_size = PAGE_ALIGN(qp->rq.len);
1207                 uresp.rq_page_addr[0] = virt_to_phys(qp->rq.va);
1208                 uresp.num_rqe_allocated = qp->rq.max_cnt;
1209         }
1210         uresp.db_page_addr = usr_db;
1211         uresp.db_page_size = dev->nic_info.db_page_size;
1212         uresp.db_sq_offset = OCRDMA_DB_GEN2_SQ_OFFSET;
1213         uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET;
1214         uresp.db_shift = OCRDMA_DB_RQ_SHIFT;
1215
1216         if (qp->dpp_enabled) {
1217                 uresp.dpp_credit = dpp_credit_lmt;
1218                 uresp.dpp_offset = dpp_offset;
1219         }
1220         status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1221         if (status) {
1222                 pr_err("%s(%d) user copy error.\n", __func__, dev->id);
1223                 goto err;
1224         }
1225         status = ocrdma_add_mmap(pd->uctx, uresp.sq_page_addr[0],
1226                                  uresp.sq_page_size);
1227         if (status)
1228                 goto err;
1229
1230         if (!srq) {
1231                 status = ocrdma_add_mmap(pd->uctx, uresp.rq_page_addr[0],
1232                                          uresp.rq_page_size);
1233                 if (status)
1234                         goto rq_map_err;
1235         }
1236         return status;
1237 rq_map_err:
1238         ocrdma_del_mmap(pd->uctx, uresp.sq_page_addr[0], uresp.sq_page_size);
1239 err:
1240         return status;
1241 }
1242
1243 static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp,
1244                              struct ocrdma_pd *pd)
1245 {
1246         if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) {
1247                 qp->sq_db = dev->nic_info.db +
1248                         (pd->id * dev->nic_info.db_page_size) +
1249                         OCRDMA_DB_GEN2_SQ_OFFSET;
1250                 qp->rq_db = dev->nic_info.db +
1251                         (pd->id * dev->nic_info.db_page_size) +
1252                         OCRDMA_DB_GEN2_RQ_OFFSET;
1253         } else {
1254                 qp->sq_db = dev->nic_info.db +
1255                         (pd->id * dev->nic_info.db_page_size) +
1256                         OCRDMA_DB_SQ_OFFSET;
1257                 qp->rq_db = dev->nic_info.db +
1258                         (pd->id * dev->nic_info.db_page_size) +
1259                         OCRDMA_DB_RQ_OFFSET;
1260         }
1261 }
1262
1263 static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp)
1264 {
1265         qp->wqe_wr_id_tbl =
1266             kcalloc(qp->sq.max_cnt, sizeof(*(qp->wqe_wr_id_tbl)),
1267                     GFP_KERNEL);
1268         if (qp->wqe_wr_id_tbl == NULL)
1269                 return -ENOMEM;
1270         qp->rqe_wr_id_tbl =
1271             kcalloc(qp->rq.max_cnt, sizeof(u64), GFP_KERNEL);
1272         if (qp->rqe_wr_id_tbl == NULL)
1273                 return -ENOMEM;
1274
1275         return 0;
1276 }
1277
1278 static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp,
1279                                       struct ocrdma_pd *pd,
1280                                       struct ib_qp_init_attr *attrs)
1281 {
1282         qp->pd = pd;
1283         spin_lock_init(&qp->q_lock);
1284         INIT_LIST_HEAD(&qp->sq_entry);
1285         INIT_LIST_HEAD(&qp->rq_entry);
1286
1287         qp->qp_type = attrs->qp_type;
1288         qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR;
1289         qp->max_inline_data = attrs->cap.max_inline_data;
1290         qp->sq.max_sges = attrs->cap.max_send_sge;
1291         qp->rq.max_sges = attrs->cap.max_recv_sge;
1292         qp->state = OCRDMA_QPS_RST;
1293         qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1294 }
1295
1296 static void ocrdma_store_gsi_qp_cq(struct ocrdma_dev *dev,
1297                                    struct ib_qp_init_attr *attrs)
1298 {
1299         if (attrs->qp_type == IB_QPT_GSI) {
1300                 dev->gsi_qp_created = 1;
1301                 dev->gsi_sqcq = get_ocrdma_cq(attrs->send_cq);
1302                 dev->gsi_rqcq = get_ocrdma_cq(attrs->recv_cq);
1303         }
1304 }
1305
1306 struct ib_qp *ocrdma_create_qp(struct ib_pd *ibpd,
1307                                struct ib_qp_init_attr *attrs,
1308                                struct ib_udata *udata)
1309 {
1310         int status;
1311         struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
1312         struct ocrdma_qp *qp;
1313         struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
1314         struct ocrdma_create_qp_ureq ureq;
1315         u16 dpp_credit_lmt, dpp_offset;
1316
1317         status = ocrdma_check_qp_params(ibpd, dev, attrs, udata);
1318         if (status)
1319                 goto gen_err;
1320
1321         memset(&ureq, 0, sizeof(ureq));
1322         if (udata) {
1323                 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
1324                         return ERR_PTR(-EFAULT);
1325         }
1326         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1327         if (!qp) {
1328                 status = -ENOMEM;
1329                 goto gen_err;
1330         }
1331         ocrdma_set_qp_init_params(qp, pd, attrs);
1332         if (udata == NULL)
1333                 qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 |
1334                                         OCRDMA_QP_FAST_REG);
1335
1336         mutex_lock(&dev->dev_lock);
1337         status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq,
1338                                         ureq.dpp_cq_id,
1339                                         &dpp_offset, &dpp_credit_lmt);
1340         if (status)
1341                 goto mbx_err;
1342
1343         /* user space QP's wr_id table are managed in library */
1344         if (udata == NULL) {
1345                 status = ocrdma_alloc_wr_id_tbl(qp);
1346                 if (status)
1347                         goto map_err;
1348         }
1349
1350         status = ocrdma_add_qpn_map(dev, qp);
1351         if (status)
1352                 goto map_err;
1353         ocrdma_set_qp_db(dev, qp, pd);
1354         if (udata) {
1355                 status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset,
1356                                               dpp_credit_lmt,
1357                                               (attrs->srq != NULL));
1358                 if (status)
1359                         goto cpy_err;
1360         }
1361         ocrdma_store_gsi_qp_cq(dev, attrs);
1362         qp->ibqp.qp_num = qp->id;
1363         mutex_unlock(&dev->dev_lock);
1364         return &qp->ibqp;
1365
1366 cpy_err:
1367         ocrdma_del_qpn_map(dev, qp);
1368 map_err:
1369         ocrdma_mbx_destroy_qp(dev, qp);
1370 mbx_err:
1371         mutex_unlock(&dev->dev_lock);
1372         kfree(qp->wqe_wr_id_tbl);
1373         kfree(qp->rqe_wr_id_tbl);
1374         kfree(qp);
1375         pr_err("%s(%d) error=%d\n", __func__, dev->id, status);
1376 gen_err:
1377         return ERR_PTR(status);
1378 }
1379
1380 int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1381                       int attr_mask)
1382 {
1383         int status = 0;
1384         struct ocrdma_qp *qp;
1385         struct ocrdma_dev *dev;
1386         enum ib_qp_state old_qps;
1387
1388         qp = get_ocrdma_qp(ibqp);
1389         dev = get_ocrdma_dev(ibqp->device);
1390         if (attr_mask & IB_QP_STATE)
1391                 status = ocrdma_qp_state_change(qp, attr->qp_state, &old_qps);
1392         /* if new and previous states are same hw doesn't need to
1393          * know about it.
1394          */
1395         if (status < 0)
1396                 return status;
1397         return ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask);
1398 }
1399
1400 int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1401                      int attr_mask, struct ib_udata *udata)
1402 {
1403         unsigned long flags;
1404         int status = -EINVAL;
1405         struct ocrdma_qp *qp;
1406         struct ocrdma_dev *dev;
1407         enum ib_qp_state old_qps, new_qps;
1408
1409         qp = get_ocrdma_qp(ibqp);
1410         dev = get_ocrdma_dev(ibqp->device);
1411
1412         /* syncronize with multiple context trying to change, retrive qps */
1413         mutex_lock(&dev->dev_lock);
1414         /* syncronize with wqe, rqe posting and cqe processing contexts */
1415         spin_lock_irqsave(&qp->q_lock, flags);
1416         old_qps = get_ibqp_state(qp->state);
1417         if (attr_mask & IB_QP_STATE)
1418                 new_qps = attr->qp_state;
1419         else
1420                 new_qps = old_qps;
1421         spin_unlock_irqrestore(&qp->q_lock, flags);
1422
1423         if (!ib_modify_qp_is_ok(old_qps, new_qps, ibqp->qp_type, attr_mask)) {
1424                 pr_err("%s(%d) invalid attribute mask=0x%x specified for\n"
1425                        "qpn=0x%x of type=0x%x old_qps=0x%x, new_qps=0x%x\n",
1426                        __func__, dev->id, attr_mask, qp->id, ibqp->qp_type,
1427                        old_qps, new_qps);
1428                 goto param_err;
1429         }
1430
1431         status = _ocrdma_modify_qp(ibqp, attr, attr_mask);
1432         if (status > 0)
1433                 status = 0;
1434 param_err:
1435         mutex_unlock(&dev->dev_lock);
1436         return status;
1437 }
1438
1439 static enum ib_mtu ocrdma_mtu_int_to_enum(u16 mtu)
1440 {
1441         switch (mtu) {
1442         case 256:
1443                 return IB_MTU_256;
1444         case 512:
1445                 return IB_MTU_512;
1446         case 1024:
1447                 return IB_MTU_1024;
1448         case 2048:
1449                 return IB_MTU_2048;
1450         case 4096:
1451                 return IB_MTU_4096;
1452         default:
1453                 return IB_MTU_1024;
1454         }
1455 }
1456
1457 static int ocrdma_to_ib_qp_acc_flags(int qp_cap_flags)
1458 {
1459         int ib_qp_acc_flags = 0;
1460
1461         if (qp_cap_flags & OCRDMA_QP_INB_WR)
1462                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
1463         if (qp_cap_flags & OCRDMA_QP_INB_RD)
1464                 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
1465         return ib_qp_acc_flags;
1466 }
1467
1468 int ocrdma_query_qp(struct ib_qp *ibqp,
1469                     struct ib_qp_attr *qp_attr,
1470                     int attr_mask, struct ib_qp_init_attr *qp_init_attr)
1471 {
1472         int status;
1473         u32 qp_state;
1474         struct ocrdma_qp_params params;
1475         struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
1476         struct ocrdma_dev *dev = get_ocrdma_dev(ibqp->device);
1477
1478         memset(&params, 0, sizeof(params));
1479         mutex_lock(&dev->dev_lock);
1480         status = ocrdma_mbx_query_qp(dev, qp, &params);
1481         mutex_unlock(&dev->dev_lock);
1482         if (status)
1483                 goto mbx_err;
1484         if (qp->qp_type == IB_QPT_UD)
1485                 qp_attr->qkey = params.qkey;
1486         qp_attr->path_mtu =
1487                 ocrdma_mtu_int_to_enum(params.path_mtu_pkey_indx &
1488                                 OCRDMA_QP_PARAMS_PATH_MTU_MASK) >>
1489                                 OCRDMA_QP_PARAMS_PATH_MTU_SHIFT;
1490         qp_attr->path_mig_state = IB_MIG_MIGRATED;
1491         qp_attr->rq_psn = params.hop_lmt_rq_psn & OCRDMA_QP_PARAMS_RQ_PSN_MASK;
1492         qp_attr->sq_psn = params.tclass_sq_psn & OCRDMA_QP_PARAMS_SQ_PSN_MASK;
1493         qp_attr->dest_qp_num =
1494             params.ack_to_rnr_rtc_dest_qpn & OCRDMA_QP_PARAMS_DEST_QPN_MASK;
1495
1496         qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp->cap_flags);
1497         qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1;
1498         qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1;
1499         qp_attr->cap.max_send_sge = qp->sq.max_sges;
1500         qp_attr->cap.max_recv_sge = qp->rq.max_sges;
1501         qp_attr->cap.max_inline_data = qp->max_inline_data;
1502         qp_init_attr->cap = qp_attr->cap;
1503         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
1504
1505         rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
1506                         params.rnt_rc_sl_fl &
1507                           OCRDMA_QP_PARAMS_FLOW_LABEL_MASK,
1508                         qp->sgid_idx,
1509                         (params.hop_lmt_rq_psn &
1510                          OCRDMA_QP_PARAMS_HOP_LMT_MASK) >>
1511                          OCRDMA_QP_PARAMS_HOP_LMT_SHIFT,
1512                         (params.tclass_sq_psn &
1513                          OCRDMA_QP_PARAMS_TCLASS_MASK) >>
1514                          OCRDMA_QP_PARAMS_TCLASS_SHIFT);
1515         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid[0]);
1516
1517         rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
1518         rdma_ah_set_sl(&qp_attr->ah_attr, (params.rnt_rc_sl_fl &
1519                                            OCRDMA_QP_PARAMS_SL_MASK) >>
1520                                            OCRDMA_QP_PARAMS_SL_SHIFT);
1521         qp_attr->timeout = (params.ack_to_rnr_rtc_dest_qpn &
1522                             OCRDMA_QP_PARAMS_ACK_TIMEOUT_MASK) >>
1523                                 OCRDMA_QP_PARAMS_ACK_TIMEOUT_SHIFT;
1524         qp_attr->rnr_retry = (params.ack_to_rnr_rtc_dest_qpn &
1525                               OCRDMA_QP_PARAMS_RNR_RETRY_CNT_MASK) >>
1526                                 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_SHIFT;
1527         qp_attr->retry_cnt =
1528             (params.rnt_rc_sl_fl & OCRDMA_QP_PARAMS_RETRY_CNT_MASK) >>
1529                 OCRDMA_QP_PARAMS_RETRY_CNT_SHIFT;
1530         qp_attr->min_rnr_timer = 0;
1531         qp_attr->pkey_index = 0;
1532         qp_attr->port_num = 1;
1533         rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
1534         rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
1535         qp_attr->alt_pkey_index = 0;
1536         qp_attr->alt_port_num = 0;
1537         qp_attr->alt_timeout = 0;
1538         memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
1539         qp_state = (params.max_sge_recv_flags & OCRDMA_QP_PARAMS_STATE_MASK) >>
1540                     OCRDMA_QP_PARAMS_STATE_SHIFT;
1541         qp_attr->qp_state = get_ibqp_state(qp_state);
1542         qp_attr->cur_qp_state = qp_attr->qp_state;
1543         qp_attr->sq_draining = (qp_state == OCRDMA_QPS_SQ_DRAINING) ? 1 : 0;
1544         qp_attr->max_dest_rd_atomic =
1545             params.max_ord_ird >> OCRDMA_QP_PARAMS_MAX_ORD_SHIFT;
1546         qp_attr->max_rd_atomic =
1547             params.max_ord_ird & OCRDMA_QP_PARAMS_MAX_IRD_MASK;
1548         qp_attr->en_sqd_async_notify = (params.max_sge_recv_flags &
1549                                 OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC) ? 1 : 0;
1550         /* Sync driver QP state with FW */
1551         ocrdma_qp_state_change(qp, qp_attr->qp_state, NULL);
1552 mbx_err:
1553         return status;
1554 }
1555
1556 static void ocrdma_srq_toggle_bit(struct ocrdma_srq *srq, unsigned int idx)
1557 {
1558         unsigned int i = idx / 32;
1559         u32 mask = (1U << (idx % 32));
1560
1561         srq->idx_bit_fields[i] ^= mask;
1562 }
1563
1564 static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q)
1565 {
1566         return ((q->max_wqe_idx - q->head) + q->tail) % q->max_cnt;
1567 }
1568
1569 static int is_hw_sq_empty(struct ocrdma_qp *qp)
1570 {
1571         return (qp->sq.tail == qp->sq.head);
1572 }
1573
1574 static int is_hw_rq_empty(struct ocrdma_qp *qp)
1575 {
1576         return (qp->rq.tail == qp->rq.head);
1577 }
1578
1579 static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q)
1580 {
1581         return q->va + (q->head * q->entry_size);
1582 }
1583
1584 static void *ocrdma_hwq_head_from_idx(struct ocrdma_qp_hwq_info *q,
1585                                       u32 idx)
1586 {
1587         return q->va + (idx * q->entry_size);
1588 }
1589
1590 static void ocrdma_hwq_inc_head(struct ocrdma_qp_hwq_info *q)
1591 {
1592         q->head = (q->head + 1) & q->max_wqe_idx;
1593 }
1594
1595 static void ocrdma_hwq_inc_tail(struct ocrdma_qp_hwq_info *q)
1596 {
1597         q->tail = (q->tail + 1) & q->max_wqe_idx;
1598 }
1599
1600 /* discard the cqe for a given QP */
1601 static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq)
1602 {
1603         unsigned long cq_flags;
1604         unsigned long flags;
1605         int discard_cnt = 0;
1606         u32 cur_getp, stop_getp;
1607         struct ocrdma_cqe *cqe;
1608         u32 qpn = 0, wqe_idx = 0;
1609
1610         spin_lock_irqsave(&cq->cq_lock, cq_flags);
1611
1612         /* traverse through the CQEs in the hw CQ,
1613          * find the matching CQE for a given qp,
1614          * mark the matching one discarded by clearing qpn.
1615          * ring the doorbell in the poll_cq() as
1616          * we don't complete out of order cqe.
1617          */
1618
1619         cur_getp = cq->getp;
1620         /* find upto when do we reap the cq. */
1621         stop_getp = cur_getp;
1622         do {
1623                 if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp)))
1624                         break;
1625
1626                 cqe = cq->va + cur_getp;
1627                 /* if (a) done reaping whole hw cq, or
1628                  *    (b) qp_xq becomes empty.
1629                  * then exit
1630                  */
1631                 qpn = cqe->cmn.qpn & OCRDMA_CQE_QPN_MASK;
1632                 /* if previously discarded cqe found, skip that too. */
1633                 /* check for matching qp */
1634                 if (qpn == 0 || qpn != qp->id)
1635                         goto skip_cqe;
1636
1637                 if (is_cqe_for_sq(cqe)) {
1638                         ocrdma_hwq_inc_tail(&qp->sq);
1639                 } else {
1640                         if (qp->srq) {
1641                                 wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >>
1642                                         OCRDMA_CQE_BUFTAG_SHIFT) &
1643                                         qp->srq->rq.max_wqe_idx;
1644                                 BUG_ON(wqe_idx < 1);
1645                                 spin_lock_irqsave(&qp->srq->q_lock, flags);
1646                                 ocrdma_hwq_inc_tail(&qp->srq->rq);
1647                                 ocrdma_srq_toggle_bit(qp->srq, wqe_idx - 1);
1648                                 spin_unlock_irqrestore(&qp->srq->q_lock, flags);
1649
1650                         } else {
1651                                 ocrdma_hwq_inc_tail(&qp->rq);
1652                         }
1653                 }
1654                 /* mark cqe discarded so that it is not picked up later
1655                  * in the poll_cq().
1656                  */
1657                 discard_cnt += 1;
1658                 cqe->cmn.qpn = 0;
1659 skip_cqe:
1660                 cur_getp = (cur_getp + 1) % cq->max_hw_cqe;
1661         } while (cur_getp != stop_getp);
1662         spin_unlock_irqrestore(&cq->cq_lock, cq_flags);
1663 }
1664
1665 void ocrdma_del_flush_qp(struct ocrdma_qp *qp)
1666 {
1667         int found = false;
1668         unsigned long flags;
1669         struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device);
1670         /* sync with any active CQ poll */
1671
1672         spin_lock_irqsave(&dev->flush_q_lock, flags);
1673         found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp);
1674         if (found)
1675                 list_del(&qp->sq_entry);
1676         if (!qp->srq) {
1677                 found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp);
1678                 if (found)
1679                         list_del(&qp->rq_entry);
1680         }
1681         spin_unlock_irqrestore(&dev->flush_q_lock, flags);
1682 }
1683
1684 int ocrdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
1685 {
1686         struct ocrdma_pd *pd;
1687         struct ocrdma_qp *qp;
1688         struct ocrdma_dev *dev;
1689         struct ib_qp_attr attrs;
1690         int attr_mask;
1691         unsigned long flags;
1692
1693         qp = get_ocrdma_qp(ibqp);
1694         dev = get_ocrdma_dev(ibqp->device);
1695
1696         pd = qp->pd;
1697
1698         /* change the QP state to ERROR */
1699         if (qp->state != OCRDMA_QPS_RST) {
1700                 attrs.qp_state = IB_QPS_ERR;
1701                 attr_mask = IB_QP_STATE;
1702                 _ocrdma_modify_qp(ibqp, &attrs, attr_mask);
1703         }
1704         /* ensure that CQEs for newly created QP (whose id may be same with
1705          * one which just getting destroyed are same), dont get
1706          * discarded until the old CQEs are discarded.
1707          */
1708         mutex_lock(&dev->dev_lock);
1709         (void) ocrdma_mbx_destroy_qp(dev, qp);
1710
1711         /*
1712          * acquire CQ lock while destroy is in progress, in order to
1713          * protect against proessing in-flight CQEs for this QP.
1714          */
1715         spin_lock_irqsave(&qp->sq_cq->cq_lock, flags);
1716         if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) {
1717                 spin_lock(&qp->rq_cq->cq_lock);
1718                 ocrdma_del_qpn_map(dev, qp);
1719                 spin_unlock(&qp->rq_cq->cq_lock);
1720         } else {
1721                 ocrdma_del_qpn_map(dev, qp);
1722         }
1723         spin_unlock_irqrestore(&qp->sq_cq->cq_lock, flags);
1724
1725         if (!pd->uctx) {
1726                 ocrdma_discard_cqes(qp, qp->sq_cq);
1727                 ocrdma_discard_cqes(qp, qp->rq_cq);
1728         }
1729         mutex_unlock(&dev->dev_lock);
1730
1731         if (pd->uctx) {
1732                 ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa,
1733                                 PAGE_ALIGN(qp->sq.len));
1734                 if (!qp->srq)
1735                         ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa,
1736                                         PAGE_ALIGN(qp->rq.len));
1737         }
1738
1739         ocrdma_del_flush_qp(qp);
1740
1741         kfree(qp->wqe_wr_id_tbl);
1742         kfree(qp->rqe_wr_id_tbl);
1743         kfree(qp);
1744         return 0;
1745 }
1746
1747 static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq,
1748                                 struct ib_udata *udata)
1749 {
1750         int status;
1751         struct ocrdma_create_srq_uresp uresp;
1752
1753         memset(&uresp, 0, sizeof(uresp));
1754         uresp.rq_dbid = srq->rq.dbid;
1755         uresp.num_rq_pages = 1;
1756         uresp.rq_page_addr[0] = virt_to_phys(srq->rq.va);
1757         uresp.rq_page_size = srq->rq.len;
1758         uresp.db_page_addr = dev->nic_info.unmapped_db +
1759             (srq->pd->id * dev->nic_info.db_page_size);
1760         uresp.db_page_size = dev->nic_info.db_page_size;
1761         uresp.num_rqe_allocated = srq->rq.max_cnt;
1762         if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) {
1763                 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET;
1764                 uresp.db_shift = 24;
1765         } else {
1766                 uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET;
1767                 uresp.db_shift = 16;
1768         }
1769
1770         status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1771         if (status)
1772                 return status;
1773         status = ocrdma_add_mmap(srq->pd->uctx, uresp.rq_page_addr[0],
1774                                  uresp.rq_page_size);
1775         if (status)
1776                 return status;
1777         return status;
1778 }
1779
1780 int ocrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1781                       struct ib_udata *udata)
1782 {
1783         int status;
1784         struct ocrdma_pd *pd = get_ocrdma_pd(ibsrq->pd);
1785         struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device);
1786         struct ocrdma_srq *srq = get_ocrdma_srq(ibsrq);
1787
1788         if (init_attr->attr.max_sge > dev->attr.max_recv_sge)
1789                 return -EINVAL;
1790         if (init_attr->attr.max_wr > dev->attr.max_rqe)
1791                 return -EINVAL;
1792
1793         spin_lock_init(&srq->q_lock);
1794         srq->pd = pd;
1795         srq->db = dev->nic_info.db + (pd->id * dev->nic_info.db_page_size);
1796         status = ocrdma_mbx_create_srq(dev, srq, init_attr, pd);
1797         if (status)
1798                 return status;
1799
1800         if (!udata) {
1801                 srq->rqe_wr_id_tbl = kcalloc(srq->rq.max_cnt, sizeof(u64),
1802                                              GFP_KERNEL);
1803                 if (!srq->rqe_wr_id_tbl) {
1804                         status = -ENOMEM;
1805                         goto arm_err;
1806                 }
1807
1808                 srq->bit_fields_len = (srq->rq.max_cnt / 32) +
1809                     (srq->rq.max_cnt % 32 ? 1 : 0);
1810                 srq->idx_bit_fields =
1811                     kmalloc_array(srq->bit_fields_len, sizeof(u32),
1812                                   GFP_KERNEL);
1813                 if (!srq->idx_bit_fields) {
1814                         status = -ENOMEM;
1815                         goto arm_err;
1816                 }
1817                 memset(srq->idx_bit_fields, 0xff,
1818                        srq->bit_fields_len * sizeof(u32));
1819         }
1820
1821         if (init_attr->attr.srq_limit) {
1822                 status = ocrdma_mbx_modify_srq(srq, &init_attr->attr);
1823                 if (status)
1824                         goto arm_err;
1825         }
1826
1827         if (udata) {
1828                 status = ocrdma_copy_srq_uresp(dev, srq, udata);
1829                 if (status)
1830                         goto arm_err;
1831         }
1832
1833         return 0;
1834
1835 arm_err:
1836         ocrdma_mbx_destroy_srq(dev, srq);
1837         kfree(srq->rqe_wr_id_tbl);
1838         kfree(srq->idx_bit_fields);
1839         return status;
1840 }
1841
1842 int ocrdma_modify_srq(struct ib_srq *ibsrq,
1843                       struct ib_srq_attr *srq_attr,
1844                       enum ib_srq_attr_mask srq_attr_mask,
1845                       struct ib_udata *udata)
1846 {
1847         int status;
1848         struct ocrdma_srq *srq;
1849
1850         srq = get_ocrdma_srq(ibsrq);
1851         if (srq_attr_mask & IB_SRQ_MAX_WR)
1852                 status = -EINVAL;
1853         else
1854                 status = ocrdma_mbx_modify_srq(srq, srq_attr);
1855         return status;
1856 }
1857
1858 int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
1859 {
1860         int status;
1861         struct ocrdma_srq *srq;
1862
1863         srq = get_ocrdma_srq(ibsrq);
1864         status = ocrdma_mbx_query_srq(srq, srq_attr);
1865         return status;
1866 }
1867
1868 void ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1869 {
1870         struct ocrdma_srq *srq;
1871         struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device);
1872
1873         srq = get_ocrdma_srq(ibsrq);
1874
1875         ocrdma_mbx_destroy_srq(dev, srq);
1876
1877         if (srq->pd->uctx)
1878                 ocrdma_del_mmap(srq->pd->uctx, (u64) srq->rq.pa,
1879                                 PAGE_ALIGN(srq->rq.len));
1880
1881         kfree(srq->idx_bit_fields);
1882         kfree(srq->rqe_wr_id_tbl);
1883 }
1884
1885 /* unprivileged verbs and their support functions. */
1886 static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp,
1887                                 struct ocrdma_hdr_wqe *hdr,
1888                                 const struct ib_send_wr *wr)
1889 {
1890         struct ocrdma_ewqe_ud_hdr *ud_hdr =
1891                 (struct ocrdma_ewqe_ud_hdr *)(hdr + 1);
1892         struct ocrdma_ah *ah = get_ocrdma_ah(ud_wr(wr)->ah);
1893
1894         ud_hdr->rsvd_dest_qpn = ud_wr(wr)->remote_qpn;
1895         if (qp->qp_type == IB_QPT_GSI)
1896                 ud_hdr->qkey = qp->qkey;
1897         else
1898                 ud_hdr->qkey = ud_wr(wr)->remote_qkey;
1899         ud_hdr->rsvd_ahid = ah->id;
1900         ud_hdr->hdr_type = ah->hdr_type;
1901         if (ah->av->valid & OCRDMA_AV_VLAN_VALID)
1902                 hdr->cw |= (OCRDMA_FLAG_AH_VLAN_PR << OCRDMA_WQE_FLAGS_SHIFT);
1903 }
1904
1905 static void ocrdma_build_sges(struct ocrdma_hdr_wqe *hdr,
1906                               struct ocrdma_sge *sge, int num_sge,
1907                               struct ib_sge *sg_list)
1908 {
1909         int i;
1910
1911         for (i = 0; i < num_sge; i++) {
1912                 sge[i].lrkey = sg_list[i].lkey;
1913                 sge[i].addr_lo = sg_list[i].addr;
1914                 sge[i].addr_hi = upper_32_bits(sg_list[i].addr);
1915                 sge[i].len = sg_list[i].length;
1916                 hdr->total_len += sg_list[i].length;
1917         }
1918         if (num_sge == 0)
1919                 memset(sge, 0, sizeof(*sge));
1920 }
1921
1922 static inline uint32_t ocrdma_sglist_len(struct ib_sge *sg_list, int num_sge)
1923 {
1924         uint32_t total_len = 0, i;
1925
1926         for (i = 0; i < num_sge; i++)
1927                 total_len += sg_list[i].length;
1928         return total_len;
1929 }
1930
1931
1932 static int ocrdma_build_inline_sges(struct ocrdma_qp *qp,
1933                                     struct ocrdma_hdr_wqe *hdr,
1934                                     struct ocrdma_sge *sge,
1935                                     const struct ib_send_wr *wr, u32 wqe_size)
1936 {
1937         int i;
1938         char *dpp_addr;
1939
1940         if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) {
1941                 hdr->total_len = ocrdma_sglist_len(wr->sg_list, wr->num_sge);
1942                 if (unlikely(hdr->total_len > qp->max_inline_data)) {
1943                         pr_err("%s() supported_len=0x%x,\n"
1944                                " unsupported len req=0x%x\n", __func__,
1945                                 qp->max_inline_data, hdr->total_len);
1946                         return -EINVAL;
1947                 }
1948                 dpp_addr = (char *)sge;
1949                 for (i = 0; i < wr->num_sge; i++) {
1950                         memcpy(dpp_addr,
1951                                (void *)(unsigned long)wr->sg_list[i].addr,
1952                                wr->sg_list[i].length);
1953                         dpp_addr += wr->sg_list[i].length;
1954                 }
1955
1956                 wqe_size += roundup(hdr->total_len, OCRDMA_WQE_ALIGN_BYTES);
1957                 if (0 == hdr->total_len)
1958                         wqe_size += sizeof(struct ocrdma_sge);
1959                 hdr->cw |= (OCRDMA_TYPE_INLINE << OCRDMA_WQE_TYPE_SHIFT);
1960         } else {
1961                 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);
1962                 if (wr->num_sge)
1963                         wqe_size += (wr->num_sge * sizeof(struct ocrdma_sge));
1964                 else
1965                         wqe_size += sizeof(struct ocrdma_sge);
1966                 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
1967         }
1968         hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
1969         return 0;
1970 }
1971
1972 static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1973                              const struct ib_send_wr *wr)
1974 {
1975         int status;
1976         struct ocrdma_sge *sge;
1977         u32 wqe_size = sizeof(*hdr);
1978
1979         if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
1980                 ocrdma_build_ud_hdr(qp, hdr, wr);
1981                 sge = (struct ocrdma_sge *)(hdr + 2);
1982                 wqe_size += sizeof(struct ocrdma_ewqe_ud_hdr);
1983         } else {
1984                 sge = (struct ocrdma_sge *)(hdr + 1);
1985         }
1986
1987         status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
1988         return status;
1989 }
1990
1991 static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1992                               const struct ib_send_wr *wr)
1993 {
1994         int status;
1995         struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);
1996         struct ocrdma_sge *sge = ext_rw + 1;
1997         u32 wqe_size = sizeof(*hdr) + sizeof(*ext_rw);
1998
1999         status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
2000         if (status)
2001                 return status;
2002         ext_rw->addr_lo = rdma_wr(wr)->remote_addr;
2003         ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr);
2004         ext_rw->lrkey = rdma_wr(wr)->rkey;
2005         ext_rw->len = hdr->total_len;
2006         return 0;
2007 }
2008
2009 static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
2010                               const struct ib_send_wr *wr)
2011 {
2012         struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);
2013         struct ocrdma_sge *sge = ext_rw + 1;
2014         u32 wqe_size = ((wr->num_sge + 1) * sizeof(struct ocrdma_sge)) +
2015             sizeof(struct ocrdma_hdr_wqe);
2016
2017         ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);
2018         hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
2019         hdr->cw |= (OCRDMA_READ << OCRDMA_WQE_OPCODE_SHIFT);
2020         hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
2021
2022         ext_rw->addr_lo = rdma_wr(wr)->remote_addr;
2023         ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr);
2024         ext_rw->lrkey = rdma_wr(wr)->rkey;
2025         ext_rw->len = hdr->total_len;
2026 }
2027
2028 static int get_encoded_page_size(int pg_sz)
2029 {
2030         /* Max size is 256M 4096 << 16 */
2031         int i = 0;
2032         for (; i < 17; i++)
2033                 if (pg_sz == (4096 << i))
2034                         break;
2035         return i;
2036 }
2037
2038 static int ocrdma_build_reg(struct ocrdma_qp *qp,
2039                             struct ocrdma_hdr_wqe *hdr,
2040                             const struct ib_reg_wr *wr)
2041 {
2042         u64 fbo;
2043         struct ocrdma_ewqe_fr *fast_reg = (struct ocrdma_ewqe_fr *)(hdr + 1);
2044         struct ocrdma_mr *mr = get_ocrdma_mr(wr->mr);
2045         struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;
2046         struct ocrdma_pbe *pbe;
2047         u32 wqe_size = sizeof(*fast_reg) + sizeof(*hdr);
2048         int num_pbes = 0, i;
2049
2050         wqe_size = roundup(wqe_size, OCRDMA_WQE_ALIGN_BYTES);
2051
2052         hdr->cw |= (OCRDMA_FR_MR << OCRDMA_WQE_OPCODE_SHIFT);
2053         hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
2054
2055         if (wr->access & IB_ACCESS_LOCAL_WRITE)
2056                 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_LOCAL_WR;
2057         if (wr->access & IB_ACCESS_REMOTE_WRITE)
2058                 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_WR;
2059         if (wr->access & IB_ACCESS_REMOTE_READ)
2060                 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_RD;
2061         hdr->lkey = wr->key;
2062         hdr->total_len = mr->ibmr.length;
2063
2064         fbo = mr->ibmr.iova - mr->pages[0];
2065
2066         fast_reg->va_hi = upper_32_bits(mr->ibmr.iova);
2067         fast_reg->va_lo = (u32) (mr->ibmr.iova & 0xffffffff);
2068         fast_reg->fbo_hi = upper_32_bits(fbo);
2069         fast_reg->fbo_lo = (u32) fbo & 0xffffffff;
2070         fast_reg->num_sges = mr->npages;
2071         fast_reg->size_sge = get_encoded_page_size(mr->ibmr.page_size);
2072
2073         pbe = pbl_tbl->va;
2074         for (i = 0; i < mr->npages; i++) {
2075                 u64 buf_addr = mr->pages[i];
2076
2077                 pbe->pa_lo = cpu_to_le32((u32) (buf_addr & PAGE_MASK));
2078                 pbe->pa_hi = cpu_to_le32((u32) upper_32_bits(buf_addr));
2079                 num_pbes += 1;
2080                 pbe++;
2081
2082                 /* if the pbl is full storing the pbes,
2083                  * move to next pbl.
2084                 */
2085                 if (num_pbes == (mr->hwmr.pbl_size/sizeof(u64))) {
2086                         pbl_tbl++;
2087                         pbe = (struct ocrdma_pbe *)pbl_tbl->va;
2088                 }
2089         }
2090
2091         return 0;
2092 }
2093
2094 static void ocrdma_ring_sq_db(struct ocrdma_qp *qp)
2095 {
2096         u32 val = qp->sq.dbid | (1 << OCRDMA_DB_SQ_SHIFT);
2097
2098         iowrite32(val, qp->sq_db);
2099 }
2100
2101 int ocrdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
2102                      const struct ib_send_wr **bad_wr)
2103 {
2104         int status = 0;
2105         struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
2106         struct ocrdma_hdr_wqe *hdr;
2107         unsigned long flags;
2108
2109         spin_lock_irqsave(&qp->q_lock, flags);
2110         if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) {
2111                 spin_unlock_irqrestore(&qp->q_lock, flags);
2112                 *bad_wr = wr;
2113                 return -EINVAL;
2114         }
2115
2116         while (wr) {
2117                 if (qp->qp_type == IB_QPT_UD &&
2118                     (wr->opcode != IB_WR_SEND &&
2119                      wr->opcode != IB_WR_SEND_WITH_IMM)) {
2120                         *bad_wr = wr;
2121                         status = -EINVAL;
2122                         break;
2123                 }
2124                 if (ocrdma_hwq_free_cnt(&qp->sq) == 0 ||
2125                     wr->num_sge > qp->sq.max_sges) {
2126                         *bad_wr = wr;
2127                         status = -ENOMEM;
2128                         break;
2129                 }
2130                 hdr = ocrdma_hwq_head(&qp->sq);
2131                 hdr->cw = 0;
2132                 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)
2133                         hdr->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);
2134                 if (wr->send_flags & IB_SEND_FENCE)
2135                         hdr->cw |=
2136                             (OCRDMA_FLAG_FENCE_L << OCRDMA_WQE_FLAGS_SHIFT);
2137                 if (wr->send_flags & IB_SEND_SOLICITED)
2138                         hdr->cw |=
2139                             (OCRDMA_FLAG_SOLICIT << OCRDMA_WQE_FLAGS_SHIFT);
2140                 hdr->total_len = 0;
2141                 switch (wr->opcode) {
2142                 case IB_WR_SEND_WITH_IMM:
2143                         hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);
2144                         hdr->immdt = ntohl(wr->ex.imm_data);
2145                         /* fall through */
2146                 case IB_WR_SEND:
2147                         hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT);
2148                         ocrdma_build_send(qp, hdr, wr);
2149                         break;
2150                 case IB_WR_SEND_WITH_INV:
2151                         hdr->cw |= (OCRDMA_FLAG_INV << OCRDMA_WQE_FLAGS_SHIFT);
2152                         hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT);
2153                         hdr->lkey = wr->ex.invalidate_rkey;
2154                         status = ocrdma_build_send(qp, hdr, wr);
2155                         break;
2156                 case IB_WR_RDMA_WRITE_WITH_IMM:
2157                         hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);
2158                         hdr->immdt = ntohl(wr->ex.imm_data);
2159                         /* fall through */
2160                 case IB_WR_RDMA_WRITE:
2161                         hdr->cw |= (OCRDMA_WRITE << OCRDMA_WQE_OPCODE_SHIFT);
2162                         status = ocrdma_build_write(qp, hdr, wr);
2163                         break;
2164                 case IB_WR_RDMA_READ:
2165                         ocrdma_build_read(qp, hdr, wr);
2166                         break;
2167                 case IB_WR_LOCAL_INV:
2168                         hdr->cw |=
2169                             (OCRDMA_LKEY_INV << OCRDMA_WQE_OPCODE_SHIFT);
2170                         hdr->cw |= ((sizeof(struct ocrdma_hdr_wqe) +
2171                                         sizeof(struct ocrdma_sge)) /
2172                                 OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT;
2173                         hdr->lkey = wr->ex.invalidate_rkey;
2174                         break;
2175                 case IB_WR_REG_MR:
2176                         status = ocrdma_build_reg(qp, hdr, reg_wr(wr));
2177                         break;
2178                 default:
2179                         status = -EINVAL;
2180                         break;
2181                 }
2182                 if (status) {
2183                         *bad_wr = wr;
2184                         break;
2185                 }
2186                 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)
2187                         qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1;
2188                 else
2189                         qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0;
2190                 qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id;
2191                 ocrdma_cpu_to_le32(hdr, ((hdr->cw >> OCRDMA_WQE_SIZE_SHIFT) &
2192                                    OCRDMA_WQE_SIZE_MASK) * OCRDMA_WQE_STRIDE);
2193                 /* make sure wqe is written before adapter can access it */
2194                 wmb();
2195                 /* inform hw to start processing it */
2196                 ocrdma_ring_sq_db(qp);
2197
2198                 /* update pointer, counter for next wr */
2199                 ocrdma_hwq_inc_head(&qp->sq);
2200                 wr = wr->next;
2201         }
2202         spin_unlock_irqrestore(&qp->q_lock, flags);
2203         return status;
2204 }
2205
2206 static void ocrdma_ring_rq_db(struct ocrdma_qp *qp)
2207 {
2208         u32 val = qp->rq.dbid | (1 << OCRDMA_DB_RQ_SHIFT);
2209
2210         iowrite32(val, qp->rq_db);
2211 }
2212
2213 static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe,
2214                              const struct ib_recv_wr *wr, u16 tag)
2215 {
2216         u32 wqe_size = 0;
2217         struct ocrdma_sge *sge;
2218         if (wr->num_sge)
2219                 wqe_size = (wr->num_sge * sizeof(*sge)) + sizeof(*rqe);
2220         else
2221                 wqe_size = sizeof(*sge) + sizeof(*rqe);
2222
2223         rqe->cw = ((wqe_size / OCRDMA_WQE_STRIDE) <<
2224                                 OCRDMA_WQE_SIZE_SHIFT);
2225         rqe->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);
2226         rqe->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
2227         rqe->total_len = 0;
2228         rqe->rsvd_tag = tag;
2229         sge = (struct ocrdma_sge *)(rqe + 1);
2230         ocrdma_build_sges(rqe, sge, wr->num_sge, wr->sg_list);
2231         ocrdma_cpu_to_le32(rqe, wqe_size);
2232 }
2233
2234 int ocrdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
2235                      const struct ib_recv_wr **bad_wr)
2236 {
2237         int status = 0;
2238         unsigned long flags;
2239         struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
2240         struct ocrdma_hdr_wqe *rqe;
2241
2242         spin_lock_irqsave(&qp->q_lock, flags);
2243         if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) {
2244                 spin_unlock_irqrestore(&qp->q_lock, flags);
2245                 *bad_wr = wr;
2246                 return -EINVAL;
2247         }
2248         while (wr) {
2249                 if (ocrdma_hwq_free_cnt(&qp->rq) == 0 ||
2250                     wr->num_sge > qp->rq.max_sges) {
2251                         *bad_wr = wr;
2252                         status = -ENOMEM;
2253                         break;
2254                 }
2255                 rqe = ocrdma_hwq_head(&qp->rq);
2256                 ocrdma_build_rqe(rqe, wr, 0);
2257
2258                 qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id;
2259                 /* make sure rqe is written before adapter can access it */
2260                 wmb();
2261
2262                 /* inform hw to start processing it */
2263                 ocrdma_ring_rq_db(qp);
2264
2265                 /* update pointer, counter for next wr */
2266                 ocrdma_hwq_inc_head(&qp->rq);
2267                 wr = wr->next;
2268         }
2269         spin_unlock_irqrestore(&qp->q_lock, flags);
2270         return status;
2271 }
2272
2273 /* cqe for srq's rqe can potentially arrive out of order.
2274  * index gives the entry in the shadow table where to store
2275  * the wr_id. tag/index is returned in cqe to reference back
2276  * for a given rqe.
2277  */
2278 static int ocrdma_srq_get_idx(struct ocrdma_srq *srq)
2279 {
2280         int row = 0;
2281         int indx = 0;
2282
2283         for (row = 0; row < srq->bit_fields_len; row++) {
2284                 if (srq->idx_bit_fields[row]) {
2285                         indx = ffs(srq->idx_bit_fields[row]);
2286                         indx = (row * 32) + (indx - 1);
2287                         BUG_ON(indx >= srq->rq.max_cnt);
2288                         ocrdma_srq_toggle_bit(srq, indx);
2289                         break;
2290                 }
2291         }
2292
2293         BUG_ON(row == srq->bit_fields_len);
2294         return indx + 1; /* Use from index 1 */
2295 }
2296
2297 static void ocrdma_ring_srq_db(struct ocrdma_srq *srq)
2298 {
2299         u32 val = srq->rq.dbid | (1 << 16);
2300
2301         iowrite32(val, srq->db + OCRDMA_DB_GEN2_SRQ_OFFSET);
2302 }
2303
2304 int ocrdma_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
2305                          const struct ib_recv_wr **bad_wr)
2306 {
2307         int status = 0;
2308         unsigned long flags;
2309         struct ocrdma_srq *srq;
2310         struct ocrdma_hdr_wqe *rqe;
2311         u16 tag;
2312
2313         srq = get_ocrdma_srq(ibsrq);
2314
2315         spin_lock_irqsave(&srq->q_lock, flags);
2316         while (wr) {
2317                 if (ocrdma_hwq_free_cnt(&srq->rq) == 0 ||
2318                     wr->num_sge > srq->rq.max_sges) {
2319                         status = -ENOMEM;
2320                         *bad_wr = wr;
2321                         break;
2322                 }
2323                 tag = ocrdma_srq_get_idx(srq);
2324                 rqe = ocrdma_hwq_head(&srq->rq);
2325                 ocrdma_build_rqe(rqe, wr, tag);
2326
2327                 srq->rqe_wr_id_tbl[tag] = wr->wr_id;
2328                 /* make sure rqe is written before adapter can perform DMA */
2329                 wmb();
2330                 /* inform hw to start processing it */
2331                 ocrdma_ring_srq_db(srq);
2332                 /* update pointer, counter for next wr */
2333                 ocrdma_hwq_inc_head(&srq->rq);
2334                 wr = wr->next;
2335         }
2336         spin_unlock_irqrestore(&srq->q_lock, flags);
2337         return status;
2338 }
2339
2340 static enum ib_wc_status ocrdma_to_ibwc_err(u16 status)
2341 {
2342         enum ib_wc_status ibwc_status;
2343
2344         switch (status) {
2345         case OCRDMA_CQE_GENERAL_ERR:
2346                 ibwc_status = IB_WC_GENERAL_ERR;
2347                 break;
2348         case OCRDMA_CQE_LOC_LEN_ERR:
2349                 ibwc_status = IB_WC_LOC_LEN_ERR;
2350                 break;
2351         case OCRDMA_CQE_LOC_QP_OP_ERR:
2352                 ibwc_status = IB_WC_LOC_QP_OP_ERR;
2353                 break;
2354         case OCRDMA_CQE_LOC_EEC_OP_ERR:
2355                 ibwc_status = IB_WC_LOC_EEC_OP_ERR;
2356                 break;
2357         case OCRDMA_CQE_LOC_PROT_ERR:
2358                 ibwc_status = IB_WC_LOC_PROT_ERR;
2359                 break;
2360         case OCRDMA_CQE_WR_FLUSH_ERR:
2361                 ibwc_status = IB_WC_WR_FLUSH_ERR;
2362                 break;
2363         case OCRDMA_CQE_MW_BIND_ERR:
2364                 ibwc_status = IB_WC_MW_BIND_ERR;
2365                 break;
2366         case OCRDMA_CQE_BAD_RESP_ERR:
2367                 ibwc_status = IB_WC_BAD_RESP_ERR;
2368                 break;
2369         case OCRDMA_CQE_LOC_ACCESS_ERR:
2370                 ibwc_status = IB_WC_LOC_ACCESS_ERR;
2371                 break;
2372         case OCRDMA_CQE_REM_INV_REQ_ERR:
2373                 ibwc_status = IB_WC_REM_INV_REQ_ERR;
2374                 break;
2375         case OCRDMA_CQE_REM_ACCESS_ERR:
2376                 ibwc_status = IB_WC_REM_ACCESS_ERR;
2377                 break;
2378         case OCRDMA_CQE_REM_OP_ERR:
2379                 ibwc_status = IB_WC_REM_OP_ERR;
2380                 break;
2381         case OCRDMA_CQE_RETRY_EXC_ERR:
2382                 ibwc_status = IB_WC_RETRY_EXC_ERR;
2383                 break;
2384         case OCRDMA_CQE_RNR_RETRY_EXC_ERR:
2385                 ibwc_status = IB_WC_RNR_RETRY_EXC_ERR;
2386                 break;
2387         case OCRDMA_CQE_LOC_RDD_VIOL_ERR:
2388                 ibwc_status = IB_WC_LOC_RDD_VIOL_ERR;
2389                 break;
2390         case OCRDMA_CQE_REM_INV_RD_REQ_ERR:
2391                 ibwc_status = IB_WC_REM_INV_RD_REQ_ERR;
2392                 break;
2393         case OCRDMA_CQE_REM_ABORT_ERR:
2394                 ibwc_status = IB_WC_REM_ABORT_ERR;
2395                 break;
2396         case OCRDMA_CQE_INV_EECN_ERR:
2397                 ibwc_status = IB_WC_INV_EECN_ERR;
2398                 break;
2399         case OCRDMA_CQE_INV_EEC_STATE_ERR:
2400                 ibwc_status = IB_WC_INV_EEC_STATE_ERR;
2401                 break;
2402         case OCRDMA_CQE_FATAL_ERR:
2403                 ibwc_status = IB_WC_FATAL_ERR;
2404                 break;
2405         case OCRDMA_CQE_RESP_TIMEOUT_ERR:
2406                 ibwc_status = IB_WC_RESP_TIMEOUT_ERR;
2407                 break;
2408         default:
2409                 ibwc_status = IB_WC_GENERAL_ERR;
2410                 break;
2411         }
2412         return ibwc_status;
2413 }
2414
2415 static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc,
2416                       u32 wqe_idx)
2417 {
2418         struct ocrdma_hdr_wqe *hdr;
2419         struct ocrdma_sge *rw;
2420         int opcode;
2421
2422         hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx);
2423
2424         ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid;
2425         /* Undo the hdr->cw swap */
2426         opcode = le32_to_cpu(hdr->cw) & OCRDMA_WQE_OPCODE_MASK;
2427         switch (opcode) {
2428         case OCRDMA_WRITE:
2429                 ibwc->opcode = IB_WC_RDMA_WRITE;
2430                 break;
2431         case OCRDMA_READ:
2432                 rw = (struct ocrdma_sge *)(hdr + 1);
2433                 ibwc->opcode = IB_WC_RDMA_READ;
2434                 ibwc->byte_len = rw->len;
2435                 break;
2436         case OCRDMA_SEND:
2437                 ibwc->opcode = IB_WC_SEND;
2438                 break;
2439         case OCRDMA_FR_MR:
2440                 ibwc->opcode = IB_WC_REG_MR;
2441                 break;
2442         case OCRDMA_LKEY_INV:
2443                 ibwc->opcode = IB_WC_LOCAL_INV;
2444                 break;
2445         default:
2446                 ibwc->status = IB_WC_GENERAL_ERR;
2447                 pr_err("%s() invalid opcode received = 0x%x\n",
2448                        __func__, hdr->cw & OCRDMA_WQE_OPCODE_MASK);
2449                 break;
2450         }
2451 }
2452
2453 static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp,
2454                                                 struct ocrdma_cqe *cqe)
2455 {
2456         if (is_cqe_for_sq(cqe)) {
2457                 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2458                                 cqe->flags_status_srcqpn) &
2459                                         ~OCRDMA_CQE_STATUS_MASK);
2460                 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2461                                 cqe->flags_status_srcqpn) |
2462                                 (OCRDMA_CQE_WR_FLUSH_ERR <<
2463                                         OCRDMA_CQE_STATUS_SHIFT));
2464         } else {
2465                 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
2466                         cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2467                                         cqe->flags_status_srcqpn) &
2468                                                 ~OCRDMA_CQE_UD_STATUS_MASK);
2469                         cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2470                                         cqe->flags_status_srcqpn) |
2471                                         (OCRDMA_CQE_WR_FLUSH_ERR <<
2472                                                 OCRDMA_CQE_UD_STATUS_SHIFT));
2473                 } else {
2474                         cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2475                                         cqe->flags_status_srcqpn) &
2476                                                 ~OCRDMA_CQE_STATUS_MASK);
2477                         cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2478                                         cqe->flags_status_srcqpn) |
2479                                         (OCRDMA_CQE_WR_FLUSH_ERR <<
2480                                                 OCRDMA_CQE_STATUS_SHIFT));
2481                 }
2482         }
2483 }
2484
2485 static bool ocrdma_update_err_cqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2486                                   struct ocrdma_qp *qp, int status)
2487 {
2488         bool expand = false;
2489
2490         ibwc->byte_len = 0;
2491         ibwc->qp = &qp->ibqp;
2492         ibwc->status = ocrdma_to_ibwc_err(status);
2493
2494         ocrdma_flush_qp(qp);
2495         ocrdma_qp_state_change(qp, IB_QPS_ERR, NULL);
2496
2497         /* if wqe/rqe pending for which cqe needs to be returned,
2498          * trigger inflating it.
2499          */
2500         if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) {
2501                 expand = true;
2502                 ocrdma_set_cqe_status_flushed(qp, cqe);
2503         }
2504         return expand;
2505 }
2506
2507 static int ocrdma_update_err_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2508                                   struct ocrdma_qp *qp, int status)
2509 {
2510         ibwc->opcode = IB_WC_RECV;
2511         ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2512         ocrdma_hwq_inc_tail(&qp->rq);
2513
2514         return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
2515 }
2516
2517 static int ocrdma_update_err_scqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2518                                   struct ocrdma_qp *qp, int status)
2519 {
2520         ocrdma_update_wc(qp, ibwc, qp->sq.tail);
2521         ocrdma_hwq_inc_tail(&qp->sq);
2522
2523         return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
2524 }
2525
2526
2527 static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp,
2528                                  struct ocrdma_cqe *cqe, struct ib_wc *ibwc,
2529                                  bool *polled, bool *stop)
2530 {
2531         bool expand;
2532         struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device);
2533         int status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2534                 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2535         if (status < OCRDMA_MAX_CQE_ERR)
2536                 atomic_inc(&dev->cqe_err_stats[status]);
2537
2538         /* when hw sq is empty, but rq is not empty, so we continue
2539          * to keep the cqe in order to get the cq event again.
2540          */
2541         if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) {
2542                 /* when cq for rq and sq is same, it is safe to return
2543                  * flush cqe for RQEs.
2544                  */
2545                 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
2546                         *polled = true;
2547                         status = OCRDMA_CQE_WR_FLUSH_ERR;
2548                         expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
2549                 } else {
2550                         /* stop processing further cqe as this cqe is used for
2551                          * triggering cq event on buddy cq of RQ.
2552                          * When QP is destroyed, this cqe will be removed
2553                          * from the cq's hardware q.
2554                          */
2555                         *polled = false;
2556                         *stop = true;
2557                         expand = false;
2558                 }
2559         } else if (is_hw_sq_empty(qp)) {
2560                 /* Do nothing */
2561                 expand = false;
2562                 *polled = false;
2563                 *stop = false;
2564         } else {
2565                 *polled = true;
2566                 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
2567         }
2568         return expand;
2569 }
2570
2571 static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp,
2572                                      struct ocrdma_cqe *cqe,
2573                                      struct ib_wc *ibwc, bool *polled)
2574 {
2575         bool expand = false;
2576         int tail = qp->sq.tail;
2577         u32 wqe_idx;
2578
2579         if (!qp->wqe_wr_id_tbl[tail].signaled) {
2580                 *polled = false;    /* WC cannot be consumed yet */
2581         } else {
2582                 ibwc->status = IB_WC_SUCCESS;
2583                 ibwc->wc_flags = 0;
2584                 ibwc->qp = &qp->ibqp;
2585                 ocrdma_update_wc(qp, ibwc, tail);
2586                 *polled = true;
2587         }
2588         wqe_idx = (le32_to_cpu(cqe->wq.wqeidx) &
2589                         OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx;
2590         if (tail != wqe_idx)
2591                 expand = true; /* Coalesced CQE can't be consumed yet */
2592
2593         ocrdma_hwq_inc_tail(&qp->sq);
2594         return expand;
2595 }
2596
2597 static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2598                              struct ib_wc *ibwc, bool *polled, bool *stop)
2599 {
2600         int status;
2601         bool expand;
2602
2603         status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2604                 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2605
2606         if (status == OCRDMA_CQE_SUCCESS)
2607                 expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled);
2608         else
2609                 expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop);
2610         return expand;
2611 }
2612
2613 static int ocrdma_update_ud_rcqe(struct ocrdma_dev *dev, struct ib_wc *ibwc,
2614                                  struct ocrdma_cqe *cqe)
2615 {
2616         int status;
2617         u16 hdr_type = 0;
2618
2619         status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2620                 OCRDMA_CQE_UD_STATUS_MASK) >> OCRDMA_CQE_UD_STATUS_SHIFT;
2621         ibwc->src_qp = le32_to_cpu(cqe->flags_status_srcqpn) &
2622                                                 OCRDMA_CQE_SRCQP_MASK;
2623         ibwc->pkey_index = 0;
2624         ibwc->wc_flags = IB_WC_GRH;
2625         ibwc->byte_len = (le32_to_cpu(cqe->ud.rxlen_pkey) >>
2626                           OCRDMA_CQE_UD_XFER_LEN_SHIFT) &
2627                           OCRDMA_CQE_UD_XFER_LEN_MASK;
2628
2629         if (ocrdma_is_udp_encap_supported(dev)) {
2630                 hdr_type = (le32_to_cpu(cqe->ud.rxlen_pkey) >>
2631                             OCRDMA_CQE_UD_L3TYPE_SHIFT) &
2632                             OCRDMA_CQE_UD_L3TYPE_MASK;
2633                 ibwc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
2634                 ibwc->network_hdr_type = hdr_type;
2635         }
2636
2637         return status;
2638 }
2639
2640 static void ocrdma_update_free_srq_cqe(struct ib_wc *ibwc,
2641                                        struct ocrdma_cqe *cqe,
2642                                        struct ocrdma_qp *qp)
2643 {
2644         unsigned long flags;
2645         struct ocrdma_srq *srq;
2646         u32 wqe_idx;
2647
2648         srq = get_ocrdma_srq(qp->ibqp.srq);
2649         wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >>
2650                 OCRDMA_CQE_BUFTAG_SHIFT) & srq->rq.max_wqe_idx;
2651         BUG_ON(wqe_idx < 1);
2652
2653         ibwc->wr_id = srq->rqe_wr_id_tbl[wqe_idx];
2654         spin_lock_irqsave(&srq->q_lock, flags);
2655         ocrdma_srq_toggle_bit(srq, wqe_idx - 1);
2656         spin_unlock_irqrestore(&srq->q_lock, flags);
2657         ocrdma_hwq_inc_tail(&srq->rq);
2658 }
2659
2660 static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2661                                 struct ib_wc *ibwc, bool *polled, bool *stop,
2662                                 int status)
2663 {
2664         bool expand;
2665         struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device);
2666
2667         if (status < OCRDMA_MAX_CQE_ERR)
2668                 atomic_inc(&dev->cqe_err_stats[status]);
2669
2670         /* when hw_rq is empty, but wq is not empty, so continue
2671          * to keep the cqe to get the cq event again.
2672          */
2673         if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) {
2674                 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
2675                         *polled = true;
2676                         status = OCRDMA_CQE_WR_FLUSH_ERR;
2677                         expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
2678                 } else {
2679                         *polled = false;
2680                         *stop = true;
2681                         expand = false;
2682                 }
2683         } else if (is_hw_rq_empty(qp)) {
2684                 /* Do nothing */
2685                 expand = false;
2686                 *polled = false;
2687                 *stop = false;
2688         } else {
2689                 *polled = true;
2690                 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
2691         }
2692         return expand;
2693 }
2694
2695 static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp,
2696                                      struct ocrdma_cqe *cqe, struct ib_wc *ibwc)
2697 {
2698         struct ocrdma_dev *dev;
2699
2700         dev = get_ocrdma_dev(qp->ibqp.device);
2701         ibwc->opcode = IB_WC_RECV;
2702         ibwc->qp = &qp->ibqp;
2703         ibwc->status = IB_WC_SUCCESS;
2704
2705         if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI)
2706                 ocrdma_update_ud_rcqe(dev, ibwc, cqe);
2707         else
2708                 ibwc->byte_len = le32_to_cpu(cqe->rq.rxlen);
2709
2710         if (is_cqe_imm(cqe)) {
2711                 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt));
2712                 ibwc->wc_flags |= IB_WC_WITH_IMM;
2713         } else if (is_cqe_wr_imm(cqe)) {
2714                 ibwc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2715                 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt));
2716                 ibwc->wc_flags |= IB_WC_WITH_IMM;
2717         } else if (is_cqe_invalidated(cqe)) {
2718                 ibwc->ex.invalidate_rkey = le32_to_cpu(cqe->rq.lkey_immdt);
2719                 ibwc->wc_flags |= IB_WC_WITH_INVALIDATE;
2720         }
2721         if (qp->ibqp.srq) {
2722                 ocrdma_update_free_srq_cqe(ibwc, cqe, qp);
2723         } else {
2724                 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2725                 ocrdma_hwq_inc_tail(&qp->rq);
2726         }
2727 }
2728
2729 static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2730                              struct ib_wc *ibwc, bool *polled, bool *stop)
2731 {
2732         int status;
2733         bool expand = false;
2734
2735         ibwc->wc_flags = 0;
2736         if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
2737                 status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2738                                         OCRDMA_CQE_UD_STATUS_MASK) >>
2739                                         OCRDMA_CQE_UD_STATUS_SHIFT;
2740         } else {
2741                 status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2742                              OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2743         }
2744
2745         if (status == OCRDMA_CQE_SUCCESS) {
2746                 *polled = true;
2747                 ocrdma_poll_success_rcqe(qp, cqe, ibwc);
2748         } else {
2749                 expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop,
2750                                               status);
2751         }
2752         return expand;
2753 }
2754
2755 static void ocrdma_change_cq_phase(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe,
2756                                    u16 cur_getp)
2757 {
2758         if (cq->phase_change) {
2759                 if (cur_getp == 0)
2760                         cq->phase = (~cq->phase & OCRDMA_CQE_VALID);
2761         } else {
2762                 /* clear valid bit */
2763                 cqe->flags_status_srcqpn = 0;
2764         }
2765 }
2766
2767 static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries,
2768                             struct ib_wc *ibwc)
2769 {
2770         u16 qpn = 0;
2771         int i = 0;
2772         bool expand = false;
2773         int polled_hw_cqes = 0;
2774         struct ocrdma_qp *qp = NULL;
2775         struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device);
2776         struct ocrdma_cqe *cqe;
2777         u16 cur_getp; bool polled = false; bool stop = false;
2778
2779         cur_getp = cq->getp;
2780         while (num_entries) {
2781                 cqe = cq->va + cur_getp;
2782                 /* check whether valid cqe or not */
2783                 if (!is_cqe_valid(cq, cqe))
2784                         break;
2785                 qpn = (le32_to_cpu(cqe->cmn.qpn) & OCRDMA_CQE_QPN_MASK);
2786                 /* ignore discarded cqe */
2787                 if (qpn == 0)
2788                         goto skip_cqe;
2789                 qp = dev->qp_tbl[qpn];
2790                 BUG_ON(qp == NULL);
2791
2792                 if (is_cqe_for_sq(cqe)) {
2793                         expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled,
2794                                                   &stop);
2795                 } else {
2796                         expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled,
2797                                                   &stop);
2798                 }
2799                 if (expand)
2800                         goto expand_cqe;
2801                 if (stop)
2802                         goto stop_cqe;
2803                 /* clear qpn to avoid duplicate processing by discard_cqe() */
2804                 cqe->cmn.qpn = 0;
2805 skip_cqe:
2806                 polled_hw_cqes += 1;
2807                 cur_getp = (cur_getp + 1) % cq->max_hw_cqe;
2808                 ocrdma_change_cq_phase(cq, cqe, cur_getp);
2809 expand_cqe:
2810                 if (polled) {
2811                         num_entries -= 1;
2812                         i += 1;
2813                         ibwc = ibwc + 1;
2814                         polled = false;
2815                 }
2816         }
2817 stop_cqe:
2818         cq->getp = cur_getp;
2819
2820         if (polled_hw_cqes)
2821                 ocrdma_ring_cq_db(dev, cq->id, false, false, polled_hw_cqes);
2822
2823         return i;
2824 }
2825
2826 /* insert error cqe if the QP's SQ or RQ's CQ matches the CQ under poll. */
2827 static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries,
2828                               struct ocrdma_qp *qp, struct ib_wc *ibwc)
2829 {
2830         int err_cqes = 0;
2831
2832         while (num_entries) {
2833                 if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp))
2834                         break;
2835                 if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) {
2836                         ocrdma_update_wc(qp, ibwc, qp->sq.tail);
2837                         ocrdma_hwq_inc_tail(&qp->sq);
2838                 } else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) {
2839                         ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2840                         ocrdma_hwq_inc_tail(&qp->rq);
2841                 } else {
2842                         return err_cqes;
2843                 }
2844                 ibwc->byte_len = 0;
2845                 ibwc->status = IB_WC_WR_FLUSH_ERR;
2846                 ibwc = ibwc + 1;
2847                 err_cqes += 1;
2848                 num_entries -= 1;
2849         }
2850         return err_cqes;
2851 }
2852
2853 int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
2854 {
2855         int cqes_to_poll = num_entries;
2856         struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
2857         struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
2858         int num_os_cqe = 0, err_cqes = 0;
2859         struct ocrdma_qp *qp;
2860         unsigned long flags;
2861
2862         /* poll cqes from adapter CQ */
2863         spin_lock_irqsave(&cq->cq_lock, flags);
2864         num_os_cqe = ocrdma_poll_hwcq(cq, cqes_to_poll, wc);
2865         spin_unlock_irqrestore(&cq->cq_lock, flags);
2866         cqes_to_poll -= num_os_cqe;
2867
2868         if (cqes_to_poll) {
2869                 wc = wc + num_os_cqe;
2870                 /* adapter returns single error cqe when qp moves to
2871                  * error state. So insert error cqes with wc_status as
2872                  * FLUSHED for pending WQEs and RQEs of QP's SQ and RQ
2873                  * respectively which uses this CQ.
2874                  */
2875                 spin_lock_irqsave(&dev->flush_q_lock, flags);
2876                 list_for_each_entry(qp, &cq->sq_head, sq_entry) {
2877                         if (cqes_to_poll == 0)
2878                                 break;
2879                         err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc);
2880                         cqes_to_poll -= err_cqes;
2881                         num_os_cqe += err_cqes;
2882                         wc = wc + err_cqes;
2883                 }
2884                 spin_unlock_irqrestore(&dev->flush_q_lock, flags);
2885         }
2886         return num_os_cqe;
2887 }
2888
2889 int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags)
2890 {
2891         struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
2892         struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
2893         u16 cq_id;
2894         unsigned long flags;
2895         bool arm_needed = false, sol_needed = false;
2896
2897         cq_id = cq->id;
2898
2899         spin_lock_irqsave(&cq->cq_lock, flags);
2900         if (cq_flags & IB_CQ_NEXT_COMP || cq_flags & IB_CQ_SOLICITED)
2901                 arm_needed = true;
2902         if (cq_flags & IB_CQ_SOLICITED)
2903                 sol_needed = true;
2904
2905         ocrdma_ring_cq_db(dev, cq_id, arm_needed, sol_needed, 0);
2906         spin_unlock_irqrestore(&cq->cq_lock, flags);
2907
2908         return 0;
2909 }
2910
2911 struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
2912                               u32 max_num_sg, struct ib_udata *udata)
2913 {
2914         int status;
2915         struct ocrdma_mr *mr;
2916         struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
2917         struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
2918
2919         if (mr_type != IB_MR_TYPE_MEM_REG)
2920                 return ERR_PTR(-EINVAL);
2921
2922         if (max_num_sg > dev->attr.max_pages_per_frmr)
2923                 return ERR_PTR(-EINVAL);
2924
2925         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2926         if (!mr)
2927                 return ERR_PTR(-ENOMEM);
2928
2929         mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
2930         if (!mr->pages) {
2931                 status = -ENOMEM;
2932                 goto pl_err;
2933         }
2934
2935         status = ocrdma_get_pbl_info(dev, mr, max_num_sg);
2936         if (status)
2937                 goto pbl_err;
2938         mr->hwmr.fr_mr = 1;
2939         mr->hwmr.remote_rd = 0;
2940         mr->hwmr.remote_wr = 0;
2941         mr->hwmr.local_rd = 0;
2942         mr->hwmr.local_wr = 0;
2943         mr->hwmr.mw_bind = 0;
2944         status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);
2945         if (status)
2946                 goto pbl_err;
2947         status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, 0);
2948         if (status)
2949                 goto mbx_err;
2950         mr->ibmr.rkey = mr->hwmr.lkey;
2951         mr->ibmr.lkey = mr->hwmr.lkey;
2952         dev->stag_arr[(mr->hwmr.lkey >> 8) & (OCRDMA_MAX_STAG - 1)] =
2953                 (unsigned long) mr;
2954         return &mr->ibmr;
2955 mbx_err:
2956         ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
2957 pbl_err:
2958         kfree(mr->pages);
2959 pl_err:
2960         kfree(mr);
2961         return ERR_PTR(-ENOMEM);
2962 }
2963
2964 static int ocrdma_set_page(struct ib_mr *ibmr, u64 addr)
2965 {
2966         struct ocrdma_mr *mr = get_ocrdma_mr(ibmr);
2967
2968         if (unlikely(mr->npages == mr->hwmr.num_pbes))
2969                 return -ENOMEM;
2970
2971         mr->pages[mr->npages++] = addr;
2972
2973         return 0;
2974 }
2975
2976 int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
2977                      unsigned int *sg_offset)
2978 {
2979         struct ocrdma_mr *mr = get_ocrdma_mr(ibmr);
2980
2981         mr->npages = 0;
2982
2983         return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, ocrdma_set_page);
2984 }