drivers/infiniband/sw/siw/siw_verbs.c

   1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
   2
   3 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
   4 /* Copyright (c) 2008-2019, IBM Corporation */
   5
   6 #include <linux/errno.h>
   7 #include <linux/types.h>
   8 #include <linux/uaccess.h>
   9 #include <linux/vmalloc.h>
  10 #include <linux/xarray.h>
  11 #include <net/addrconf.h>
  12
  13 #include <rdma/iw_cm.h>
  14 #include <rdma/ib_verbs.h>
  15 #include <rdma/ib_user_verbs.h>
  16 #include <rdma/uverbs_ioctl.h>
  17
  18 #include "siw.h"
  19 #include "siw_verbs.h"
  20 #include "siw_mem.h"
  21
  22 static int ib_qp_state_to_siw_qp_state[IB_QPS_ERR + 1] = {
  23         [IB_QPS_RESET] = SIW_QP_STATE_IDLE,
  24         [IB_QPS_INIT] = SIW_QP_STATE_IDLE,
  25         [IB_QPS_RTR] = SIW_QP_STATE_RTR,
  26         [IB_QPS_RTS] = SIW_QP_STATE_RTS,
  27         [IB_QPS_SQD] = SIW_QP_STATE_CLOSING,
  28         [IB_QPS_SQE] = SIW_QP_STATE_TERMINATE,
  29         [IB_QPS_ERR] = SIW_QP_STATE_ERROR
  30 };
  31
  32 static char ib_qp_state_to_string[IB_QPS_ERR + 1][sizeof("RESET")] = {
  33         [IB_QPS_RESET] = "RESET", [IB_QPS_INIT] = "INIT", [IB_QPS_RTR] = "RTR",
  34         [IB_QPS_RTS] = "RTS",     [IB_QPS_SQD] = "SQD",   [IB_QPS_SQE] = "SQE",
  35         [IB_QPS_ERR] = "ERR"
  36 };
  37
  38 void siw_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
  39 {
  40         struct siw_user_mmap_entry *entry = to_siw_mmap_entry(rdma_entry);
  41
  42         kfree(entry);
  43 }
  44
  45 int siw_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
  46 {
  47         struct siw_ucontext *uctx = to_siw_ctx(ctx);
  48         size_t size = vma->vm_end - vma->vm_start;
  49         struct rdma_user_mmap_entry *rdma_entry;
  50         struct siw_user_mmap_entry *entry;
  51         int rv = -EINVAL;
  52
  53         /*
  54          * Must be page aligned
  55          */
  56         if (vma->vm_start & (PAGE_SIZE - 1)) {
  57                 pr_warn("siw: mmap not page aligned\n");
  58                 return -EINVAL;
  59         }
  60         rdma_entry = rdma_user_mmap_entry_get(&uctx->base_ucontext, vma);
  61         if (!rdma_entry) {
  62                 siw_dbg(&uctx->sdev->base_dev, "mmap lookup failed: %lu, %#zx\n",
  63                         vma->vm_pgoff, size);
  64                 return -EINVAL;
  65         }
  66         entry = to_siw_mmap_entry(rdma_entry);
  67
  68         rv = remap_vmalloc_range(vma, entry->address, 0);
  69         if (rv) {
  70                 pr_warn("remap_vmalloc_range failed: %lu, %zu\n", vma->vm_pgoff,
  71                         size);
  72                 goto out;
  73         }
  74 out:
  75         rdma_user_mmap_entry_put(rdma_entry);
  76
  77         return rv;
  78 }
  79
  80 int siw_alloc_ucontext(struct ib_ucontext *base_ctx, struct ib_udata *udata)
  81 {
  82         struct siw_device *sdev = to_siw_dev(base_ctx->device);
  83         struct siw_ucontext *ctx = to_siw_ctx(base_ctx);
  84         struct siw_uresp_alloc_ctx uresp = {};
  85         int rv;
  86
  87         if (atomic_inc_return(&sdev->num_ctx) > SIW_MAX_CONTEXT) {
  88                 rv = -ENOMEM;
  89                 goto err_out;
  90         }
  91         ctx->sdev = sdev;
  92
  93         uresp.dev_id = sdev->vendor_part_id;
  94
  95         if (udata->outlen < sizeof(uresp)) {
  96                 rv = -EINVAL;
  97                 goto err_out;
  98         }
  99         rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 100         if (rv)
 101                 goto err_out;
 102
 103         siw_dbg(base_ctx->device, "success. now %d context(s)\n",
 104                 atomic_read(&sdev->num_ctx));
 105
 106         return 0;
 107
 108 err_out:
 109         atomic_dec(&sdev->num_ctx);
 110         siw_dbg(base_ctx->device, "failure %d. now %d context(s)\n", rv,
 111                 atomic_read(&sdev->num_ctx));
 112
 113         return rv;
 114 }
 115
 116 void siw_dealloc_ucontext(struct ib_ucontext *base_ctx)
 117 {
 118         struct siw_ucontext *uctx = to_siw_ctx(base_ctx);
 119
 120         atomic_dec(&uctx->sdev->num_ctx);
 121 }
 122
 123 int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr,
 124                      struct ib_udata *udata)
 125 {
 126         struct siw_device *sdev = to_siw_dev(base_dev);
 127
 128         if (udata->inlen || udata->outlen)
 129                 return -EINVAL;
 130
 131         memset(attr, 0, sizeof(*attr));
 132
 133         /* Revisit atomic caps if RFC 7306 gets supported */
 134         attr->atomic_cap = 0;
 135         attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS;
 136         attr->kernel_cap_flags = IBK_ALLOW_USER_UNREG;
 137         attr->max_cq = sdev->attrs.max_cq;
 138         attr->max_cqe = sdev->attrs.max_cqe;
 139         attr->max_fast_reg_page_list_len = SIW_MAX_SGE_PBL;
 140         attr->max_mr = sdev->attrs.max_mr;
 141         attr->max_mw = sdev->attrs.max_mw;
 142         attr->max_mr_size = ~0ull;
 143         attr->max_pd = sdev->attrs.max_pd;
 144         attr->max_qp = sdev->attrs.max_qp;
 145         attr->max_qp_init_rd_atom = sdev->attrs.max_ird;
 146         attr->max_qp_rd_atom = sdev->attrs.max_ord;
 147         attr->max_qp_wr = sdev->attrs.max_qp_wr;
 148         attr->max_recv_sge = sdev->attrs.max_sge;
 149         attr->max_res_rd_atom = sdev->attrs.max_qp * sdev->attrs.max_ird;
 150         attr->max_send_sge = sdev->attrs.max_sge;
 151         attr->max_sge_rd = sdev->attrs.max_sge_rd;
 152         attr->max_srq = sdev->attrs.max_srq;
 153         attr->max_srq_sge = sdev->attrs.max_srq_sge;
 154         attr->max_srq_wr = sdev->attrs.max_srq_wr;
 155         attr->page_size_cap = PAGE_SIZE;
 156         attr->vendor_id = SIW_VENDOR_ID;
 157         attr->vendor_part_id = sdev->vendor_part_id;
 158
 159         addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
 160                             sdev->netdev->dev_addr);
 161
 162         return 0;
 163 }
 164
 165 int siw_query_port(struct ib_device *base_dev, u32 port,
 166                    struct ib_port_attr *attr)
 167 {
 168         struct siw_device *sdev = to_siw_dev(base_dev);
 169         int rv;
 170
 171         memset(attr, 0, sizeof(*attr));
 172
 173         rv = ib_get_eth_speed(base_dev, port, &attr->active_speed,
 174                          &attr->active_width);
 175         attr->gid_tbl_len = 1;
 176         attr->max_msg_sz = -1;
 177         attr->max_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
 178         attr->active_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
 179         attr->phys_state = sdev->state == IB_PORT_ACTIVE ?
 180                 IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED;
 181         attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
 182         attr->state = sdev->state;
 183         /*
 184          * All zero
 185          *
 186          * attr->lid = 0;
 187          * attr->bad_pkey_cntr = 0;
 188          * attr->qkey_viol_cntr = 0;
 189          * attr->sm_lid = 0;
 190          * attr->lmc = 0;
 191          * attr->max_vl_num = 0;
 192          * attr->sm_sl = 0;
 193          * attr->subnet_timeout = 0;
 194          * attr->init_type_repy = 0;
 195          */
 196         return rv;
 197 }
 198
 199 int siw_get_port_immutable(struct ib_device *base_dev, u32 port,
 200                            struct ib_port_immutable *port_immutable)
 201 {
 202         struct ib_port_attr attr;
 203         int rv = siw_query_port(base_dev, port, &attr);
 204
 205         if (rv)
 206                 return rv;
 207
 208         port_immutable->gid_tbl_len = attr.gid_tbl_len;
 209         port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
 210
 211         return 0;
 212 }
 213
 214 int siw_query_gid(struct ib_device *base_dev, u32 port, int idx,
 215                   union ib_gid *gid)
 216 {
 217         struct siw_device *sdev = to_siw_dev(base_dev);
 218
 219         /* subnet_prefix == interface_id == 0; */
 220         memset(gid, 0, sizeof(*gid));
 221         memcpy(&gid->raw[0], sdev->netdev->dev_addr, 6);
 222
 223         return 0;
 224 }
 225
 226 int siw_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 227 {
 228         struct siw_device *sdev = to_siw_dev(pd->device);
 229
 230         if (atomic_inc_return(&sdev->num_pd) > SIW_MAX_PD) {
 231                 atomic_dec(&sdev->num_pd);
 232                 return -ENOMEM;
 233         }
 234         siw_dbg_pd(pd, "now %d PD's(s)\n", atomic_read(&sdev->num_pd));
 235
 236         return 0;
 237 }
 238
 239 int siw_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 240 {
 241         struct siw_device *sdev = to_siw_dev(pd->device);
 242
 243         siw_dbg_pd(pd, "free PD\n");
 244         atomic_dec(&sdev->num_pd);
 245         return 0;
 246 }
 247
 248 void siw_qp_get_ref(struct ib_qp *base_qp)
 249 {
 250         siw_qp_get(to_siw_qp(base_qp));
 251 }
 252
 253 void siw_qp_put_ref(struct ib_qp *base_qp)
 254 {
 255         siw_qp_put(to_siw_qp(base_qp));
 256 }
 257
 258 static struct rdma_user_mmap_entry *
 259 siw_mmap_entry_insert(struct siw_ucontext *uctx,
 260                       void *address, size_t length,
 261                       u64 *offset)
 262 {
 263         struct siw_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 264         int rv;
 265
 266         *offset = SIW_INVAL_UOBJ_KEY;
 267         if (!entry)
 268                 return NULL;
 269
 270         entry->address = address;
 271
 272         rv = rdma_user_mmap_entry_insert(&uctx->base_ucontext,
 273                                          &entry->rdma_entry,
 274                                          length);
 275         if (rv) {
 276                 kfree(entry);
 277                 return NULL;
 278         }
 279
 280         *offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
 281
 282         return &entry->rdma_entry;
 283 }
 284
 285 /*
 286  * siw_create_qp()
 287  *
 288  * Create QP of requested size on given device.
 289  *
 290  * @qp:         Queue pait
 291  * @attrs:      Initial QP attributes.
 292  * @udata:      used to provide QP ID, SQ and RQ size back to user.
 293  */
 294
 295 int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
 296                   struct ib_udata *udata)
 297 {
 298         struct ib_pd *pd = ibqp->pd;
 299         struct siw_qp *qp = to_siw_qp(ibqp);
 300         struct ib_device *base_dev = pd->device;
 301         struct siw_device *sdev = to_siw_dev(base_dev);
 302         struct siw_ucontext *uctx =
 303                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
 304                                           base_ucontext);
 305         unsigned long flags;
 306         int num_sqe, num_rqe, rv = 0;
 307         size_t length;
 308
 309         siw_dbg(base_dev, "create new QP\n");
 310
 311         if (attrs->create_flags)
 312                 return -EOPNOTSUPP;
 313
 314         if (atomic_inc_return(&sdev->num_qp) > SIW_MAX_QP) {
 315                 siw_dbg(base_dev, "too many QP's\n");
 316                 rv = -ENOMEM;
 317                 goto err_atomic;
 318         }
 319         if (attrs->qp_type != IB_QPT_RC) {
 320                 siw_dbg(base_dev, "only RC QP's supported\n");
 321                 rv = -EOPNOTSUPP;
 322                 goto err_atomic;
 323         }
 324         if ((attrs->cap.max_send_wr > SIW_MAX_QP_WR) ||
 325             (attrs->cap.max_recv_wr > SIW_MAX_QP_WR) ||
 326             (attrs->cap.max_send_sge > SIW_MAX_SGE) ||
 327             (attrs->cap.max_recv_sge > SIW_MAX_SGE)) {
 328                 siw_dbg(base_dev, "QP size error\n");
 329                 rv = -EINVAL;
 330                 goto err_atomic;
 331         }
 332         if (attrs->cap.max_inline_data > SIW_MAX_INLINE) {
 333                 siw_dbg(base_dev, "max inline send: %d > %d\n",
 334                         attrs->cap.max_inline_data, (int)SIW_MAX_INLINE);
 335                 rv = -EINVAL;
 336                 goto err_atomic;
 337         }
 338         /*
 339          * NOTE: we allow for zero element SQ and RQ WQE's SGL's
 340          * but not for a QP unable to hold any WQE (SQ + RQ)
 341          */
 342         if (attrs->cap.max_send_wr + attrs->cap.max_recv_wr == 0) {
 343                 siw_dbg(base_dev, "QP must have send or receive queue\n");
 344                 rv = -EINVAL;
 345                 goto err_atomic;
 346         }
 347
 348         if (!attrs->send_cq || (!attrs->recv_cq && !attrs->srq)) {
 349                 siw_dbg(base_dev, "send CQ or receive CQ invalid\n");
 350                 rv = -EINVAL;
 351                 goto err_atomic;
 352         }
 353
 354         init_rwsem(&qp->state_lock);
 355         spin_lock_init(&qp->sq_lock);
 356         spin_lock_init(&qp->rq_lock);
 357         spin_lock_init(&qp->orq_lock);
 358
 359         rv = siw_qp_add(sdev, qp);
 360         if (rv)
 361                 goto err_atomic;
 362
 363         num_sqe = attrs->cap.max_send_wr;
 364         num_rqe = attrs->cap.max_recv_wr;
 365
 366         /* All queue indices are derived from modulo operations
 367          * on a free running 'get' (consumer) and 'put' (producer)
 368          * unsigned counter. Having queue sizes at power of two
 369          * avoids handling counter wrap around.
 370          */
 371         if (num_sqe)
 372                 num_sqe = roundup_pow_of_two(num_sqe);
 373         else {
 374                 /* Zero sized SQ is not supported */
 375                 rv = -EINVAL;
 376                 goto err_out_xa;
 377         }
 378         if (num_rqe)
 379                 num_rqe = roundup_pow_of_two(num_rqe);
 380
 381         if (udata)
 382                 qp->sendq = vmalloc_user(num_sqe * sizeof(struct siw_sqe));
 383         else
 384                 qp->sendq = vzalloc(num_sqe * sizeof(struct siw_sqe));
 385
 386         if (qp->sendq == NULL) {
 387                 rv = -ENOMEM;
 388                 goto err_out_xa;
 389         }
 390         if (attrs->sq_sig_type != IB_SIGNAL_REQ_WR) {
 391                 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
 392                         qp->attrs.flags |= SIW_SIGNAL_ALL_WR;
 393                 else {
 394                         rv = -EINVAL;
 395                         goto err_out_xa;
 396                 }
 397         }
 398         qp->pd = pd;
 399         qp->scq = to_siw_cq(attrs->send_cq);
 400         qp->rcq = to_siw_cq(attrs->recv_cq);
 401
 402         if (attrs->srq) {
 403                 /*
 404                  * SRQ support.
 405                  * Verbs 6.3.7: ignore RQ size, if SRQ present
 406                  * Verbs 6.3.5: do not check PD of SRQ against PD of QP
 407                  */
 408                 qp->srq = to_siw_srq(attrs->srq);
 409                 qp->attrs.rq_size = 0;
 410                 siw_dbg(base_dev, "QP [%u]: SRQ attached\n",
 411                         qp->base_qp.qp_num);
 412         } else if (num_rqe) {
 413                 if (udata)
 414                         qp->recvq =
 415                                 vmalloc_user(num_rqe * sizeof(struct siw_rqe));
 416                 else
 417                         qp->recvq = vzalloc(num_rqe * sizeof(struct siw_rqe));
 418
 419                 if (qp->recvq == NULL) {
 420                         rv = -ENOMEM;
 421                         goto err_out_xa;
 422                 }
 423                 qp->attrs.rq_size = num_rqe;
 424         }
 425         qp->attrs.sq_size = num_sqe;
 426         qp->attrs.sq_max_sges = attrs->cap.max_send_sge;
 427         qp->attrs.rq_max_sges = attrs->cap.max_recv_sge;
 428
 429         /* Make those two tunables fixed for now. */
 430         qp->tx_ctx.gso_seg_limit = 1;
 431         qp->tx_ctx.zcopy_tx = zcopy_tx;
 432
 433         qp->attrs.state = SIW_QP_STATE_IDLE;
 434
 435         if (udata) {
 436                 struct siw_uresp_create_qp uresp = {};
 437
 438                 uresp.num_sqe = num_sqe;
 439                 uresp.num_rqe = num_rqe;
 440                 uresp.qp_id = qp_id(qp);
 441
 442                 if (qp->sendq) {
 443                         length = num_sqe * sizeof(struct siw_sqe);
 444                         qp->sq_entry =
 445                                 siw_mmap_entry_insert(uctx, qp->sendq,
 446                                                       length, &uresp.sq_key);
 447                         if (!qp->sq_entry) {
 448                                 rv = -ENOMEM;
 449                                 goto err_out_xa;
 450                         }
 451                 }
 452
 453                 if (qp->recvq) {
 454                         length = num_rqe * sizeof(struct siw_rqe);
 455                         qp->rq_entry =
 456                                 siw_mmap_entry_insert(uctx, qp->recvq,
 457                                                       length, &uresp.rq_key);
 458                         if (!qp->rq_entry) {
 459                                 uresp.sq_key = SIW_INVAL_UOBJ_KEY;
 460                                 rv = -ENOMEM;
 461                                 goto err_out_xa;
 462                         }
 463                 }
 464
 465                 if (udata->outlen < sizeof(uresp)) {
 466                         rv = -EINVAL;
 467                         goto err_out_xa;
 468                 }
 469                 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 470                 if (rv)
 471                         goto err_out_xa;
 472         }
 473         qp->tx_cpu = siw_get_tx_cpu(sdev);
 474         if (qp->tx_cpu < 0) {
 475                 rv = -EINVAL;
 476                 goto err_out_xa;
 477         }
 478         INIT_LIST_HEAD(&qp->devq);
 479         spin_lock_irqsave(&sdev->lock, flags);
 480         list_add_tail(&qp->devq, &sdev->qp_list);
 481         spin_unlock_irqrestore(&sdev->lock, flags);
 482
 483         init_completion(&qp->qp_free);
 484
 485         return 0;
 486
 487 err_out_xa:
 488         xa_erase(&sdev->qp_xa, qp_id(qp));
 489         if (uctx) {
 490                 rdma_user_mmap_entry_remove(qp->sq_entry);
 491                 rdma_user_mmap_entry_remove(qp->rq_entry);
 492         }
 493         vfree(qp->sendq);
 494         vfree(qp->recvq);
 495
 496 err_atomic:
 497         atomic_dec(&sdev->num_qp);
 498         return rv;
 499 }
 500
 501 /*
 502  * Minimum siw_query_qp() verb interface.
 503  *
 504  * @qp_attr_mask is not used but all available information is provided
 505  */
 506 int siw_query_qp(struct ib_qp *base_qp, struct ib_qp_attr *qp_attr,
 507                  int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
 508 {
 509         struct siw_qp *qp;
 510         struct siw_device *sdev;
 511
 512         if (base_qp && qp_attr && qp_init_attr) {
 513                 qp = to_siw_qp(base_qp);
 514                 sdev = to_siw_dev(base_qp->device);
 515         } else {
 516                 return -EINVAL;
 517         }
 518         qp_attr->cap.max_inline_data = SIW_MAX_INLINE;
 519         qp_attr->cap.max_send_wr = qp->attrs.sq_size;
 520         qp_attr->cap.max_send_sge = qp->attrs.sq_max_sges;
 521         qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
 522         qp_attr->cap.max_recv_sge = qp->attrs.rq_max_sges;
 523         qp_attr->path_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
 524         qp_attr->max_rd_atomic = qp->attrs.irq_size;
 525         qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
 526
 527         qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
 528                                    IB_ACCESS_REMOTE_WRITE |
 529                                    IB_ACCESS_REMOTE_READ;
 530
 531         qp_init_attr->qp_type = base_qp->qp_type;
 532         qp_init_attr->send_cq = base_qp->send_cq;
 533         qp_init_attr->recv_cq = base_qp->recv_cq;
 534         qp_init_attr->srq = base_qp->srq;
 535
 536         qp_init_attr->cap = qp_attr->cap;
 537
 538         return 0;
 539 }
 540
 541 int siw_verbs_modify_qp(struct ib_qp *base_qp, struct ib_qp_attr *attr,
 542                         int attr_mask, struct ib_udata *udata)
 543 {
 544         struct siw_qp_attrs new_attrs;
 545         enum siw_qp_attr_mask siw_attr_mask = 0;
 546         struct siw_qp *qp = to_siw_qp(base_qp);
 547         int rv = 0;
 548
 549         if (!attr_mask)
 550                 return 0;
 551
 552         if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
 553                 return -EOPNOTSUPP;
 554
 555         memset(&new_attrs, 0, sizeof(new_attrs));
 556
 557         if (attr_mask & IB_QP_ACCESS_FLAGS) {
 558                 siw_attr_mask = SIW_QP_ATTR_ACCESS_FLAGS;
 559
 560                 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
 561                         new_attrs.flags |= SIW_RDMA_READ_ENABLED;
 562                 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
 563                         new_attrs.flags |= SIW_RDMA_WRITE_ENABLED;
 564                 if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
 565                         new_attrs.flags |= SIW_RDMA_BIND_ENABLED;
 566         }
 567         if (attr_mask & IB_QP_STATE) {
 568                 siw_dbg_qp(qp, "desired IB QP state: %s\n",
 569                            ib_qp_state_to_string[attr->qp_state]);
 570
 571                 new_attrs.state = ib_qp_state_to_siw_qp_state[attr->qp_state];
 572
 573                 if (new_attrs.state > SIW_QP_STATE_RTS)
 574                         qp->tx_ctx.tx_suspend = 1;
 575
 576                 siw_attr_mask |= SIW_QP_ATTR_STATE;
 577         }
 578         if (!siw_attr_mask)
 579                 goto out;
 580
 581         down_write(&qp->state_lock);
 582
 583         rv = siw_qp_modify(qp, &new_attrs, siw_attr_mask);
 584
 585         up_write(&qp->state_lock);
 586 out:
 587         return rv;
 588 }
 589
 590 int siw_destroy_qp(struct ib_qp *base_qp, struct ib_udata *udata)
 591 {
 592         struct siw_qp *qp = to_siw_qp(base_qp);
 593         struct siw_ucontext *uctx =
 594                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
 595                                           base_ucontext);
 596         struct siw_qp_attrs qp_attrs;
 597
 598         siw_dbg_qp(qp, "state %d\n", qp->attrs.state);
 599
 600         /*
 601          * Mark QP as in process of destruction to prevent from
 602          * any async callbacks to RDMA core
 603          */
 604         qp->attrs.flags |= SIW_QP_IN_DESTROY;
 605         qp->rx_stream.rx_suspend = 1;
 606
 607         if (uctx) {
 608                 rdma_user_mmap_entry_remove(qp->sq_entry);
 609                 rdma_user_mmap_entry_remove(qp->rq_entry);
 610         }
 611
 612         down_write(&qp->state_lock);
 613
 614         qp_attrs.state = SIW_QP_STATE_ERROR;
 615         siw_qp_modify(qp, &qp_attrs, SIW_QP_ATTR_STATE);
 616
 617         if (qp->cep) {
 618                 siw_cep_put(qp->cep);
 619                 qp->cep = NULL;
 620         }
 621         up_write(&qp->state_lock);
 622
 623         kfree(qp->tx_ctx.mpa_crc_hd);
 624         kfree(qp->rx_stream.mpa_crc_hd);
 625
 626         qp->scq = qp->rcq = NULL;
 627
 628         siw_qp_put(qp);
 629         wait_for_completion(&qp->qp_free);
 630
 631         return 0;
 632 }
 633
 634 /*
 635  * siw_copy_inline_sgl()
 636  *
 637  * Prepare sgl of inlined data for sending. For userland callers
 638  * function checks if given buffer addresses and len's are within
 639  * process context bounds.
 640  * Data from all provided sge's are copied together into the wqe,
 641  * referenced by a single sge.
 642  */
 643 static int siw_copy_inline_sgl(const struct ib_send_wr *core_wr,
 644                                struct siw_sqe *sqe)
 645 {
 646         struct ib_sge *core_sge = core_wr->sg_list;
 647         void *kbuf = &sqe->sge[1];
 648         int num_sge = core_wr->num_sge, bytes = 0;
 649
 650         sqe->sge[0].laddr = (uintptr_t)kbuf;
 651         sqe->sge[0].lkey = 0;
 652
 653         while (num_sge--) {
 654                 if (!core_sge->length) {
 655                         core_sge++;
 656                         continue;
 657                 }
 658                 bytes += core_sge->length;
 659                 if (bytes > SIW_MAX_INLINE) {
 660                         bytes = -EINVAL;
 661                         break;
 662                 }
 663                 memcpy(kbuf, ib_virt_dma_to_ptr(core_sge->addr),
 664                        core_sge->length);
 665
 666                 kbuf += core_sge->length;
 667                 core_sge++;
 668         }
 669         sqe->sge[0].length = max(bytes, 0);
 670         sqe->num_sge = bytes > 0 ? 1 : 0;
 671
 672         return bytes;
 673 }
 674
 675 /* Complete SQ WR's without processing */
 676 static int siw_sq_flush_wr(struct siw_qp *qp, const struct ib_send_wr *wr,
 677                            const struct ib_send_wr **bad_wr)
 678 {
 679         int rv = 0;
 680
 681         while (wr) {
 682                 struct siw_sqe sqe = {};
 683
 684                 switch (wr->opcode) {
 685                 case IB_WR_RDMA_WRITE:
 686                         sqe.opcode = SIW_OP_WRITE;
 687                         break;
 688                 case IB_WR_RDMA_READ:
 689                         sqe.opcode = SIW_OP_READ;
 690                         break;
 691                 case IB_WR_RDMA_READ_WITH_INV:
 692                         sqe.opcode = SIW_OP_READ_LOCAL_INV;
 693                         break;
 694                 case IB_WR_SEND:
 695                         sqe.opcode = SIW_OP_SEND;
 696                         break;
 697                 case IB_WR_SEND_WITH_IMM:
 698                         sqe.opcode = SIW_OP_SEND_WITH_IMM;
 699                         break;
 700                 case IB_WR_SEND_WITH_INV:
 701                         sqe.opcode = SIW_OP_SEND_REMOTE_INV;
 702                         break;
 703                 case IB_WR_LOCAL_INV:
 704                         sqe.opcode = SIW_OP_INVAL_STAG;
 705                         break;
 706                 case IB_WR_REG_MR:
 707                         sqe.opcode = SIW_OP_REG_MR;
 708                         break;
 709                 default:
 710                         rv = -EINVAL;
 711                         break;
 712                 }
 713                 if (!rv) {
 714                         sqe.id = wr->wr_id;
 715                         rv = siw_sqe_complete(qp, &sqe, 0,
 716                                               SIW_WC_WR_FLUSH_ERR);
 717                 }
 718                 if (rv) {
 719                         if (bad_wr)
 720                                 *bad_wr = wr;
 721                         break;
 722                 }
 723                 wr = wr->next;
 724         }
 725         return rv;
 726 }
 727
 728 /* Complete RQ WR's without processing */
 729 static int siw_rq_flush_wr(struct siw_qp *qp, const struct ib_recv_wr *wr,
 730                            const struct ib_recv_wr **bad_wr)
 731 {
 732         struct siw_rqe rqe = {};
 733         int rv = 0;
 734
 735         while (wr) {
 736                 rqe.id = wr->wr_id;
 737                 rv = siw_rqe_complete(qp, &rqe, 0, 0, SIW_WC_WR_FLUSH_ERR);
 738                 if (rv) {
 739                         if (bad_wr)
 740                                 *bad_wr = wr;
 741                         break;
 742                 }
 743                 wr = wr->next;
 744         }
 745         return rv;
 746 }
 747
 748 /*
 749  * siw_post_send()
 750  *
 751  * Post a list of S-WR's to a SQ.
 752  *
 753  * @base_qp:    Base QP contained in siw QP
 754  * @wr:         Null terminated list of user WR's
 755  * @bad_wr:     Points to failing WR in case of synchronous failure.
 756  */
 757 int siw_post_send(struct ib_qp *base_qp, const struct ib_send_wr *wr,
 758                   const struct ib_send_wr **bad_wr)
 759 {
 760         struct siw_qp *qp = to_siw_qp(base_qp);
 761         struct siw_wqe *wqe = tx_wqe(qp);
 762
 763         unsigned long flags;
 764         int rv = 0;
 765
 766         if (wr && !rdma_is_kernel_res(&qp->base_qp.res)) {
 767                 siw_dbg_qp(qp, "wr must be empty for user mapped sq\n");
 768                 *bad_wr = wr;
 769                 return -EINVAL;
 770         }
 771
 772         /*
 773          * Try to acquire QP state lock. Must be non-blocking
 774          * to accommodate kernel clients needs.
 775          */
 776         if (!down_read_trylock(&qp->state_lock)) {
 777                 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
 778                         /*
 779                          * ERROR state is final, so we can be sure
 780                          * this state will not change as long as the QP
 781                          * exists.
 782                          *
 783                          * This handles an ib_drain_sq() call with
 784                          * a concurrent request to set the QP state
 785                          * to ERROR.
 786                          */
 787                         rv = siw_sq_flush_wr(qp, wr, bad_wr);
 788                 } else {
 789                         siw_dbg_qp(qp, "QP locked, state %d\n",
 790                                    qp->attrs.state);
 791                         *bad_wr = wr;
 792                         rv = -ENOTCONN;
 793                 }
 794                 return rv;
 795         }
 796         if (unlikely(qp->attrs.state != SIW_QP_STATE_RTS)) {
 797                 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
 798                         /*
 799                          * Immediately flush this WR to CQ, if QP
 800                          * is in ERROR state. SQ is guaranteed to
 801                          * be empty, so WR complets in-order.
 802                          *
 803                          * Typically triggered by ib_drain_sq().
 804                          */
 805                         rv = siw_sq_flush_wr(qp, wr, bad_wr);
 806                 } else {
 807                         siw_dbg_qp(qp, "QP out of state %d\n",
 808                                    qp->attrs.state);
 809                         *bad_wr = wr;
 810                         rv = -ENOTCONN;
 811                 }
 812                 up_read(&qp->state_lock);
 813                 return rv;
 814         }
 815         spin_lock_irqsave(&qp->sq_lock, flags);
 816
 817         while (wr) {
 818                 u32 idx = qp->sq_put % qp->attrs.sq_size;
 819                 struct siw_sqe *sqe = &qp->sendq[idx];
 820
 821                 if (sqe->flags) {
 822                         siw_dbg_qp(qp, "sq full\n");
 823                         rv = -ENOMEM;
 824                         break;
 825                 }
 826                 if (wr->num_sge > qp->attrs.sq_max_sges) {
 827                         siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
 828                         rv = -EINVAL;
 829                         break;
 830                 }
 831                 sqe->id = wr->wr_id;
 832
 833                 if ((wr->send_flags & IB_SEND_SIGNALED) ||
 834                     (qp->attrs.flags & SIW_SIGNAL_ALL_WR))
 835                         sqe->flags |= SIW_WQE_SIGNALLED;
 836
 837                 if (wr->send_flags & IB_SEND_FENCE)
 838                         sqe->flags |= SIW_WQE_READ_FENCE;
 839
 840                 switch (wr->opcode) {
 841                 case IB_WR_SEND:
 842                 case IB_WR_SEND_WITH_INV:
 843                         if (wr->send_flags & IB_SEND_SOLICITED)
 844                                 sqe->flags |= SIW_WQE_SOLICITED;
 845
 846                         if (!(wr->send_flags & IB_SEND_INLINE)) {
 847                                 siw_copy_sgl(wr->sg_list, sqe->sge,
 848                                              wr->num_sge);
 849                                 sqe->num_sge = wr->num_sge;
 850                         } else {
 851                                 rv = siw_copy_inline_sgl(wr, sqe);
 852                                 if (rv <= 0) {
 853                                         rv = -EINVAL;
 854                                         break;
 855                                 }
 856                                 sqe->flags |= SIW_WQE_INLINE;
 857                                 sqe->num_sge = 1;
 858                         }
 859                         if (wr->opcode == IB_WR_SEND)
 860                                 sqe->opcode = SIW_OP_SEND;
 861                         else {
 862                                 sqe->opcode = SIW_OP_SEND_REMOTE_INV;
 863                                 sqe->rkey = wr->ex.invalidate_rkey;
 864                         }
 865                         break;
 866
 867                 case IB_WR_RDMA_READ_WITH_INV:
 868                 case IB_WR_RDMA_READ:
 869                         /*
 870                          * iWarp restricts RREAD sink to SGL containing
 871                          * 1 SGE only. we could relax to SGL with multiple
 872                          * elements referring the SAME ltag or even sending
 873                          * a private per-rreq tag referring to a checked
 874                          * local sgl with MULTIPLE ltag's.
 875                          */
 876                         if (unlikely(wr->num_sge != 1)) {
 877                                 rv = -EINVAL;
 878                                 break;
 879                         }
 880                         siw_copy_sgl(wr->sg_list, &sqe->sge[0], 1);
 881                         /*
 882                          * NOTE: zero length RREAD is allowed!
 883                          */
 884                         sqe->raddr = rdma_wr(wr)->remote_addr;
 885                         sqe->rkey = rdma_wr(wr)->rkey;
 886                         sqe->num_sge = 1;
 887
 888                         if (wr->opcode == IB_WR_RDMA_READ)
 889                                 sqe->opcode = SIW_OP_READ;
 890                         else
 891                                 sqe->opcode = SIW_OP_READ_LOCAL_INV;
 892                         break;
 893
 894                 case IB_WR_RDMA_WRITE:
 895                         if (!(wr->send_flags & IB_SEND_INLINE)) {
 896                                 siw_copy_sgl(wr->sg_list, &sqe->sge[0],
 897                                              wr->num_sge);
 898                                 sqe->num_sge = wr->num_sge;
 899                         } else {
 900                                 rv = siw_copy_inline_sgl(wr, sqe);
 901                                 if (unlikely(rv < 0)) {
 902                                         rv = -EINVAL;
 903                                         break;
 904                                 }
 905                                 sqe->flags |= SIW_WQE_INLINE;
 906                                 sqe->num_sge = 1;
 907                         }
 908                         sqe->raddr = rdma_wr(wr)->remote_addr;
 909                         sqe->rkey = rdma_wr(wr)->rkey;
 910                         sqe->opcode = SIW_OP_WRITE;
 911                         break;
 912
 913                 case IB_WR_REG_MR:
 914                         sqe->base_mr = (uintptr_t)reg_wr(wr)->mr;
 915                         sqe->rkey = reg_wr(wr)->key;
 916                         sqe->access = reg_wr(wr)->access & IWARP_ACCESS_MASK;
 917                         sqe->opcode = SIW_OP_REG_MR;
 918                         break;
 919
 920                 case IB_WR_LOCAL_INV:
 921                         sqe->rkey = wr->ex.invalidate_rkey;
 922                         sqe->opcode = SIW_OP_INVAL_STAG;
 923                         break;
 924
 925                 default:
 926                         siw_dbg_qp(qp, "ib wr type %d unsupported\n",
 927                                    wr->opcode);
 928                         rv = -EINVAL;
 929                         break;
 930                 }
 931                 siw_dbg_qp(qp, "opcode %d, flags 0x%x, wr_id 0x%pK\n",
 932                            sqe->opcode, sqe->flags,
 933                            (void *)(uintptr_t)sqe->id);
 934
 935                 if (unlikely(rv < 0))
 936                         break;
 937
 938                 /* make SQE only valid after completely written */
 939                 smp_wmb();
 940                 sqe->flags |= SIW_WQE_VALID;
 941
 942                 qp->sq_put++;
 943                 wr = wr->next;
 944         }
 945
 946         /*
 947          * Send directly if SQ processing is not in progress.
 948          * Eventual immediate errors (rv < 0) do not affect the involved
 949          * RI resources (Verbs, 8.3.1) and thus do not prevent from SQ
 950          * processing, if new work is already pending. But rv must be passed
 951          * to caller.
 952          */
 953         if (wqe->wr_status != SIW_WR_IDLE) {
 954                 spin_unlock_irqrestore(&qp->sq_lock, flags);
 955                 goto skip_direct_sending;
 956         }
 957         rv = siw_activate_tx(qp);
 958         spin_unlock_irqrestore(&qp->sq_lock, flags);
 959
 960         if (rv <= 0)
 961                 goto skip_direct_sending;
 962
 963         if (rdma_is_kernel_res(&qp->base_qp.res)) {
 964                 rv = siw_sq_start(qp);
 965         } else {
 966                 qp->tx_ctx.in_syscall = 1;
 967
 968                 if (siw_qp_sq_process(qp) != 0 && !(qp->tx_ctx.tx_suspend))
 969                         siw_qp_cm_drop(qp, 0);
 970
 971                 qp->tx_ctx.in_syscall = 0;
 972         }
 973 skip_direct_sending:
 974
 975         up_read(&qp->state_lock);
 976
 977         if (rv >= 0)
 978                 return 0;
 979         /*
 980          * Immediate error
 981          */
 982         siw_dbg_qp(qp, "error %d\n", rv);
 983
 984         *bad_wr = wr;
 985         return rv;
 986 }
 987
 988 /*
 989  * siw_post_receive()
 990  *
 991  * Post a list of R-WR's to a RQ.
 992  *
 993  * @base_qp:    Base QP contained in siw QP
 994  * @wr:         Null terminated list of user WR's
 995  * @bad_wr:     Points to failing WR in case of synchronous failure.
 996  */
 997 int siw_post_receive(struct ib_qp *base_qp, const struct ib_recv_wr *wr,
 998                      const struct ib_recv_wr **bad_wr)
 999 {
1000         struct siw_qp *qp = to_siw_qp(base_qp);
1001         unsigned long flags;
1002         int rv = 0;
1003
1004         if (qp->srq || qp->attrs.rq_size == 0) {
1005                 *bad_wr = wr;
1006                 return -EINVAL;
1007         }
1008         if (!rdma_is_kernel_res(&qp->base_qp.res)) {
1009                 siw_dbg_qp(qp, "no kernel post_recv for user mapped rq\n");
1010                 *bad_wr = wr;
1011                 return -EINVAL;
1012         }
1013
1014         /*
1015          * Try to acquire QP state lock. Must be non-blocking
1016          * to accommodate kernel clients needs.
1017          */
1018         if (!down_read_trylock(&qp->state_lock)) {
1019                 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1020                         /*
1021                          * ERROR state is final, so we can be sure
1022                          * this state will not change as long as the QP
1023                          * exists.
1024                          *
1025                          * This handles an ib_drain_rq() call with
1026                          * a concurrent request to set the QP state
1027                          * to ERROR.
1028                          */
1029                         rv = siw_rq_flush_wr(qp, wr, bad_wr);
1030                 } else {
1031                         siw_dbg_qp(qp, "QP locked, state %d\n",
1032                                    qp->attrs.state);
1033                         *bad_wr = wr;
1034                         rv = -ENOTCONN;
1035                 }
1036                 return rv;
1037         }
1038         if (qp->attrs.state > SIW_QP_STATE_RTS) {
1039                 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1040                         /*
1041                          * Immediately flush this WR to CQ, if QP
1042                          * is in ERROR state. RQ is guaranteed to
1043                          * be empty, so WR complets in-order.
1044                          *
1045                          * Typically triggered by ib_drain_rq().
1046                          */
1047                         rv = siw_rq_flush_wr(qp, wr, bad_wr);
1048                 } else {
1049                         siw_dbg_qp(qp, "QP out of state %d\n",
1050                                    qp->attrs.state);
1051                         *bad_wr = wr;
1052                         rv = -ENOTCONN;
1053                 }
1054                 up_read(&qp->state_lock);
1055                 return rv;
1056         }
1057         /*
1058          * Serialize potentially multiple producers.
1059          * Not needed for single threaded consumer side.
1060          */
1061         spin_lock_irqsave(&qp->rq_lock, flags);
1062
1063         while (wr) {
1064                 u32 idx = qp->rq_put % qp->attrs.rq_size;
1065                 struct siw_rqe *rqe = &qp->recvq[idx];
1066
1067                 if (rqe->flags) {
1068                         siw_dbg_qp(qp, "RQ full\n");
1069                         rv = -ENOMEM;
1070                         break;
1071                 }
1072                 if (wr->num_sge > qp->attrs.rq_max_sges) {
1073                         siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
1074                         rv = -EINVAL;
1075                         break;
1076                 }
1077                 rqe->id = wr->wr_id;
1078                 rqe->num_sge = wr->num_sge;
1079                 siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1080
1081                 /* make sure RQE is completely written before valid */
1082                 smp_wmb();
1083
1084                 rqe->flags = SIW_WQE_VALID;
1085
1086                 qp->rq_put++;
1087                 wr = wr->next;
1088         }
1089         spin_unlock_irqrestore(&qp->rq_lock, flags);
1090
1091         up_read(&qp->state_lock);
1092
1093         if (rv < 0) {
1094                 siw_dbg_qp(qp, "error %d\n", rv);
1095                 *bad_wr = wr;
1096         }
1097         return rv > 0 ? 0 : rv;
1098 }
1099
1100 int siw_destroy_cq(struct ib_cq *base_cq, struct ib_udata *udata)
1101 {
1102         struct siw_cq *cq = to_siw_cq(base_cq);
1103         struct siw_device *sdev = to_siw_dev(base_cq->device);
1104         struct siw_ucontext *ctx =
1105                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1106                                           base_ucontext);
1107
1108         siw_dbg_cq(cq, "free CQ resources\n");
1109
1110         siw_cq_flush(cq);
1111
1112         if (ctx)
1113                 rdma_user_mmap_entry_remove(cq->cq_entry);
1114
1115         atomic_dec(&sdev->num_cq);
1116
1117         vfree(cq->queue);
1118         return 0;
1119 }
1120
1121 /*
1122  * siw_create_cq()
1123  *
1124  * Populate CQ of requested size
1125  *
1126  * @base_cq: CQ as allocated by RDMA midlayer
1127  * @attr: Initial CQ attributes
1128  * @udata: relates to user context
1129  */
1130
1131 int siw_create_cq(struct ib_cq *base_cq, const struct ib_cq_init_attr *attr,
1132                   struct ib_udata *udata)
1133 {
1134         struct siw_device *sdev = to_siw_dev(base_cq->device);
1135         struct siw_cq *cq = to_siw_cq(base_cq);
1136         int rv, size = attr->cqe;
1137
1138         if (attr->flags)
1139                 return -EOPNOTSUPP;
1140
1141         if (atomic_inc_return(&sdev->num_cq) > SIW_MAX_CQ) {
1142                 siw_dbg(base_cq->device, "too many CQ's\n");
1143                 rv = -ENOMEM;
1144                 goto err_out;
1145         }
1146         if (size < 1 || size > sdev->attrs.max_cqe) {
1147                 siw_dbg(base_cq->device, "CQ size error: %d\n", size);
1148                 rv = -EINVAL;
1149                 goto err_out;
1150         }
1151         size = roundup_pow_of_two(size);
1152         cq->base_cq.cqe = size;
1153         cq->num_cqe = size;
1154
1155         if (udata)
1156                 cq->queue = vmalloc_user(size * sizeof(struct siw_cqe) +
1157                                          sizeof(struct siw_cq_ctrl));
1158         else
1159                 cq->queue = vzalloc(size * sizeof(struct siw_cqe) +
1160                                     sizeof(struct siw_cq_ctrl));
1161
1162         if (cq->queue == NULL) {
1163                 rv = -ENOMEM;
1164                 goto err_out;
1165         }
1166         get_random_bytes(&cq->id, 4);
1167         siw_dbg(base_cq->device, "new CQ [%u]\n", cq->id);
1168
1169         spin_lock_init(&cq->lock);
1170
1171         cq->notify = (struct siw_cq_ctrl *)&cq->queue[size];
1172
1173         if (udata) {
1174                 struct siw_uresp_create_cq uresp = {};
1175                 struct siw_ucontext *ctx =
1176                         rdma_udata_to_drv_context(udata, struct siw_ucontext,
1177                                                   base_ucontext);
1178                 size_t length = size * sizeof(struct siw_cqe) +
1179                         sizeof(struct siw_cq_ctrl);
1180
1181                 cq->cq_entry =
1182                         siw_mmap_entry_insert(ctx, cq->queue,
1183                                               length, &uresp.cq_key);
1184                 if (!cq->cq_entry) {
1185                         rv = -ENOMEM;
1186                         goto err_out;
1187                 }
1188
1189                 uresp.cq_id = cq->id;
1190                 uresp.num_cqe = size;
1191
1192                 if (udata->outlen < sizeof(uresp)) {
1193                         rv = -EINVAL;
1194                         goto err_out;
1195                 }
1196                 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1197                 if (rv)
1198                         goto err_out;
1199         }
1200         return 0;
1201
1202 err_out:
1203         siw_dbg(base_cq->device, "CQ creation failed: %d", rv);
1204
1205         if (cq->queue) {
1206                 struct siw_ucontext *ctx =
1207                         rdma_udata_to_drv_context(udata, struct siw_ucontext,
1208                                                   base_ucontext);
1209                 if (ctx)
1210                         rdma_user_mmap_entry_remove(cq->cq_entry);
1211                 vfree(cq->queue);
1212         }
1213         atomic_dec(&sdev->num_cq);
1214
1215         return rv;
1216 }
1217
1218 /*
1219  * siw_poll_cq()
1220  *
1221  * Reap CQ entries if available and copy work completion status into
1222  * array of WC's provided by caller. Returns number of reaped CQE's.
1223  *
1224  * @base_cq:    Base CQ contained in siw CQ.
1225  * @num_cqe:    Maximum number of CQE's to reap.
1226  * @wc:         Array of work completions to be filled by siw.
1227  */
1228 int siw_poll_cq(struct ib_cq *base_cq, int num_cqe, struct ib_wc *wc)
1229 {
1230         struct siw_cq *cq = to_siw_cq(base_cq);
1231         int i;
1232
1233         for (i = 0; i < num_cqe; i++) {
1234                 if (!siw_reap_cqe(cq, wc))
1235                         break;
1236                 wc++;
1237         }
1238         return i;
1239 }
1240
1241 /*
1242  * siw_req_notify_cq()
1243  *
1244  * Request notification for new CQE's added to that CQ.
1245  * Defined flags:
1246  * o SIW_CQ_NOTIFY_SOLICITED lets siw trigger a notification
1247  *   event if a WQE with notification flag set enters the CQ
1248  * o SIW_CQ_NOTIFY_NEXT_COMP lets siw trigger a notification
1249  *   event if a WQE enters the CQ.
1250  * o IB_CQ_REPORT_MISSED_EVENTS: return value will provide the
1251  *   number of not reaped CQE's regardless of its notification
1252  *   type and current or new CQ notification settings.
1253  *
1254  * @base_cq:    Base CQ contained in siw CQ.
1255  * @flags:      Requested notification flags.
1256  */
1257 int siw_req_notify_cq(struct ib_cq *base_cq, enum ib_cq_notify_flags flags)
1258 {
1259         struct siw_cq *cq = to_siw_cq(base_cq);
1260
1261         siw_dbg_cq(cq, "flags: 0x%02x\n", flags);
1262
1263         if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
1264                 /*
1265                  * Enable CQ event for next solicited completion.
1266                  * and make it visible to all associated producers.
1267                  */
1268                 smp_store_mb(cq->notify->flags, SIW_NOTIFY_SOLICITED);
1269         else
1270                 /*
1271                  * Enable CQ event for any signalled completion.
1272                  * and make it visible to all associated producers.
1273                  */
1274                 smp_store_mb(cq->notify->flags, SIW_NOTIFY_ALL);
1275
1276         if (flags & IB_CQ_REPORT_MISSED_EVENTS)
1277                 return cq->cq_put - cq->cq_get;
1278
1279         return 0;
1280 }
1281
1282 /*
1283  * siw_dereg_mr()
1284  *
1285  * Release Memory Region.
1286  *
1287  * @base_mr: Base MR contained in siw MR.
1288  * @udata: points to user context, unused.
1289  */
1290 int siw_dereg_mr(struct ib_mr *base_mr, struct ib_udata *udata)
1291 {
1292         struct siw_mr *mr = to_siw_mr(base_mr);
1293         struct siw_device *sdev = to_siw_dev(base_mr->device);
1294
1295         siw_dbg_mem(mr->mem, "deregister MR\n");
1296
1297         atomic_dec(&sdev->num_mr);
1298
1299         siw_mr_drop_mem(mr);
1300         kfree_rcu(mr, rcu);
1301
1302         return 0;
1303 }
1304
1305 /*
1306  * siw_reg_user_mr()
1307  *
1308  * Register Memory Region.
1309  *
1310  * @pd:         Protection Domain
1311  * @start:      starting address of MR (virtual address)
1312  * @len:        len of MR
1313  * @rnic_va:    not used by siw
1314  * @rights:     MR access rights
1315  * @udata:      user buffer to communicate STag and Key.
1316  */
1317 struct ib_mr *siw_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
1318                               u64 rnic_va, int rights, struct ib_udata *udata)
1319 {
1320         struct siw_mr *mr = NULL;
1321         struct siw_umem *umem = NULL;
1322         struct siw_ureq_reg_mr ureq;
1323         struct siw_device *sdev = to_siw_dev(pd->device);
1324
1325         unsigned long mem_limit = rlimit(RLIMIT_MEMLOCK);
1326         int rv;
1327
1328         siw_dbg_pd(pd, "start: 0x%pK, va: 0x%pK, len: %llu\n",
1329                    (void *)(uintptr_t)start, (void *)(uintptr_t)rnic_va,
1330                    (unsigned long long)len);
1331
1332         if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1333                 siw_dbg_pd(pd, "too many mr's\n");
1334                 rv = -ENOMEM;
1335                 goto err_out;
1336         }
1337         if (!len) {
1338                 rv = -EINVAL;
1339                 goto err_out;
1340         }
1341         if (mem_limit != RLIM_INFINITY) {
1342                 unsigned long num_pages =
1343                         (PAGE_ALIGN(len + (start & ~PAGE_MASK))) >> PAGE_SHIFT;
1344                 mem_limit >>= PAGE_SHIFT;
1345
1346                 if (num_pages > mem_limit - current->mm->locked_vm) {
1347                         siw_dbg_pd(pd, "pages req %lu, max %lu, lock %lu\n",
1348                                    num_pages, mem_limit,
1349                                    current->mm->locked_vm);
1350                         rv = -ENOMEM;
1351                         goto err_out;
1352                 }
1353         }
1354         umem = siw_umem_get(start, len, ib_access_writable(rights));
1355         if (IS_ERR(umem)) {
1356                 rv = PTR_ERR(umem);
1357                 siw_dbg_pd(pd, "getting user memory failed: %d\n", rv);
1358                 umem = NULL;
1359                 goto err_out;
1360         }
1361         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1362         if (!mr) {
1363                 rv = -ENOMEM;
1364                 goto err_out;
1365         }
1366         rv = siw_mr_add_mem(mr, pd, umem, start, len, rights);
1367         if (rv)
1368                 goto err_out;
1369
1370         if (udata) {
1371                 struct siw_uresp_reg_mr uresp = {};
1372                 struct siw_mem *mem = mr->mem;
1373
1374                 if (udata->inlen < sizeof(ureq)) {
1375                         rv = -EINVAL;
1376                         goto err_out;
1377                 }
1378                 rv = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1379                 if (rv)
1380                         goto err_out;
1381
1382                 mr->base_mr.lkey |= ureq.stag_key;
1383                 mr->base_mr.rkey |= ureq.stag_key;
1384                 mem->stag |= ureq.stag_key;
1385                 uresp.stag = mem->stag;
1386
1387                 if (udata->outlen < sizeof(uresp)) {
1388                         rv = -EINVAL;
1389                         goto err_out;
1390                 }
1391                 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1392                 if (rv)
1393                         goto err_out;
1394         }
1395         mr->mem->stag_valid = 1;
1396
1397         return &mr->base_mr;
1398
1399 err_out:
1400         atomic_dec(&sdev->num_mr);
1401         if (mr) {
1402                 if (mr->mem)
1403                         siw_mr_drop_mem(mr);
1404                 kfree_rcu(mr, rcu);
1405         } else {
1406                 if (umem)
1407                         siw_umem_release(umem, false);
1408         }
1409         return ERR_PTR(rv);
1410 }
1411
1412 struct ib_mr *siw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1413                            u32 max_sge)
1414 {
1415         struct siw_device *sdev = to_siw_dev(pd->device);
1416         struct siw_mr *mr = NULL;
1417         struct siw_pbl *pbl = NULL;
1418         int rv;
1419
1420         if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1421                 siw_dbg_pd(pd, "too many mr's\n");
1422                 rv = -ENOMEM;
1423                 goto err_out;
1424         }
1425         if (mr_type != IB_MR_TYPE_MEM_REG) {
1426                 siw_dbg_pd(pd, "mr type %d unsupported\n", mr_type);
1427                 rv = -EOPNOTSUPP;
1428                 goto err_out;
1429         }
1430         if (max_sge > SIW_MAX_SGE_PBL) {
1431                 siw_dbg_pd(pd, "too many sge's: %d\n", max_sge);
1432                 rv = -ENOMEM;
1433                 goto err_out;
1434         }
1435         pbl = siw_pbl_alloc(max_sge);
1436         if (IS_ERR(pbl)) {
1437                 rv = PTR_ERR(pbl);
1438                 siw_dbg_pd(pd, "pbl allocation failed: %d\n", rv);
1439                 pbl = NULL;
1440                 goto err_out;
1441         }
1442         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1443         if (!mr) {
1444                 rv = -ENOMEM;
1445                 goto err_out;
1446         }
1447         rv = siw_mr_add_mem(mr, pd, pbl, 0, max_sge * PAGE_SIZE, 0);
1448         if (rv)
1449                 goto err_out;
1450
1451         mr->mem->is_pbl = 1;
1452
1453         siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1454
1455         return &mr->base_mr;
1456
1457 err_out:
1458         atomic_dec(&sdev->num_mr);
1459
1460         if (!mr) {
1461                 kfree(pbl);
1462         } else {
1463                 if (mr->mem)
1464                         siw_mr_drop_mem(mr);
1465                 kfree_rcu(mr, rcu);
1466         }
1467         siw_dbg_pd(pd, "failed: %d\n", rv);
1468
1469         return ERR_PTR(rv);
1470 }
1471
1472 /* Just used to count number of pages being mapped */
1473 static int siw_set_pbl_page(struct ib_mr *base_mr, u64 buf_addr)
1474 {
1475         return 0;
1476 }
1477
1478 int siw_map_mr_sg(struct ib_mr *base_mr, struct scatterlist *sl, int num_sle,
1479                   unsigned int *sg_off)
1480 {
1481         struct scatterlist *slp;
1482         struct siw_mr *mr = to_siw_mr(base_mr);
1483         struct siw_mem *mem = mr->mem;
1484         struct siw_pbl *pbl = mem->pbl;
1485         struct siw_pble *pble;
1486         unsigned long pbl_size;
1487         int i, rv;
1488
1489         if (!pbl) {
1490                 siw_dbg_mem(mem, "no PBL allocated\n");
1491                 return -EINVAL;
1492         }
1493         pble = pbl->pbe;
1494
1495         if (pbl->max_buf < num_sle) {
1496                 siw_dbg_mem(mem, "too many SGE's: %d > %d\n",
1497                             mem->pbl->max_buf, num_sle);
1498                 return -ENOMEM;
1499         }
1500         for_each_sg(sl, slp, num_sle, i) {
1501                 if (sg_dma_len(slp) == 0) {
1502                         siw_dbg_mem(mem, "empty SGE\n");
1503                         return -EINVAL;
1504                 }
1505                 if (i == 0) {
1506                         pble->addr = sg_dma_address(slp);
1507                         pble->size = sg_dma_len(slp);
1508                         pble->pbl_off = 0;
1509                         pbl_size = pble->size;
1510                         pbl->num_buf = 1;
1511                 } else {
1512                         /* Merge PBL entries if adjacent */
1513                         if (pble->addr + pble->size == sg_dma_address(slp)) {
1514                                 pble->size += sg_dma_len(slp);
1515                         } else {
1516                                 pble++;
1517                                 pbl->num_buf++;
1518                                 pble->addr = sg_dma_address(slp);
1519                                 pble->size = sg_dma_len(slp);
1520                                 pble->pbl_off = pbl_size;
1521                         }
1522                         pbl_size += sg_dma_len(slp);
1523                 }
1524                 siw_dbg_mem(mem,
1525                         "sge[%d], size %u, addr 0x%p, total %lu\n",
1526                         i, pble->size, ib_virt_dma_to_ptr(pble->addr),
1527                         pbl_size);
1528         }
1529         rv = ib_sg_to_pages(base_mr, sl, num_sle, sg_off, siw_set_pbl_page);
1530         if (rv > 0) {
1531                 mem->len = base_mr->length;
1532                 mem->va = base_mr->iova;
1533                 siw_dbg_mem(mem,
1534                         "%llu bytes, start 0x%pK, %u SLE to %u entries\n",
1535                         mem->len, (void *)(uintptr_t)mem->va, num_sle,
1536                         pbl->num_buf);
1537         }
1538         return rv;
1539 }
1540
1541 /*
1542  * siw_get_dma_mr()
1543  *
1544  * Create a (empty) DMA memory region, where no umem is attached.
1545  */
1546 struct ib_mr *siw_get_dma_mr(struct ib_pd *pd, int rights)
1547 {
1548         struct siw_device *sdev = to_siw_dev(pd->device);
1549         struct siw_mr *mr = NULL;
1550         int rv;
1551
1552         if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1553                 siw_dbg_pd(pd, "too many mr's\n");
1554                 rv = -ENOMEM;
1555                 goto err_out;
1556         }
1557         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1558         if (!mr) {
1559                 rv = -ENOMEM;
1560                 goto err_out;
1561         }
1562         rv = siw_mr_add_mem(mr, pd, NULL, 0, ULONG_MAX, rights);
1563         if (rv)
1564                 goto err_out;
1565
1566         mr->mem->stag_valid = 1;
1567
1568         siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1569
1570         return &mr->base_mr;
1571
1572 err_out:
1573         if (rv)
1574                 kfree(mr);
1575
1576         atomic_dec(&sdev->num_mr);
1577
1578         return ERR_PTR(rv);
1579 }
1580
1581 /*
1582  * siw_create_srq()
1583  *
1584  * Create Shared Receive Queue of attributes @init_attrs
1585  * within protection domain given by @pd.
1586  *
1587  * @base_srq:   Base SRQ contained in siw SRQ.
1588  * @init_attrs: SRQ init attributes.
1589  * @udata:      points to user context
1590  */
1591 int siw_create_srq(struct ib_srq *base_srq,
1592                    struct ib_srq_init_attr *init_attrs, struct ib_udata *udata)
1593 {
1594         struct siw_srq *srq = to_siw_srq(base_srq);
1595         struct ib_srq_attr *attrs = &init_attrs->attr;
1596         struct siw_device *sdev = to_siw_dev(base_srq->device);
1597         struct siw_ucontext *ctx =
1598                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1599                                           base_ucontext);
1600         int rv;
1601
1602         if (init_attrs->srq_type != IB_SRQT_BASIC)
1603                 return -EOPNOTSUPP;
1604
1605         if (atomic_inc_return(&sdev->num_srq) > SIW_MAX_SRQ) {
1606                 siw_dbg_pd(base_srq->pd, "too many SRQ's\n");
1607                 rv = -ENOMEM;
1608                 goto err_out;
1609         }
1610         if (attrs->max_wr == 0 || attrs->max_wr > SIW_MAX_SRQ_WR ||
1611             attrs->max_sge > SIW_MAX_SGE || attrs->srq_limit > attrs->max_wr) {
1612                 rv = -EINVAL;
1613                 goto err_out;
1614         }
1615         srq->max_sge = attrs->max_sge;
1616         srq->num_rqe = roundup_pow_of_two(attrs->max_wr);
1617         srq->limit = attrs->srq_limit;
1618         if (srq->limit)
1619                 srq->armed = true;
1620
1621         srq->is_kernel_res = !udata;
1622
1623         if (udata)
1624                 srq->recvq =
1625                         vmalloc_user(srq->num_rqe * sizeof(struct siw_rqe));
1626         else
1627                 srq->recvq = vzalloc(srq->num_rqe * sizeof(struct siw_rqe));
1628
1629         if (srq->recvq == NULL) {
1630                 rv = -ENOMEM;
1631                 goto err_out;
1632         }
1633         if (udata) {
1634                 struct siw_uresp_create_srq uresp = {};
1635                 size_t length = srq->num_rqe * sizeof(struct siw_rqe);
1636
1637                 srq->srq_entry =
1638                         siw_mmap_entry_insert(ctx, srq->recvq,
1639                                               length, &uresp.srq_key);
1640                 if (!srq->srq_entry) {
1641                         rv = -ENOMEM;
1642                         goto err_out;
1643                 }
1644
1645                 uresp.num_rqe = srq->num_rqe;
1646
1647                 if (udata->outlen < sizeof(uresp)) {
1648                         rv = -EINVAL;
1649                         goto err_out;
1650                 }
1651                 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1652                 if (rv)
1653                         goto err_out;
1654         }
1655         spin_lock_init(&srq->lock);
1656
1657         siw_dbg_pd(base_srq->pd, "[SRQ]: success\n");
1658
1659         return 0;
1660
1661 err_out:
1662         if (srq->recvq) {
1663                 if (ctx)
1664                         rdma_user_mmap_entry_remove(srq->srq_entry);
1665                 vfree(srq->recvq);
1666         }
1667         atomic_dec(&sdev->num_srq);
1668
1669         return rv;
1670 }
1671
1672 /*
1673  * siw_modify_srq()
1674  *
1675  * Modify SRQ. The caller may resize SRQ and/or set/reset notification
1676  * limit and (re)arm IB_EVENT_SRQ_LIMIT_REACHED notification.
1677  *
1678  * NOTE: it is unclear if RDMA core allows for changing the MAX_SGE
1679  * parameter. siw_modify_srq() does not check the attrs->max_sge param.
1680  */
1681 int siw_modify_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs,
1682                    enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1683 {
1684         struct siw_srq *srq = to_siw_srq(base_srq);
1685         unsigned long flags;
1686         int rv = 0;
1687
1688         spin_lock_irqsave(&srq->lock, flags);
1689
1690         if (attr_mask & IB_SRQ_MAX_WR) {
1691                 /* resize request not yet supported */
1692                 rv = -EOPNOTSUPP;
1693                 goto out;
1694         }
1695         if (attr_mask & IB_SRQ_LIMIT) {
1696                 if (attrs->srq_limit) {
1697                         if (unlikely(attrs->srq_limit > srq->num_rqe)) {
1698                                 rv = -EINVAL;
1699                                 goto out;
1700                         }
1701                         srq->armed = true;
1702                 } else {
1703                         srq->armed = false;
1704                 }
1705                 srq->limit = attrs->srq_limit;
1706         }
1707 out:
1708         spin_unlock_irqrestore(&srq->lock, flags);
1709
1710         return rv;
1711 }
1712
1713 /*
1714  * siw_query_srq()
1715  *
1716  * Query SRQ attributes.
1717  */
1718 int siw_query_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs)
1719 {
1720         struct siw_srq *srq = to_siw_srq(base_srq);
1721         unsigned long flags;
1722
1723         spin_lock_irqsave(&srq->lock, flags);
1724
1725         attrs->max_wr = srq->num_rqe;
1726         attrs->max_sge = srq->max_sge;
1727         attrs->srq_limit = srq->limit;
1728
1729         spin_unlock_irqrestore(&srq->lock, flags);
1730
1731         return 0;
1732 }
1733
1734 /*
1735  * siw_destroy_srq()
1736  *
1737  * Destroy SRQ.
1738  * It is assumed that the SRQ is not referenced by any
1739  * QP anymore - the code trusts the RDMA core environment to keep track
1740  * of QP references.
1741  */
1742 int siw_destroy_srq(struct ib_srq *base_srq, struct ib_udata *udata)
1743 {
1744         struct siw_srq *srq = to_siw_srq(base_srq);
1745         struct siw_device *sdev = to_siw_dev(base_srq->device);
1746         struct siw_ucontext *ctx =
1747                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1748                                           base_ucontext);
1749
1750         if (ctx)
1751                 rdma_user_mmap_entry_remove(srq->srq_entry);
1752         vfree(srq->recvq);
1753         atomic_dec(&sdev->num_srq);
1754         return 0;
1755 }
1756
1757 /*
1758  * siw_post_srq_recv()
1759  *
1760  * Post a list of receive queue elements to SRQ.
1761  * NOTE: The function does not check or lock a certain SRQ state
1762  *       during the post operation. The code simply trusts the
1763  *       RDMA core environment.
1764  *
1765  * @base_srq:   Base SRQ contained in siw SRQ
1766  * @wr:         List of R-WR's
1767  * @bad_wr:     Updated to failing WR if posting fails.
1768  */
1769 int siw_post_srq_recv(struct ib_srq *base_srq, const struct ib_recv_wr *wr,
1770                       const struct ib_recv_wr **bad_wr)
1771 {
1772         struct siw_srq *srq = to_siw_srq(base_srq);
1773         unsigned long flags;
1774         int rv = 0;
1775
1776         if (unlikely(!srq->is_kernel_res)) {
1777                 siw_dbg_pd(base_srq->pd,
1778                            "[SRQ]: no kernel post_recv for mapped srq\n");
1779                 rv = -EINVAL;
1780                 goto out;
1781         }
1782         /*
1783          * Serialize potentially multiple producers.
1784          * Also needed to serialize potentially multiple
1785          * consumers.
1786          */
1787         spin_lock_irqsave(&srq->lock, flags);
1788
1789         while (wr) {
1790                 u32 idx = srq->rq_put % srq->num_rqe;
1791                 struct siw_rqe *rqe = &srq->recvq[idx];
1792
1793                 if (rqe->flags) {
1794                         siw_dbg_pd(base_srq->pd, "SRQ full\n");
1795                         rv = -ENOMEM;
1796                         break;
1797                 }
1798                 if (unlikely(wr->num_sge > srq->max_sge)) {
1799                         siw_dbg_pd(base_srq->pd,
1800                                    "[SRQ]: too many sge's: %d\n", wr->num_sge);
1801                         rv = -EINVAL;
1802                         break;
1803                 }
1804                 rqe->id = wr->wr_id;
1805                 rqe->num_sge = wr->num_sge;
1806                 siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1807
1808                 /* Make sure S-RQE is completely written before valid */
1809                 smp_wmb();
1810
1811                 rqe->flags = SIW_WQE_VALID;
1812
1813                 srq->rq_put++;
1814                 wr = wr->next;
1815         }
1816         spin_unlock_irqrestore(&srq->lock, flags);
1817 out:
1818         if (unlikely(rv < 0)) {
1819                 siw_dbg_pd(base_srq->pd, "[SRQ]: error %d\n", rv);
1820                 *bad_wr = wr;
1821         }
1822         return rv;
1823 }
1824
1825 void siw_qp_event(struct siw_qp *qp, enum ib_event_type etype)
1826 {
1827         struct ib_event event;
1828         struct ib_qp *base_qp = &qp->base_qp;
1829
1830         /*
1831          * Do not report asynchronous errors on QP which gets
1832          * destroyed via verbs interface (siw_destroy_qp())
1833          */
1834         if (qp->attrs.flags & SIW_QP_IN_DESTROY)
1835                 return;
1836
1837         event.event = etype;
1838         event.device = base_qp->device;
1839         event.element.qp = base_qp;
1840
1841         if (base_qp->event_handler) {
1842                 siw_dbg_qp(qp, "reporting event %d\n", etype);
1843                 base_qp->event_handler(&event, base_qp->qp_context);
1844         }
1845 }
1846
1847 void siw_cq_event(struct siw_cq *cq, enum ib_event_type etype)
1848 {
1849         struct ib_event event;
1850         struct ib_cq *base_cq = &cq->base_cq;
1851
1852         event.event = etype;
1853         event.device = base_cq->device;
1854         event.element.cq = base_cq;
1855
1856         if (base_cq->event_handler) {
1857                 siw_dbg_cq(cq, "reporting CQ event %d\n", etype);
1858                 base_cq->event_handler(&event, base_cq->cq_context);
1859         }
1860 }
1861
1862 void siw_srq_event(struct siw_srq *srq, enum ib_event_type etype)
1863 {
1864         struct ib_event event;
1865         struct ib_srq *base_srq = &srq->base_srq;
1866
1867         event.event = etype;
1868         event.device = base_srq->device;
1869         event.element.srq = base_srq;
1870
1871         if (base_srq->event_handler) {
1872                 siw_dbg_pd(srq->base_srq.pd,
1873                            "reporting SRQ event %d\n", etype);
1874                 base_srq->event_handler(&event, base_srq->srq_context);
1875         }
1876 }
1877
1878 void siw_port_event(struct siw_device *sdev, u32 port, enum ib_event_type etype)
1879 {
1880         struct ib_event event;
1881
1882         event.event = etype;
1883         event.device = &sdev->base_dev;
1884         event.element.port_num = port;
1885
1886         siw_dbg(&sdev->base_dev, "reporting port event %d\n", etype);
1887
1888         ib_dispatch_event(&event);
1889 }