drivers/infiniband/hw/amso1100/c2_qp.c

   1 /*
   2  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
   3  * Copyright (c) 2005 Cisco Systems. All rights reserved.
   4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
   5  * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
   6  * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
   7  *
   8  * This software is available to you under a choice of one of two
   9  * licenses.  You may choose to be licensed under the terms of the GNU
  10  * General Public License (GPL) Version 2, available from the file
  11  * COPYING in the main directory of this source tree, or the
  12  * OpenIB.org BSD license below:
  13  *
  14  *     Redistribution and use in source and binary forms, with or
  15  *     without modification, are permitted provided that the following
  16  *     conditions are met:
  17  *
  18  *      - Redistributions of source code must retain the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer.
  21  *
  22  *      - Redistributions in binary form must reproduce the above
  23  *        copyright notice, this list of conditions and the following
  24  *        disclaimer in the documentation and/or other materials
  25  *        provided with the distribution.
  26  *
  27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  34  * SOFTWARE.
  35  *
  36  */
  37
  38 #include <linux/delay.h>
  39
  40 #include "c2.h"
  41 #include "c2_vq.h"
  42 #include "c2_status.h"
  43
  44 #define C2_MAX_ORD_PER_QP 128
  45 #define C2_MAX_IRD_PER_QP 128
  46
  47 #define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count)
  48 #define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16)
  49 #define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF)
  50
  51 #define NO_SUPPORT -1
  52 static const u8 c2_opcode[] = {
  53         [IB_WR_SEND] = C2_WR_TYPE_SEND,
  54         [IB_WR_SEND_WITH_IMM] = NO_SUPPORT,
  55         [IB_WR_RDMA_WRITE] = C2_WR_TYPE_RDMA_WRITE,
  56         [IB_WR_RDMA_WRITE_WITH_IMM] = NO_SUPPORT,
  57         [IB_WR_RDMA_READ] = C2_WR_TYPE_RDMA_READ,
  58         [IB_WR_ATOMIC_CMP_AND_SWP] = NO_SUPPORT,
  59         [IB_WR_ATOMIC_FETCH_AND_ADD] = NO_SUPPORT,
  60 };
  61
  62 static int to_c2_state(enum ib_qp_state ib_state)
  63 {
  64         switch (ib_state) {
  65         case IB_QPS_RESET:
  66                 return C2_QP_STATE_IDLE;
  67         case IB_QPS_RTS:
  68                 return C2_QP_STATE_RTS;
  69         case IB_QPS_SQD:
  70                 return C2_QP_STATE_CLOSING;
  71         case IB_QPS_SQE:
  72                 return C2_QP_STATE_CLOSING;
  73         case IB_QPS_ERR:
  74                 return C2_QP_STATE_ERROR;
  75         default:
  76                 return -1;
  77         }
  78 }
  79
  80 static int to_ib_state(enum c2_qp_state c2_state)
  81 {
  82         switch (c2_state) {
  83         case C2_QP_STATE_IDLE:
  84                 return IB_QPS_RESET;
  85         case C2_QP_STATE_CONNECTING:
  86                 return IB_QPS_RTR;
  87         case C2_QP_STATE_RTS:
  88                 return IB_QPS_RTS;
  89         case C2_QP_STATE_CLOSING:
  90                 return IB_QPS_SQD;
  91         case C2_QP_STATE_ERROR:
  92                 return IB_QPS_ERR;
  93         case C2_QP_STATE_TERMINATE:
  94                 return IB_QPS_SQE;
  95         default:
  96                 return -1;
  97         }
  98 }
  99
 100 static const char *to_ib_state_str(int ib_state)
 101 {
 102         static const char *state_str[] = {
 103                 "IB_QPS_RESET",
 104                 "IB_QPS_INIT",
 105                 "IB_QPS_RTR",
 106                 "IB_QPS_RTS",
 107                 "IB_QPS_SQD",
 108                 "IB_QPS_SQE",
 109                 "IB_QPS_ERR"
 110         };
 111         if (ib_state < IB_QPS_RESET ||
 112             ib_state > IB_QPS_ERR)
 113                 return "<invalid IB QP state>";
 114
 115         ib_state -= IB_QPS_RESET;
 116         return state_str[ib_state];
 117 }
 118
 119 void c2_set_qp_state(struct c2_qp *qp, int c2_state)
 120 {
 121         int new_state = to_ib_state(c2_state);
 122
 123         pr_debug("%s: qp[%p] state modify %s --> %s\n",
 124                __func__,
 125                 qp,
 126                 to_ib_state_str(qp->state),
 127                 to_ib_state_str(new_state));
 128         qp->state = new_state;
 129 }
 130
 131 #define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF
 132
 133 int c2_qp_modify(struct c2_dev *c2dev, struct c2_qp *qp,
 134                  struct ib_qp_attr *attr, int attr_mask)
 135 {
 136         struct c2wr_qp_modify_req wr;
 137         struct c2wr_qp_modify_rep *reply;
 138         struct c2_vq_req *vq_req;
 139         unsigned long flags;
 140         u8 next_state;
 141         int err;
 142
 143         pr_debug("%s:%d qp=%p, %s --> %s\n",
 144                 __func__, __LINE__,
 145                 qp,
 146                 to_ib_state_str(qp->state),
 147                 to_ib_state_str(attr->qp_state));
 148
 149         vq_req = vq_req_alloc(c2dev);
 150         if (!vq_req)
 151                 return -ENOMEM;
 152
 153         c2_wr_set_id(&wr, CCWR_QP_MODIFY);
 154         wr.hdr.context = (unsigned long) vq_req;
 155         wr.rnic_handle = c2dev->adapter_handle;
 156         wr.qp_handle = qp->adapter_handle;
 157         wr.ord = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 158         wr.ird = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 159         wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 160         wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 161
 162         if (attr_mask & IB_QP_STATE) {
 163                 /* Ensure the state is valid */
 164                 if (attr->qp_state < 0 || attr->qp_state > IB_QPS_ERR) {
 165                         err = -EINVAL;
 166                         goto bail0;
 167                 }
 168
 169                 wr.next_qp_state = cpu_to_be32(to_c2_state(attr->qp_state));
 170
 171                 if (attr->qp_state == IB_QPS_ERR) {
 172                         spin_lock_irqsave(&qp->lock, flags);
 173                         if (qp->cm_id && qp->state == IB_QPS_RTS) {
 174                                 pr_debug("Generating CLOSE event for QP-->ERR, "
 175                                         "qp=%p, cm_id=%p\n",qp,qp->cm_id);
 176                                 /* Generate an CLOSE event */
 177                                 vq_req->cm_id = qp->cm_id;
 178                                 vq_req->event = IW_CM_EVENT_CLOSE;
 179                         }
 180                         spin_unlock_irqrestore(&qp->lock, flags);
 181                 }
 182                 next_state =  attr->qp_state;
 183
 184         } else if (attr_mask & IB_QP_CUR_STATE) {
 185
 186                 if (attr->cur_qp_state != IB_QPS_RTR &&
 187                     attr->cur_qp_state != IB_QPS_RTS &&
 188                     attr->cur_qp_state != IB_QPS_SQD &&
 189                     attr->cur_qp_state != IB_QPS_SQE) {
 190                         err = -EINVAL;
 191                         goto bail0;
 192                 } else
 193                         wr.next_qp_state =
 194                             cpu_to_be32(to_c2_state(attr->cur_qp_state));
 195
 196                 next_state = attr->cur_qp_state;
 197
 198         } else {
 199                 err = 0;
 200                 goto bail0;
 201         }
 202
 203         /* reference the request struct */
 204         vq_req_get(c2dev, vq_req);
 205
 206         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 207         if (err) {
 208                 vq_req_put(c2dev, vq_req);
 209                 goto bail0;
 210         }
 211
 212         err = vq_wait_for_reply(c2dev, vq_req);
 213         if (err)
 214                 goto bail0;
 215
 216         reply = (struct c2wr_qp_modify_rep *) (unsigned long) vq_req->reply_msg;
 217         if (!reply) {
 218                 err = -ENOMEM;
 219                 goto bail0;
 220         }
 221
 222         err = c2_errno(reply);
 223         if (!err)
 224                 qp->state = next_state;
 225 #ifdef DEBUG
 226         else
 227                 pr_debug("%s: c2_errno=%d\n", __func__, err);
 228 #endif
 229         /*
 230          * If we're going to error and generating the event here, then
 231          * we need to remove the reference because there will be no
 232          * close event generated by the adapter
 233         */
 234         spin_lock_irqsave(&qp->lock, flags);
 235         if (vq_req->event==IW_CM_EVENT_CLOSE && qp->cm_id) {
 236                 qp->cm_id->rem_ref(qp->cm_id);
 237                 qp->cm_id = NULL;
 238         }
 239         spin_unlock_irqrestore(&qp->lock, flags);
 240
 241         vq_repbuf_free(c2dev, reply);
 242       bail0:
 243         vq_req_free(c2dev, vq_req);
 244
 245         pr_debug("%s:%d qp=%p, cur_state=%s\n",
 246                 __func__, __LINE__,
 247                 qp,
 248                 to_ib_state_str(qp->state));
 249         return err;
 250 }
 251
 252 int c2_qp_set_read_limits(struct c2_dev *c2dev, struct c2_qp *qp,
 253                           int ord, int ird)
 254 {
 255         struct c2wr_qp_modify_req wr;
 256         struct c2wr_qp_modify_rep *reply;
 257         struct c2_vq_req *vq_req;
 258         int err;
 259
 260         vq_req = vq_req_alloc(c2dev);
 261         if (!vq_req)
 262                 return -ENOMEM;
 263
 264         c2_wr_set_id(&wr, CCWR_QP_MODIFY);
 265         wr.hdr.context = (unsigned long) vq_req;
 266         wr.rnic_handle = c2dev->adapter_handle;
 267         wr.qp_handle = qp->adapter_handle;
 268         wr.ord = cpu_to_be32(ord);
 269         wr.ird = cpu_to_be32(ird);
 270         wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 271         wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 272         wr.next_qp_state = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 273
 274         /* reference the request struct */
 275         vq_req_get(c2dev, vq_req);
 276
 277         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 278         if (err) {
 279                 vq_req_put(c2dev, vq_req);
 280                 goto bail0;
 281         }
 282
 283         err = vq_wait_for_reply(c2dev, vq_req);
 284         if (err)
 285                 goto bail0;
 286
 287         reply = (struct c2wr_qp_modify_rep *) (unsigned long)
 288                 vq_req->reply_msg;
 289         if (!reply) {
 290                 err = -ENOMEM;
 291                 goto bail0;
 292         }
 293
 294         err = c2_errno(reply);
 295         vq_repbuf_free(c2dev, reply);
 296       bail0:
 297         vq_req_free(c2dev, vq_req);
 298         return err;
 299 }
 300
 301 static int destroy_qp(struct c2_dev *c2dev, struct c2_qp *qp)
 302 {
 303         struct c2_vq_req *vq_req;
 304         struct c2wr_qp_destroy_req wr;
 305         struct c2wr_qp_destroy_rep *reply;
 306         unsigned long flags;
 307         int err;
 308
 309         /*
 310          * Allocate a verb request message
 311          */
 312         vq_req = vq_req_alloc(c2dev);
 313         if (!vq_req) {
 314                 return -ENOMEM;
 315         }
 316
 317         /*
 318          * Initialize the WR
 319          */
 320         c2_wr_set_id(&wr, CCWR_QP_DESTROY);
 321         wr.hdr.context = (unsigned long) vq_req;
 322         wr.rnic_handle = c2dev->adapter_handle;
 323         wr.qp_handle = qp->adapter_handle;
 324
 325         /*
 326          * reference the request struct.  dereferenced in the int handler.
 327          */
 328         vq_req_get(c2dev, vq_req);
 329
 330         spin_lock_irqsave(&qp->lock, flags);
 331         if (qp->cm_id && qp->state == IB_QPS_RTS) {
 332                 pr_debug("destroy_qp: generating CLOSE event for QP-->ERR, "
 333                         "qp=%p, cm_id=%p\n",qp,qp->cm_id);
 334                 /* Generate an CLOSE event */
 335                 vq_req->qp = qp;
 336                 vq_req->cm_id = qp->cm_id;
 337                 vq_req->event = IW_CM_EVENT_CLOSE;
 338         }
 339         spin_unlock_irqrestore(&qp->lock, flags);
 340
 341         /*
 342          * Send WR to adapter
 343          */
 344         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 345         if (err) {
 346                 vq_req_put(c2dev, vq_req);
 347                 goto bail0;
 348         }
 349
 350         /*
 351          * Wait for reply from adapter
 352          */
 353         err = vq_wait_for_reply(c2dev, vq_req);
 354         if (err) {
 355                 goto bail0;
 356         }
 357
 358         /*
 359          * Process reply
 360          */
 361         reply = (struct c2wr_qp_destroy_rep *) (unsigned long) (vq_req->reply_msg);
 362         if (!reply) {
 363                 err = -ENOMEM;
 364                 goto bail0;
 365         }
 366
 367         spin_lock_irqsave(&qp->lock, flags);
 368         if (qp->cm_id) {
 369                 qp->cm_id->rem_ref(qp->cm_id);
 370                 qp->cm_id = NULL;
 371         }
 372         spin_unlock_irqrestore(&qp->lock, flags);
 373
 374         vq_repbuf_free(c2dev, reply);
 375       bail0:
 376         vq_req_free(c2dev, vq_req);
 377         return err;
 378 }
 379
 380 static int c2_alloc_qpn(struct c2_dev *c2dev, struct c2_qp *qp)
 381 {
 382         int ret;
 383
 384         do {
 385                 spin_lock_irq(&c2dev->qp_table.lock);
 386                 ret = idr_get_new_above(&c2dev->qp_table.idr, qp,
 387                                         c2dev->qp_table.last++, &qp->qpn);
 388                 spin_unlock_irq(&c2dev->qp_table.lock);
 389         } while ((ret == -EAGAIN) &&
 390                  idr_pre_get(&c2dev->qp_table.idr, GFP_KERNEL));
 391         return ret;
 392 }
 393
 394 static void c2_free_qpn(struct c2_dev *c2dev, int qpn)
 395 {
 396         spin_lock_irq(&c2dev->qp_table.lock);
 397         idr_remove(&c2dev->qp_table.idr, qpn);
 398         spin_unlock_irq(&c2dev->qp_table.lock);
 399 }
 400
 401 struct c2_qp *c2_find_qpn(struct c2_dev *c2dev, int qpn)
 402 {
 403         unsigned long flags;
 404         struct c2_qp *qp;
 405
 406         spin_lock_irqsave(&c2dev->qp_table.lock, flags);
 407         qp = idr_find(&c2dev->qp_table.idr, qpn);
 408         spin_unlock_irqrestore(&c2dev->qp_table.lock, flags);
 409         return qp;
 410 }
 411
 412 int c2_alloc_qp(struct c2_dev *c2dev,
 413                 struct c2_pd *pd,
 414                 struct ib_qp_init_attr *qp_attrs, struct c2_qp *qp)
 415 {
 416         struct c2wr_qp_create_req wr;
 417         struct c2wr_qp_create_rep *reply;
 418         struct c2_vq_req *vq_req;
 419         struct c2_cq *send_cq = to_c2cq(qp_attrs->send_cq);
 420         struct c2_cq *recv_cq = to_c2cq(qp_attrs->recv_cq);
 421         unsigned long peer_pa;
 422         u32 q_size, msg_size, mmap_size;
 423         void __iomem *mmap;
 424         int err;
 425
 426         err = c2_alloc_qpn(c2dev, qp);
 427         if (err)
 428                 return err;
 429         qp->ibqp.qp_num = qp->qpn;
 430         qp->ibqp.qp_type = IB_QPT_RC;
 431
 432         /* Allocate the SQ and RQ shared pointers */
 433         qp->sq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
 434                                          &qp->sq_mq.shared_dma, GFP_KERNEL);
 435         if (!qp->sq_mq.shared) {
 436                 err = -ENOMEM;
 437                 goto bail0;
 438         }
 439
 440         qp->rq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
 441                                          &qp->rq_mq.shared_dma, GFP_KERNEL);
 442         if (!qp->rq_mq.shared) {
 443                 err = -ENOMEM;
 444                 goto bail1;
 445         }
 446
 447         /* Allocate the verbs request */
 448         vq_req = vq_req_alloc(c2dev);
 449         if (vq_req == NULL) {
 450                 err = -ENOMEM;
 451                 goto bail2;
 452         }
 453
 454         /* Initialize the work request */
 455         memset(&wr, 0, sizeof(wr));
 456         c2_wr_set_id(&wr, CCWR_QP_CREATE);
 457         wr.hdr.context = (unsigned long) vq_req;
 458         wr.rnic_handle = c2dev->adapter_handle;
 459         wr.sq_cq_handle = send_cq->adapter_handle;
 460         wr.rq_cq_handle = recv_cq->adapter_handle;
 461         wr.sq_depth = cpu_to_be32(qp_attrs->cap.max_send_wr + 1);
 462         wr.rq_depth = cpu_to_be32(qp_attrs->cap.max_recv_wr + 1);
 463         wr.srq_handle = 0;
 464         wr.flags = cpu_to_be32(QP_RDMA_READ | QP_RDMA_WRITE | QP_MW_BIND |
 465                                QP_ZERO_STAG | QP_RDMA_READ_RESPONSE);
 466         wr.send_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
 467         wr.recv_sgl_depth = cpu_to_be32(qp_attrs->cap.max_recv_sge);
 468         wr.rdma_write_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
 469         wr.shared_sq_ht = cpu_to_be64(qp->sq_mq.shared_dma);
 470         wr.shared_rq_ht = cpu_to_be64(qp->rq_mq.shared_dma);
 471         wr.ord = cpu_to_be32(C2_MAX_ORD_PER_QP);
 472         wr.ird = cpu_to_be32(C2_MAX_IRD_PER_QP);
 473         wr.pd_id = pd->pd_id;
 474         wr.user_context = (unsigned long) qp;
 475
 476         vq_req_get(c2dev, vq_req);
 477
 478         /* Send the WR to the adapter */
 479         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 480         if (err) {
 481                 vq_req_put(c2dev, vq_req);
 482                 goto bail3;
 483         }
 484
 485         /* Wait for the verb reply  */
 486         err = vq_wait_for_reply(c2dev, vq_req);
 487         if (err) {
 488                 goto bail3;
 489         }
 490
 491         /* Process the reply */
 492         reply = (struct c2wr_qp_create_rep *) (unsigned long) (vq_req->reply_msg);
 493         if (!reply) {
 494                 err = -ENOMEM;
 495                 goto bail3;
 496         }
 497
 498         if ((err = c2_wr_get_result(reply)) != 0) {
 499                 goto bail4;
 500         }
 501
 502         /* Fill in the kernel QP struct */
 503         atomic_set(&qp->refcount, 1);
 504         qp->adapter_handle = reply->qp_handle;
 505         qp->state = IB_QPS_RESET;
 506         qp->send_sgl_depth = qp_attrs->cap.max_send_sge;
 507         qp->rdma_write_sgl_depth = qp_attrs->cap.max_send_sge;
 508         qp->recv_sgl_depth = qp_attrs->cap.max_recv_sge;
 509         init_waitqueue_head(&qp->wait);
 510
 511         /* Initialize the SQ MQ */
 512         q_size = be32_to_cpu(reply->sq_depth);
 513         msg_size = be32_to_cpu(reply->sq_msg_size);
 514         peer_pa = c2dev->pa + be32_to_cpu(reply->sq_mq_start);
 515         mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
 516         mmap = ioremap_nocache(peer_pa, mmap_size);
 517         if (!mmap) {
 518                 err = -ENOMEM;
 519                 goto bail5;
 520         }
 521
 522         c2_mq_req_init(&qp->sq_mq,
 523                        be32_to_cpu(reply->sq_mq_index),
 524                        q_size,
 525                        msg_size,
 526                        mmap + sizeof(struct c2_mq_shared),      /* pool start */
 527                        mmap,                            /* peer */
 528                        C2_MQ_ADAPTER_TARGET);
 529
 530         /* Initialize the RQ mq */
 531         q_size = be32_to_cpu(reply->rq_depth);
 532         msg_size = be32_to_cpu(reply->rq_msg_size);
 533         peer_pa = c2dev->pa + be32_to_cpu(reply->rq_mq_start);
 534         mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
 535         mmap = ioremap_nocache(peer_pa, mmap_size);
 536         if (!mmap) {
 537                 err = -ENOMEM;
 538                 goto bail6;
 539         }
 540
 541         c2_mq_req_init(&qp->rq_mq,
 542                        be32_to_cpu(reply->rq_mq_index),
 543                        q_size,
 544                        msg_size,
 545                        mmap + sizeof(struct c2_mq_shared),      /* pool start */
 546                        mmap,                            /* peer */
 547                        C2_MQ_ADAPTER_TARGET);
 548
 549         vq_repbuf_free(c2dev, reply);
 550         vq_req_free(c2dev, vq_req);
 551
 552         return 0;
 553
 554       bail6:
 555         iounmap(qp->sq_mq.peer);
 556       bail5:
 557         destroy_qp(c2dev, qp);
 558       bail4:
 559         vq_repbuf_free(c2dev, reply);
 560       bail3:
 561         vq_req_free(c2dev, vq_req);
 562       bail2:
 563         c2_free_mqsp(qp->rq_mq.shared);
 564       bail1:
 565         c2_free_mqsp(qp->sq_mq.shared);
 566       bail0:
 567         c2_free_qpn(c2dev, qp->qpn);
 568         return err;
 569 }
 570
 571 static inline void c2_lock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
 572 {
 573         if (send_cq == recv_cq)
 574                 spin_lock_irq(&send_cq->lock);
 575         else if (send_cq > recv_cq) {
 576                 spin_lock_irq(&send_cq->lock);
 577                 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
 578         } else {
 579                 spin_lock_irq(&recv_cq->lock);
 580                 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
 581         }
 582 }
 583
 584 static inline void c2_unlock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
 585 {
 586         if (send_cq == recv_cq)
 587                 spin_unlock_irq(&send_cq->lock);
 588         else if (send_cq > recv_cq) {
 589                 spin_unlock(&recv_cq->lock);
 590                 spin_unlock_irq(&send_cq->lock);
 591         } else {
 592                 spin_unlock(&send_cq->lock);
 593                 spin_unlock_irq(&recv_cq->lock);
 594         }
 595 }
 596
 597 void c2_free_qp(struct c2_dev *c2dev, struct c2_qp *qp)
 598 {
 599         struct c2_cq *send_cq;
 600         struct c2_cq *recv_cq;
 601
 602         send_cq = to_c2cq(qp->ibqp.send_cq);
 603         recv_cq = to_c2cq(qp->ibqp.recv_cq);
 604
 605         /*
 606          * Lock CQs here, so that CQ polling code can do QP lookup
 607          * without taking a lock.
 608          */
 609         c2_lock_cqs(send_cq, recv_cq);
 610         c2_free_qpn(c2dev, qp->qpn);
 611         c2_unlock_cqs(send_cq, recv_cq);
 612
 613         /*
 614          * Destory qp in the rnic...
 615          */
 616         destroy_qp(c2dev, qp);
 617
 618         /*
 619          * Mark any unreaped CQEs as null and void.
 620          */
 621         c2_cq_clean(c2dev, qp, send_cq->cqn);
 622         if (send_cq != recv_cq)
 623                 c2_cq_clean(c2dev, qp, recv_cq->cqn);
 624         /*
 625          * Unmap the MQs and return the shared pointers
 626          * to the message pool.
 627          */
 628         iounmap(qp->sq_mq.peer);
 629         iounmap(qp->rq_mq.peer);
 630         c2_free_mqsp(qp->sq_mq.shared);
 631         c2_free_mqsp(qp->rq_mq.shared);
 632
 633         atomic_dec(&qp->refcount);
 634         wait_event(qp->wait, !atomic_read(&qp->refcount));
 635 }
 636
 637 /*
 638  * Function: move_sgl
 639  *
 640  * Description:
 641  * Move an SGL from the user's work request struct into a CCIL Work Request
 642  * message, swapping to WR byte order and ensure the total length doesn't
 643  * overflow.
 644  *
 645  * IN:
 646  * dst          - ptr to CCIL Work Request message SGL memory.
 647  * src          - ptr to the consumers SGL memory.
 648  *
 649  * OUT: none
 650  *
 651  * Return:
 652  * CCIL status codes.
 653  */
 654 static int
 655 move_sgl(struct c2_data_addr * dst, struct ib_sge *src, int count, u32 * p_len,
 656          u8 * actual_count)
 657 {
 658         u32 tot = 0;            /* running total */
 659         u8 acount = 0;          /* running total non-0 len sge's */
 660
 661         while (count > 0) {
 662                 /*
 663                  * If the addition of this SGE causes the
 664                  * total SGL length to exceed 2^32-1, then
 665                  * fail-n-bail.
 666                  *
 667                  * If the current total plus the next element length
 668                  * wraps, then it will go negative and be less than the
 669                  * current total...
 670                  */
 671                 if ((tot + src->length) < tot) {
 672                         return -EINVAL;
 673                 }
 674                 /*
 675                  * Bug: 1456 (as well as 1498 & 1643)
 676                  * Skip over any sge's supplied with len=0
 677                  */
 678                 if (src->length) {
 679                         tot += src->length;
 680                         dst->stag = cpu_to_be32(src->lkey);
 681                         dst->to = cpu_to_be64(src->addr);
 682                         dst->length = cpu_to_be32(src->length);
 683                         dst++;
 684                         acount++;
 685                 }
 686                 src++;
 687                 count--;
 688         }
 689
 690         if (acount == 0) {
 691                 /*
 692                  * Bug: 1476 (as well as 1498, 1456 and 1643)
 693                  * Setup the SGL in the WR to make it easier for the RNIC.
 694                  * This way, the FW doesn't have to deal with special cases.
 695                  * Setting length=0 should be sufficient.
 696                  */
 697                 dst->stag = 0;
 698                 dst->to = 0;
 699                 dst->length = 0;
 700         }
 701
 702         *p_len = tot;
 703         *actual_count = acount;
 704         return 0;
 705 }
 706
 707 /*
 708  * Function: c2_activity (private function)
 709  *
 710  * Description:
 711  * Post an mq index to the host->adapter activity fifo.
 712  *
 713  * IN:
 714  * c2dev        - ptr to c2dev structure
 715  * mq_index     - mq index to post
 716  * shared       - value most recently written to shared
 717  *
 718  * OUT:
 719  *
 720  * Return:
 721  * none
 722  */
 723 static inline void c2_activity(struct c2_dev *c2dev, u32 mq_index, u16 shared)
 724 {
 725         /*
 726          * First read the register to see if the FIFO is full, and if so,
 727          * spin until it's not.  This isn't perfect -- there is no
 728          * synchronization among the clients of the register, but in
 729          * practice it prevents multiple CPU from hammering the bus
 730          * with PCI RETRY. Note that when this does happen, the card
 731          * cannot get on the bus and the card and system hang in a
 732          * deadlock -- thus the need for this code. [TOT]
 733          */
 734         while (readl(c2dev->regs + PCI_BAR0_ADAPTER_HINT) & 0x80000000)
 735                 udelay(10);
 736
 737         __raw_writel(C2_HINT_MAKE(mq_index, shared),
 738                      c2dev->regs + PCI_BAR0_ADAPTER_HINT);
 739 }
 740
 741 /*
 742  * Function: qp_wr_post
 743  *
 744  * Description:
 745  * This in-line function allocates a MQ msg, then moves the host-copy of
 746  * the completed WR into msg.  Then it posts the message.
 747  *
 748  * IN:
 749  * q            - ptr to user MQ.
 750  * wr           - ptr to host-copy of the WR.
 751  * qp           - ptr to user qp
 752  * size         - Number of bytes to post.  Assumed to be divisible by 4.
 753  *
 754  * OUT: none
 755  *
 756  * Return:
 757  * CCIL status codes.
 758  */
 759 static int qp_wr_post(struct c2_mq *q, union c2wr * wr, struct c2_qp *qp, u32 size)
 760 {
 761         union c2wr *msg;
 762
 763         msg = c2_mq_alloc(q);
 764         if (msg == NULL) {
 765                 return -EINVAL;
 766         }
 767 #ifdef CCMSGMAGIC
 768         ((c2wr_hdr_t *) wr)->magic = cpu_to_be32(CCWR_MAGIC);
 769 #endif
 770
 771         /*
 772          * Since all header fields in the WR are the same as the
 773          * CQE, set the following so the adapter need not.
 774          */
 775         c2_wr_set_result(wr, CCERR_PENDING);
 776
 777         /*
 778          * Copy the wr down to the adapter
 779          */
 780         memcpy((void *) msg, (void *) wr, size);
 781
 782         c2_mq_produce(q);
 783         return 0;
 784 }
 785
 786
 787 int c2_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
 788                  struct ib_send_wr **bad_wr)
 789 {
 790         struct c2_dev *c2dev = to_c2dev(ibqp->device);
 791         struct c2_qp *qp = to_c2qp(ibqp);
 792         union c2wr wr;
 793         unsigned long lock_flags;
 794         int err = 0;
 795
 796         u32 flags;
 797         u32 tot_len;
 798         u8 actual_sge_count;
 799         u32 msg_size;
 800
 801         if (qp->state > IB_QPS_RTS)
 802                 return -EINVAL;
 803
 804         while (ib_wr) {
 805
 806                 flags = 0;
 807                 wr.sqwr.sq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
 808                 if (ib_wr->send_flags & IB_SEND_SIGNALED) {
 809                         flags |= SQ_SIGNALED;
 810                 }
 811
 812                 switch (ib_wr->opcode) {
 813                 case IB_WR_SEND:
 814                 case IB_WR_SEND_WITH_INV:
 815                         if (ib_wr->opcode == IB_WR_SEND) {
 816                                 if (ib_wr->send_flags & IB_SEND_SOLICITED)
 817                                         c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE);
 818                                 else
 819                                         c2_wr_set_id(&wr, C2_WR_TYPE_SEND);
 820                                 wr.sqwr.send.remote_stag = 0;
 821                         } else {
 822                                 if (ib_wr->send_flags & IB_SEND_SOLICITED)
 823                                         c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE_INV);
 824                                 else
 825                                         c2_wr_set_id(&wr, C2_WR_TYPE_SEND_INV);
 826                                 wr.sqwr.send.remote_stag =
 827                                         cpu_to_be32(ib_wr->ex.invalidate_rkey);
 828                         }
 829
 830                         msg_size = sizeof(struct c2wr_send_req) +
 831                                 sizeof(struct c2_data_addr) * ib_wr->num_sge;
 832                         if (ib_wr->num_sge > qp->send_sgl_depth) {
 833                                 err = -EINVAL;
 834                                 break;
 835                         }
 836                         if (ib_wr->send_flags & IB_SEND_FENCE) {
 837                                 flags |= SQ_READ_FENCE;
 838                         }
 839                         err = move_sgl((struct c2_data_addr *) & (wr.sqwr.send.data),
 840                                        ib_wr->sg_list,
 841                                        ib_wr->num_sge,
 842                                        &tot_len, &actual_sge_count);
 843                         wr.sqwr.send.sge_len = cpu_to_be32(tot_len);
 844                         c2_wr_set_sge_count(&wr, actual_sge_count);
 845                         break;
 846                 case IB_WR_RDMA_WRITE:
 847                         c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_WRITE);
 848                         msg_size = sizeof(struct c2wr_rdma_write_req) +
 849                             (sizeof(struct c2_data_addr) * ib_wr->num_sge);
 850                         if (ib_wr->num_sge > qp->rdma_write_sgl_depth) {
 851                                 err = -EINVAL;
 852                                 break;
 853                         }
 854                         if (ib_wr->send_flags & IB_SEND_FENCE) {
 855                                 flags |= SQ_READ_FENCE;
 856                         }
 857                         wr.sqwr.rdma_write.remote_stag =
 858                             cpu_to_be32(ib_wr->wr.rdma.rkey);
 859                         wr.sqwr.rdma_write.remote_to =
 860                             cpu_to_be64(ib_wr->wr.rdma.remote_addr);
 861                         err = move_sgl((struct c2_data_addr *)
 862                                        & (wr.sqwr.rdma_write.data),
 863                                        ib_wr->sg_list,
 864                                        ib_wr->num_sge,
 865                                        &tot_len, &actual_sge_count);
 866                         wr.sqwr.rdma_write.sge_len = cpu_to_be32(tot_len);
 867                         c2_wr_set_sge_count(&wr, actual_sge_count);
 868                         break;
 869                 case IB_WR_RDMA_READ:
 870                         c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_READ);
 871                         msg_size = sizeof(struct c2wr_rdma_read_req);
 872
 873                         /* IWarp only suppots 1 sge for RDMA reads */
 874                         if (ib_wr->num_sge > 1) {
 875                                 err = -EINVAL;
 876                                 break;
 877                         }
 878
 879                         /*
 880                          * Move the local and remote stag/to/len into the WR.
 881                          */
 882                         wr.sqwr.rdma_read.local_stag =
 883                             cpu_to_be32(ib_wr->sg_list->lkey);
 884                         wr.sqwr.rdma_read.local_to =
 885                             cpu_to_be64(ib_wr->sg_list->addr);
 886                         wr.sqwr.rdma_read.remote_stag =
 887                             cpu_to_be32(ib_wr->wr.rdma.rkey);
 888                         wr.sqwr.rdma_read.remote_to =
 889                             cpu_to_be64(ib_wr->wr.rdma.remote_addr);
 890                         wr.sqwr.rdma_read.length =
 891                             cpu_to_be32(ib_wr->sg_list->length);
 892                         break;
 893                 default:
 894                         /* error */
 895                         msg_size = 0;
 896                         err = -EINVAL;
 897                         break;
 898                 }
 899
 900                 /*
 901                  * If we had an error on the last wr build, then
 902                  * break out.  Possible errors include bogus WR
 903                  * type, and a bogus SGL length...
 904                  */
 905                 if (err) {
 906                         break;
 907                 }
 908
 909                 /*
 910                  * Store flags
 911                  */
 912                 c2_wr_set_flags(&wr, flags);
 913
 914                 /*
 915                  * Post the puppy!
 916                  */
 917                 spin_lock_irqsave(&qp->lock, lock_flags);
 918                 err = qp_wr_post(&qp->sq_mq, &wr, qp, msg_size);
 919                 if (err) {
 920                         spin_unlock_irqrestore(&qp->lock, lock_flags);
 921                         break;
 922                 }
 923
 924                 /*
 925                  * Enqueue mq index to activity FIFO.
 926                  */
 927                 c2_activity(c2dev, qp->sq_mq.index, qp->sq_mq.hint_count);
 928                 spin_unlock_irqrestore(&qp->lock, lock_flags);
 929
 930                 ib_wr = ib_wr->next;
 931         }
 932
 933         if (err)
 934                 *bad_wr = ib_wr;
 935         return err;
 936 }
 937
 938 int c2_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
 939                     struct ib_recv_wr **bad_wr)
 940 {
 941         struct c2_dev *c2dev = to_c2dev(ibqp->device);
 942         struct c2_qp *qp = to_c2qp(ibqp);
 943         union c2wr wr;
 944         unsigned long lock_flags;
 945         int err = 0;
 946
 947         if (qp->state > IB_QPS_RTS)
 948                 return -EINVAL;
 949
 950         /*
 951          * Try and post each work request
 952          */
 953         while (ib_wr) {
 954                 u32 tot_len;
 955                 u8 actual_sge_count;
 956
 957                 if (ib_wr->num_sge > qp->recv_sgl_depth) {
 958                         err = -EINVAL;
 959                         break;
 960                 }
 961
 962                 /*
 963                  * Create local host-copy of the WR
 964                  */
 965                 wr.rqwr.rq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
 966                 c2_wr_set_id(&wr, CCWR_RECV);
 967                 c2_wr_set_flags(&wr, 0);
 968
 969                 /* sge_count is limited to eight bits. */
 970                 BUG_ON(ib_wr->num_sge >= 256);
 971                 err = move_sgl((struct c2_data_addr *) & (wr.rqwr.data),
 972                                ib_wr->sg_list,
 973                                ib_wr->num_sge, &tot_len, &actual_sge_count);
 974                 c2_wr_set_sge_count(&wr, actual_sge_count);
 975
 976                 /*
 977                  * If we had an error on the last wr build, then
 978                  * break out.  Possible errors include bogus WR
 979                  * type, and a bogus SGL length...
 980                  */
 981                 if (err) {
 982                         break;
 983                 }
 984
 985                 spin_lock_irqsave(&qp->lock, lock_flags);
 986                 err = qp_wr_post(&qp->rq_mq, &wr, qp, qp->rq_mq.msg_size);
 987                 if (err) {
 988                         spin_unlock_irqrestore(&qp->lock, lock_flags);
 989                         break;
 990                 }
 991
 992                 /*
 993                  * Enqueue mq index to activity FIFO
 994                  */
 995                 c2_activity(c2dev, qp->rq_mq.index, qp->rq_mq.hint_count);
 996                 spin_unlock_irqrestore(&qp->lock, lock_flags);
 997
 998                 ib_wr = ib_wr->next;
 999         }
1000
1001         if (err)
1002                 *bad_wr = ib_wr;
1003         return err;
1004 }
1005
1006 void __devinit c2_init_qp_table(struct c2_dev *c2dev)
1007 {
1008         spin_lock_init(&c2dev->qp_table.lock);
1009         idr_init(&c2dev->qp_table.idr);
1010 }
1011
1012 void __devexit c2_cleanup_qp_table(struct c2_dev *c2dev)
1013 {
1014         idr_destroy(&c2dev->qp_table.idr);
1015 }