drivers/infiniband/hw/ipath/ipath_verbs.c

   1 /*
   2  * Copyright (c) 2006 QLogic, Inc. All rights reserved.
   3  * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <rdma/ib_mad.h>
  35 #include <rdma/ib_user_verbs.h>
  36 #include <linux/utsname.h>
  37
  38 #include "ipath_kernel.h"
  39 #include "ipath_verbs.h"
  40 #include "ips_common.h"
  41
  42 /* Not static, because we don't want the compiler removing it */
  43 const char ipath_verbs_version[] = "ipath_verbs " IPATH_IDSTR;
  44
  45 static unsigned int ib_ipath_qp_table_size = 251;
  46 module_param_named(qp_table_size, ib_ipath_qp_table_size, uint, S_IRUGO);
  47 MODULE_PARM_DESC(qp_table_size, "QP table size");
  48
  49 unsigned int ib_ipath_lkey_table_size = 12;
  50 module_param_named(lkey_table_size, ib_ipath_lkey_table_size, uint,
  51                    S_IRUGO);
  52 MODULE_PARM_DESC(lkey_table_size,
  53                  "LKEY table size in bits (2^n, 1 <= n <= 23)");
  54
  55 unsigned int ib_ipath_debug;    /* debug mask */
  56 module_param_named(debug, ib_ipath_debug, uint, S_IWUSR | S_IRUGO);
  57 MODULE_PARM_DESC(debug, "Verbs debug mask");
  58
  59 static unsigned int ib_ipath_max_pds = 0xFFFF;
  60 module_param_named(max_pds, ib_ipath_max_pds, uint, S_IWUSR | S_IRUGO);
  61 MODULE_PARM_DESC(max_pds,
  62                  "Maximum number of protection domains to support");
  63
  64 static unsigned int ib_ipath_max_ahs = 0xFFFF;
  65 module_param_named(max_ahs, ib_ipath_max_ahs, uint, S_IWUSR | S_IRUGO);
  66 MODULE_PARM_DESC(max_ahs, "Maximum number of address handles to support");
  67
  68 unsigned int ib_ipath_max_cqes = 0x2FFFF;
  69 module_param_named(max_cqes, ib_ipath_max_cqes, uint, S_IWUSR | S_IRUGO);
  70 MODULE_PARM_DESC(max_cqes,
  71                  "Maximum number of completion queue entries to support");
  72
  73 unsigned int ib_ipath_max_cqs = 0x1FFFF;
  74 module_param_named(max_cqs, ib_ipath_max_cqs, uint, S_IWUSR | S_IRUGO);
  75 MODULE_PARM_DESC(max_cqs, "Maximum number of completion queues to support");
  76
  77 unsigned int ib_ipath_max_qp_wrs = 0x3FFF;
  78 module_param_named(max_qp_wrs, ib_ipath_max_qp_wrs, uint,
  79                    S_IWUSR | S_IRUGO);
  80 MODULE_PARM_DESC(max_qp_wrs, "Maximum number of QP WRs to support");
  81
  82 unsigned int ib_ipath_max_sges = 0x60;
  83 module_param_named(max_sges, ib_ipath_max_sges, uint, S_IWUSR | S_IRUGO);
  84 MODULE_PARM_DESC(max_sges, "Maximum number of SGEs to support");
  85
  86 unsigned int ib_ipath_max_mcast_grps = 16384;
  87 module_param_named(max_mcast_grps, ib_ipath_max_mcast_grps, uint,
  88                    S_IWUSR | S_IRUGO);
  89 MODULE_PARM_DESC(max_mcast_grps,
  90                  "Maximum number of multicast groups to support");
  91
  92 unsigned int ib_ipath_max_mcast_qp_attached = 16;
  93 module_param_named(max_mcast_qp_attached, ib_ipath_max_mcast_qp_attached,
  94                    uint, S_IWUSR | S_IRUGO);
  95 MODULE_PARM_DESC(max_mcast_qp_attached,
  96                  "Maximum number of attached QPs to support");
  97
  98 unsigned int ib_ipath_max_srqs = 1024;
  99 module_param_named(max_srqs, ib_ipath_max_srqs, uint, S_IWUSR | S_IRUGO);
 100 MODULE_PARM_DESC(max_srqs, "Maximum number of SRQs to support");
 101
 102 unsigned int ib_ipath_max_srq_sges = 128;
 103 module_param_named(max_srq_sges, ib_ipath_max_srq_sges,
 104                    uint, S_IWUSR | S_IRUGO);
 105 MODULE_PARM_DESC(max_srq_sges, "Maximum number of SRQ SGEs to support");
 106
 107 unsigned int ib_ipath_max_srq_wrs = 0x1FFFF;
 108 module_param_named(max_srq_wrs, ib_ipath_max_srq_wrs,
 109                    uint, S_IWUSR | S_IRUGO);
 110 MODULE_PARM_DESC(max_srq_wrs, "Maximum number of SRQ WRs support");
 111
 112 MODULE_LICENSE("GPL");
 113 MODULE_AUTHOR("QLogic <support@pathscale.com>");
 114 MODULE_DESCRIPTION("QLogic InfiniPath driver");
 115
 116 const int ib_ipath_state_ops[IB_QPS_ERR + 1] = {
 117         [IB_QPS_RESET] = 0,
 118         [IB_QPS_INIT] = IPATH_POST_RECV_OK,
 119         [IB_QPS_RTR] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK,
 120         [IB_QPS_RTS] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK |
 121             IPATH_POST_SEND_OK | IPATH_PROCESS_SEND_OK,
 122         [IB_QPS_SQD] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK |
 123             IPATH_POST_SEND_OK,
 124         [IB_QPS_SQE] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK,
 125         [IB_QPS_ERR] = 0,
 126 };
 127
 128 /*
 129  * Translate ib_wr_opcode into ib_wc_opcode.
 130  */
 131 const enum ib_wc_opcode ib_ipath_wc_opcode[] = {
 132         [IB_WR_RDMA_WRITE] = IB_WC_RDMA_WRITE,
 133         [IB_WR_RDMA_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE,
 134         [IB_WR_SEND] = IB_WC_SEND,
 135         [IB_WR_SEND_WITH_IMM] = IB_WC_SEND,
 136         [IB_WR_RDMA_READ] = IB_WC_RDMA_READ,
 137         [IB_WR_ATOMIC_CMP_AND_SWP] = IB_WC_COMP_SWAP,
 138         [IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD
 139 };
 140
 141 /*
 142  * System image GUID.
 143  */
 144 static __be64 sys_image_guid;
 145
 146 /**
 147  * ipath_copy_sge - copy data to SGE memory
 148  * @ss: the SGE state
 149  * @data: the data to copy
 150  * @length: the length of the data
 151  */
 152 void ipath_copy_sge(struct ipath_sge_state *ss, void *data, u32 length)
 153 {
 154         struct ipath_sge *sge = &ss->sge;
 155
 156         while (length) {
 157                 u32 len = sge->length;
 158
 159                 BUG_ON(len == 0);
 160                 if (len > length)
 161                         len = length;
 162                 memcpy(sge->vaddr, data, len);
 163                 sge->vaddr += len;
 164                 sge->length -= len;
 165                 sge->sge_length -= len;
 166                 if (sge->sge_length == 0) {
 167                         if (--ss->num_sge)
 168                                 *sge = *ss->sg_list++;
 169                 } else if (sge->length == 0 && sge->mr != NULL) {
 170                         if (++sge->n >= IPATH_SEGSZ) {
 171                                 if (++sge->m >= sge->mr->mapsz)
 172                                         break;
 173                                 sge->n = 0;
 174                         }
 175                         sge->vaddr =
 176                                 sge->mr->map[sge->m]->segs[sge->n].vaddr;
 177                         sge->length =
 178                                 sge->mr->map[sge->m]->segs[sge->n].length;
 179                 }
 180                 data += len;
 181                 length -= len;
 182         }
 183 }
 184
 185 /**
 186  * ipath_skip_sge - skip over SGE memory - XXX almost dup of prev func
 187  * @ss: the SGE state
 188  * @length: the number of bytes to skip
 189  */
 190 void ipath_skip_sge(struct ipath_sge_state *ss, u32 length)
 191 {
 192         struct ipath_sge *sge = &ss->sge;
 193
 194         while (length > sge->sge_length) {
 195                 length -= sge->sge_length;
 196                 ss->sge = *ss->sg_list++;
 197         }
 198         while (length) {
 199                 u32 len = sge->length;
 200
 201                 BUG_ON(len == 0);
 202                 if (len > length)
 203                         len = length;
 204                 sge->vaddr += len;
 205                 sge->length -= len;
 206                 sge->sge_length -= len;
 207                 if (sge->sge_length == 0) {
 208                         if (--ss->num_sge)
 209                                 *sge = *ss->sg_list++;
 210                 } else if (sge->length == 0 && sge->mr != NULL) {
 211                         if (++sge->n >= IPATH_SEGSZ) {
 212                                 if (++sge->m >= sge->mr->mapsz)
 213                                         break;
 214                                 sge->n = 0;
 215                         }
 216                         sge->vaddr =
 217                                 sge->mr->map[sge->m]->segs[sge->n].vaddr;
 218                         sge->length =
 219                                 sge->mr->map[sge->m]->segs[sge->n].length;
 220                 }
 221                 length -= len;
 222         }
 223 }
 224
 225 /**
 226  * ipath_post_send - post a send on a QP
 227  * @ibqp: the QP to post the send on
 228  * @wr: the list of work requests to post
 229  * @bad_wr: the first bad WR is put here
 230  *
 231  * This may be called from interrupt context.
 232  */
 233 static int ipath_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 234                            struct ib_send_wr **bad_wr)
 235 {
 236         struct ipath_qp *qp = to_iqp(ibqp);
 237         int err = 0;
 238
 239         /* Check that state is OK to post send. */
 240         if (!(ib_ipath_state_ops[qp->state] & IPATH_POST_SEND_OK)) {
 241                 *bad_wr = wr;
 242                 err = -EINVAL;
 243                 goto bail;
 244         }
 245
 246         for (; wr; wr = wr->next) {
 247                 switch (qp->ibqp.qp_type) {
 248                 case IB_QPT_UC:
 249                 case IB_QPT_RC:
 250                         err = ipath_post_ruc_send(qp, wr);
 251                         break;
 252
 253                 case IB_QPT_SMI:
 254                 case IB_QPT_GSI:
 255                 case IB_QPT_UD:
 256                         err = ipath_post_ud_send(qp, wr);
 257                         break;
 258
 259                 default:
 260                         err = -EINVAL;
 261                 }
 262                 if (err) {
 263                         *bad_wr = wr;
 264                         break;
 265                 }
 266         }
 267
 268 bail:
 269         return err;
 270 }
 271
 272 /**
 273  * ipath_post_receive - post a receive on a QP
 274  * @ibqp: the QP to post the receive on
 275  * @wr: the WR to post
 276  * @bad_wr: the first bad WR is put here
 277  *
 278  * This may be called from interrupt context.
 279  */
 280 static int ipath_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 281                               struct ib_recv_wr **bad_wr)
 282 {
 283         struct ipath_qp *qp = to_iqp(ibqp);
 284         unsigned long flags;
 285         int ret;
 286
 287         /* Check that state is OK to post receive. */
 288         if (!(ib_ipath_state_ops[qp->state] & IPATH_POST_RECV_OK)) {
 289                 *bad_wr = wr;
 290                 ret = -EINVAL;
 291                 goto bail;
 292         }
 293
 294         for (; wr; wr = wr->next) {
 295                 struct ipath_rwqe *wqe;
 296                 u32 next;
 297                 int i, j;
 298
 299                 if (wr->num_sge > qp->r_rq.max_sge) {
 300                         *bad_wr = wr;
 301                         ret = -ENOMEM;
 302                         goto bail;
 303                 }
 304
 305                 spin_lock_irqsave(&qp->r_rq.lock, flags);
 306                 next = qp->r_rq.head + 1;
 307                 if (next >= qp->r_rq.size)
 308                         next = 0;
 309                 if (next == qp->r_rq.tail) {
 310                         spin_unlock_irqrestore(&qp->r_rq.lock, flags);
 311                         *bad_wr = wr;
 312                         ret = -ENOMEM;
 313                         goto bail;
 314                 }
 315
 316                 wqe = get_rwqe_ptr(&qp->r_rq, qp->r_rq.head);
 317                 wqe->wr_id = wr->wr_id;
 318                 wqe->sg_list[0].mr = NULL;
 319                 wqe->sg_list[0].vaddr = NULL;
 320                 wqe->sg_list[0].length = 0;
 321                 wqe->sg_list[0].sge_length = 0;
 322                 wqe->length = 0;
 323                 for (i = 0, j = 0; i < wr->num_sge; i++) {
 324                         /* Check LKEY */
 325                         if (to_ipd(qp->ibqp.pd)->user &&
 326                             wr->sg_list[i].lkey == 0) {
 327                                 spin_unlock_irqrestore(&qp->r_rq.lock,
 328                                                        flags);
 329                                 *bad_wr = wr;
 330                                 ret = -EINVAL;
 331                                 goto bail;
 332                         }
 333                         if (wr->sg_list[i].length == 0)
 334                                 continue;
 335                         if (!ipath_lkey_ok(
 336                                     &to_idev(qp->ibqp.device)->lk_table,
 337                                     &wqe->sg_list[j], &wr->sg_list[i],
 338                                     IB_ACCESS_LOCAL_WRITE)) {
 339                                 spin_unlock_irqrestore(&qp->r_rq.lock,
 340                                                        flags);
 341                                 *bad_wr = wr;
 342                                 ret = -EINVAL;
 343                                 goto bail;
 344                         }
 345                         wqe->length += wr->sg_list[i].length;
 346                         j++;
 347                 }
 348                 wqe->num_sge = j;
 349                 qp->r_rq.head = next;
 350                 spin_unlock_irqrestore(&qp->r_rq.lock, flags);
 351         }
 352         ret = 0;
 353
 354 bail:
 355         return ret;
 356 }
 357
 358 /**
 359  * ipath_qp_rcv - processing an incoming packet on a QP
 360  * @dev: the device the packet came on
 361  * @hdr: the packet header
 362  * @has_grh: true if the packet has a GRH
 363  * @data: the packet data
 364  * @tlen: the packet length
 365  * @qp: the QP the packet came on
 366  *
 367  * This is called from ipath_ib_rcv() to process an incoming packet
 368  * for the given QP.
 369  * Called at interrupt level.
 370  */
 371 static void ipath_qp_rcv(struct ipath_ibdev *dev,
 372                          struct ipath_ib_header *hdr, int has_grh,
 373                          void *data, u32 tlen, struct ipath_qp *qp)
 374 {
 375         /* Check for valid receive state. */
 376         if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) {
 377                 dev->n_pkt_drops++;
 378                 return;
 379         }
 380
 381         switch (qp->ibqp.qp_type) {
 382         case IB_QPT_SMI:
 383         case IB_QPT_GSI:
 384         case IB_QPT_UD:
 385                 ipath_ud_rcv(dev, hdr, has_grh, data, tlen, qp);
 386                 break;
 387
 388         case IB_QPT_RC:
 389                 ipath_rc_rcv(dev, hdr, has_grh, data, tlen, qp);
 390                 break;
 391
 392         case IB_QPT_UC:
 393                 ipath_uc_rcv(dev, hdr, has_grh, data, tlen, qp);
 394                 break;
 395
 396         default:
 397                 break;
 398         }
 399 }
 400
 401 /**
 402  * ipath_ib_rcv - process and incoming packet
 403  * @arg: the device pointer
 404  * @rhdr: the header of the packet
 405  * @data: the packet data
 406  * @tlen: the packet length
 407  *
 408  * This is called from ipath_kreceive() to process an incoming packet at
 409  * interrupt level. Tlen is the length of the header + data + CRC in bytes.
 410  */
 411 static void ipath_ib_rcv(void *arg, void *rhdr, void *data, u32 tlen)
 412 {
 413         struct ipath_ibdev *dev = (struct ipath_ibdev *) arg;
 414         struct ipath_ib_header *hdr = rhdr;
 415         struct ipath_other_headers *ohdr;
 416         struct ipath_qp *qp;
 417         u32 qp_num;
 418         int lnh;
 419         u8 opcode;
 420         u16 lid;
 421
 422         if (unlikely(dev == NULL))
 423                 goto bail;
 424
 425         if (unlikely(tlen < 24)) {      /* LRH+BTH+CRC */
 426                 dev->rcv_errors++;
 427                 goto bail;
 428         }
 429
 430         /* Check for a valid destination LID (see ch. 7.11.1). */
 431         lid = be16_to_cpu(hdr->lrh[1]);
 432         if (lid < IPS_MULTICAST_LID_BASE) {
 433                 lid &= ~((1 << (dev->mkeyprot_resv_lmc & 7)) - 1);
 434                 if (unlikely(lid != ipath_layer_get_lid(dev->dd))) {
 435                         dev->rcv_errors++;
 436                         goto bail;
 437                 }
 438         }
 439
 440         /* Check for GRH */
 441         lnh = be16_to_cpu(hdr->lrh[0]) & 3;
 442         if (lnh == IPS_LRH_BTH)
 443                 ohdr = &hdr->u.oth;
 444         else if (lnh == IPS_LRH_GRH)
 445                 ohdr = &hdr->u.l.oth;
 446         else {
 447                 dev->rcv_errors++;
 448                 goto bail;
 449         }
 450
 451         opcode = be32_to_cpu(ohdr->bth[0]) >> 24;
 452         dev->opstats[opcode].n_bytes += tlen;
 453         dev->opstats[opcode].n_packets++;
 454
 455         /* Get the destination QP number. */
 456         qp_num = be32_to_cpu(ohdr->bth[1]) & IPS_QPN_MASK;
 457         if (qp_num == IPS_MULTICAST_QPN) {
 458                 struct ipath_mcast *mcast;
 459                 struct ipath_mcast_qp *p;
 460
 461                 mcast = ipath_mcast_find(&hdr->u.l.grh.dgid);
 462                 if (mcast == NULL) {
 463                         dev->n_pkt_drops++;
 464                         goto bail;
 465                 }
 466                 dev->n_multicast_rcv++;
 467                 list_for_each_entry_rcu(p, &mcast->qp_list, list)
 468                         ipath_qp_rcv(dev, hdr, lnh == IPS_LRH_GRH, data,
 469                                      tlen, p->qp);
 470                 /*
 471                  * Notify ipath_multicast_detach() if it is waiting for us
 472                  * to finish.
 473                  */
 474                 if (atomic_dec_return(&mcast->refcount) <= 1)
 475                         wake_up(&mcast->wait);
 476         } else {
 477                 qp = ipath_lookup_qpn(&dev->qp_table, qp_num);
 478                 if (qp) {
 479                         dev->n_unicast_rcv++;
 480                         ipath_qp_rcv(dev, hdr, lnh == IPS_LRH_GRH, data,
 481                                      tlen, qp);
 482                         /*
 483                          * Notify ipath_destroy_qp() if it is waiting
 484                          * for us to finish.
 485                          */
 486                         if (atomic_dec_and_test(&qp->refcount))
 487                                 wake_up(&qp->wait);
 488                 } else
 489                         dev->n_pkt_drops++;
 490         }
 491
 492 bail:;
 493 }
 494
 495 /**
 496  * ipath_ib_timer - verbs timer
 497  * @arg: the device pointer
 498  *
 499  * This is called from ipath_do_rcv_timer() at interrupt level to check for
 500  * QPs which need retransmits and to collect performance numbers.
 501  */
 502 static void ipath_ib_timer(void *arg)
 503 {
 504         struct ipath_ibdev *dev = (struct ipath_ibdev *) arg;
 505         struct ipath_qp *resend = NULL;
 506         struct list_head *last;
 507         struct ipath_qp *qp;
 508         unsigned long flags;
 509
 510         if (dev == NULL)
 511                 return;
 512
 513         spin_lock_irqsave(&dev->pending_lock, flags);
 514         /* Start filling the next pending queue. */
 515         if (++dev->pending_index >= ARRAY_SIZE(dev->pending))
 516                 dev->pending_index = 0;
 517         /* Save any requests still in the new queue, they have timed out. */
 518         last = &dev->pending[dev->pending_index];
 519         while (!list_empty(last)) {
 520                 qp = list_entry(last->next, struct ipath_qp, timerwait);
 521                 list_del_init(&qp->timerwait);
 522                 qp->timer_next = resend;
 523                 resend = qp;
 524                 atomic_inc(&qp->refcount);
 525         }
 526         last = &dev->rnrwait;
 527         if (!list_empty(last)) {
 528                 qp = list_entry(last->next, struct ipath_qp, timerwait);
 529                 if (--qp->s_rnr_timeout == 0) {
 530                         do {
 531                                 list_del_init(&qp->timerwait);
 532                                 tasklet_hi_schedule(&qp->s_task);
 533                                 if (list_empty(last))
 534                                         break;
 535                                 qp = list_entry(last->next, struct ipath_qp,
 536                                                 timerwait);
 537                         } while (qp->s_rnr_timeout == 0);
 538                 }
 539         }
 540         /*
 541          * We should only be in the started state if pma_sample_start != 0
 542          */
 543         if (dev->pma_sample_status == IB_PMA_SAMPLE_STATUS_STARTED &&
 544             --dev->pma_sample_start == 0) {
 545                 dev->pma_sample_status = IB_PMA_SAMPLE_STATUS_RUNNING;
 546                 ipath_layer_snapshot_counters(dev->dd, &dev->ipath_sword,
 547                                               &dev->ipath_rword,
 548                                               &dev->ipath_spkts,
 549                                               &dev->ipath_rpkts,
 550                                               &dev->ipath_xmit_wait);
 551         }
 552         if (dev->pma_sample_status == IB_PMA_SAMPLE_STATUS_RUNNING) {
 553                 if (dev->pma_sample_interval == 0) {
 554                         u64 ta, tb, tc, td, te;
 555
 556                         dev->pma_sample_status = IB_PMA_SAMPLE_STATUS_DONE;
 557                         ipath_layer_snapshot_counters(dev->dd, &ta, &tb,
 558                                                       &tc, &td, &te);
 559
 560                         dev->ipath_sword = ta - dev->ipath_sword;
 561                         dev->ipath_rword = tb - dev->ipath_rword;
 562                         dev->ipath_spkts = tc - dev->ipath_spkts;
 563                         dev->ipath_rpkts = td - dev->ipath_rpkts;
 564                         dev->ipath_xmit_wait = te - dev->ipath_xmit_wait;
 565                 }
 566                 else
 567                         dev->pma_sample_interval--;
 568         }
 569         spin_unlock_irqrestore(&dev->pending_lock, flags);
 570
 571         /* XXX What if timer fires again while this is running? */
 572         for (qp = resend; qp != NULL; qp = qp->timer_next) {
 573                 struct ib_wc wc;
 574
 575                 spin_lock_irqsave(&qp->s_lock, flags);
 576                 if (qp->s_last != qp->s_tail && qp->state == IB_QPS_RTS) {
 577                         dev->n_timeouts++;
 578                         ipath_restart_rc(qp, qp->s_last_psn + 1, &wc);
 579                 }
 580                 spin_unlock_irqrestore(&qp->s_lock, flags);
 581
 582                 /* Notify ipath_destroy_qp() if it is waiting. */
 583                 if (atomic_dec_and_test(&qp->refcount))
 584                         wake_up(&qp->wait);
 585         }
 586 }
 587
 588 /**
 589  * ipath_ib_piobufavail - callback when a PIO buffer is available
 590  * @arg: the device pointer
 591  *
 592  * This is called from ipath_intr() at interrupt level when a PIO buffer is
 593  * available after ipath_verbs_send() returned an error that no buffers were
 594  * available.  Return 1 if we consumed all the PIO buffers and we still have
 595  * QPs waiting for buffers (for now, just do a tasklet_hi_schedule and
 596  * return zero).
 597  */
 598 static int ipath_ib_piobufavail(void *arg)
 599 {
 600         struct ipath_ibdev *dev = (struct ipath_ibdev *) arg;
 601         struct ipath_qp *qp;
 602         unsigned long flags;
 603
 604         if (dev == NULL)
 605                 goto bail;
 606
 607         spin_lock_irqsave(&dev->pending_lock, flags);
 608         while (!list_empty(&dev->piowait)) {
 609                 qp = list_entry(dev->piowait.next, struct ipath_qp,
 610                                 piowait);
 611                 list_del_init(&qp->piowait);
 612                 tasklet_hi_schedule(&qp->s_task);
 613         }
 614         spin_unlock_irqrestore(&dev->pending_lock, flags);
 615
 616 bail:
 617         return 0;
 618 }
 619
 620 static int ipath_query_device(struct ib_device *ibdev,
 621                               struct ib_device_attr *props)
 622 {
 623         struct ipath_ibdev *dev = to_idev(ibdev);
 624
 625         memset(props, 0, sizeof(*props));
 626
 627         props->device_cap_flags = IB_DEVICE_BAD_PKEY_CNTR |
 628                 IB_DEVICE_BAD_QKEY_CNTR | IB_DEVICE_SHUTDOWN_PORT |
 629                 IB_DEVICE_SYS_IMAGE_GUID;
 630         props->vendor_id = ipath_layer_get_vendorid(dev->dd);
 631         props->vendor_part_id = ipath_layer_get_deviceid(dev->dd);
 632         props->hw_ver = ipath_layer_get_pcirev(dev->dd);
 633
 634         props->sys_image_guid = dev->sys_image_guid;
 635
 636         props->max_mr_size = ~0ull;
 637         props->max_qp = dev->qp_table.max;
 638         props->max_qp_wr = ib_ipath_max_qp_wrs;
 639         props->max_sge = ib_ipath_max_sges;
 640         props->max_cq = ib_ipath_max_cqs;
 641         props->max_ah = ib_ipath_max_ahs;
 642         props->max_cqe = ib_ipath_max_cqes;
 643         props->max_mr = dev->lk_table.max;
 644         props->max_pd = ib_ipath_max_pds;
 645         props->max_qp_rd_atom = 1;
 646         props->max_qp_init_rd_atom = 1;
 647         /* props->max_res_rd_atom */
 648         props->max_srq = ib_ipath_max_srqs;
 649         props->max_srq_wr = ib_ipath_max_srq_wrs;
 650         props->max_srq_sge = ib_ipath_max_srq_sges;
 651         /* props->local_ca_ack_delay */
 652         props->atomic_cap = IB_ATOMIC_HCA;
 653         props->max_pkeys = ipath_layer_get_npkeys(dev->dd);
 654         props->max_mcast_grp = ib_ipath_max_mcast_grps;
 655         props->max_mcast_qp_attach = ib_ipath_max_mcast_qp_attached;
 656         props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
 657                 props->max_mcast_grp;
 658
 659         return 0;
 660 }
 661
 662 const u8 ipath_cvt_physportstate[16] = {
 663         [INFINIPATH_IBCS_LT_STATE_DISABLED] = 3,
 664         [INFINIPATH_IBCS_LT_STATE_LINKUP] = 5,
 665         [INFINIPATH_IBCS_LT_STATE_POLLACTIVE] = 2,
 666         [INFINIPATH_IBCS_LT_STATE_POLLQUIET] = 2,
 667         [INFINIPATH_IBCS_LT_STATE_SLEEPDELAY] = 1,
 668         [INFINIPATH_IBCS_LT_STATE_SLEEPQUIET] = 1,
 669         [INFINIPATH_IBCS_LT_STATE_CFGDEBOUNCE] = 4,
 670         [INFINIPATH_IBCS_LT_STATE_CFGRCVFCFG] = 4,
 671         [INFINIPATH_IBCS_LT_STATE_CFGWAITRMT] = 4,
 672         [INFINIPATH_IBCS_LT_STATE_CFGIDLE] = 4,
 673         [INFINIPATH_IBCS_LT_STATE_RECOVERRETRAIN] = 6,
 674         [INFINIPATH_IBCS_LT_STATE_RECOVERWAITRMT] = 6,
 675         [INFINIPATH_IBCS_LT_STATE_RECOVERIDLE] = 6,
 676 };
 677
 678 static int ipath_query_port(struct ib_device *ibdev,
 679                             u8 port, struct ib_port_attr *props)
 680 {
 681         struct ipath_ibdev *dev = to_idev(ibdev);
 682         enum ib_mtu mtu;
 683         u16 lid = ipath_layer_get_lid(dev->dd);
 684         u64 ibcstat;
 685
 686         memset(props, 0, sizeof(*props));
 687         props->lid = lid ? lid : __constant_be16_to_cpu(IB_LID_PERMISSIVE);
 688         props->lmc = dev->mkeyprot_resv_lmc & 7;
 689         props->sm_lid = dev->sm_lid;
 690         props->sm_sl = dev->sm_sl;
 691         ibcstat = ipath_layer_get_lastibcstat(dev->dd);
 692         props->state = ((ibcstat >> 4) & 0x3) + 1;
 693         /* See phys_state_show() */
 694         props->phys_state = ipath_cvt_physportstate[
 695                 ipath_layer_get_lastibcstat(dev->dd) & 0xf];
 696         props->port_cap_flags = dev->port_cap_flags;
 697         props->gid_tbl_len = 1;
 698         props->max_msg_sz = 0x80000000;
 699         props->pkey_tbl_len = ipath_layer_get_npkeys(dev->dd);
 700         props->bad_pkey_cntr = ipath_layer_get_cr_errpkey(dev->dd) -
 701                 dev->z_pkey_violations;
 702         props->qkey_viol_cntr = dev->qkey_violations;
 703         props->active_width = IB_WIDTH_4X;
 704         /* See rate_show() */
 705         props->active_speed = 1;        /* Regular 10Mbs speed. */
 706         props->max_vl_num = 1;          /* VLCap = VL0 */
 707         props->init_type_reply = 0;
 708
 709         props->max_mtu = IB_MTU_4096;
 710         switch (ipath_layer_get_ibmtu(dev->dd)) {
 711         case 4096:
 712                 mtu = IB_MTU_4096;
 713                 break;
 714         case 2048:
 715                 mtu = IB_MTU_2048;
 716                 break;
 717         case 1024:
 718                 mtu = IB_MTU_1024;
 719                 break;
 720         case 512:
 721                 mtu = IB_MTU_512;
 722                 break;
 723         case 256:
 724                 mtu = IB_MTU_256;
 725                 break;
 726         default:
 727                 mtu = IB_MTU_2048;
 728         }
 729         props->active_mtu = mtu;
 730         props->subnet_timeout = dev->subnet_timeout;
 731
 732         return 0;
 733 }
 734
 735 static int ipath_modify_device(struct ib_device *device,
 736                                int device_modify_mask,
 737                                struct ib_device_modify *device_modify)
 738 {
 739         int ret;
 740
 741         if (device_modify_mask & ~(IB_DEVICE_MODIFY_SYS_IMAGE_GUID |
 742                                    IB_DEVICE_MODIFY_NODE_DESC)) {
 743                 ret = -EOPNOTSUPP;
 744                 goto bail;
 745         }
 746
 747         if (device_modify_mask & IB_DEVICE_MODIFY_NODE_DESC)
 748                 memcpy(device->node_desc, device_modify->node_desc, 64);
 749
 750         if (device_modify_mask & IB_DEVICE_MODIFY_SYS_IMAGE_GUID)
 751                 to_idev(device)->sys_image_guid =
 752                         cpu_to_be64(device_modify->sys_image_guid);
 753
 754         ret = 0;
 755
 756 bail:
 757         return ret;
 758 }
 759
 760 static int ipath_modify_port(struct ib_device *ibdev,
 761                              u8 port, int port_modify_mask,
 762                              struct ib_port_modify *props)
 763 {
 764         struct ipath_ibdev *dev = to_idev(ibdev);
 765
 766         dev->port_cap_flags |= props->set_port_cap_mask;
 767         dev->port_cap_flags &= ~props->clr_port_cap_mask;
 768         if (port_modify_mask & IB_PORT_SHUTDOWN)
 769                 ipath_layer_set_linkstate(dev->dd, IPATH_IB_LINKDOWN);
 770         if (port_modify_mask & IB_PORT_RESET_QKEY_CNTR)
 771                 dev->qkey_violations = 0;
 772         return 0;
 773 }
 774
 775 static int ipath_query_gid(struct ib_device *ibdev, u8 port,
 776                            int index, union ib_gid *gid)
 777 {
 778         struct ipath_ibdev *dev = to_idev(ibdev);
 779         int ret;
 780
 781         if (index >= 1) {
 782                 ret = -EINVAL;
 783                 goto bail;
 784         }
 785         gid->global.subnet_prefix = dev->gid_prefix;
 786         gid->global.interface_id = ipath_layer_get_guid(dev->dd);
 787
 788         ret = 0;
 789
 790 bail:
 791         return ret;
 792 }
 793
 794 static struct ib_pd *ipath_alloc_pd(struct ib_device *ibdev,
 795                                     struct ib_ucontext *context,
 796                                     struct ib_udata *udata)
 797 {
 798         struct ipath_ibdev *dev = to_idev(ibdev);
 799         struct ipath_pd *pd;
 800         struct ib_pd *ret;
 801
 802         /*
 803          * This is actually totally arbitrary.  Some correctness tests
 804          * assume there's a maximum number of PDs that can be allocated.
 805          * We don't actually have this limit, but we fail the test if
 806          * we allow allocations of more than we report for this value.
 807          */
 808
 809         if (dev->n_pds_allocated == ib_ipath_max_pds) {
 810                 ret = ERR_PTR(-ENOMEM);
 811                 goto bail;
 812         }
 813
 814         pd = kmalloc(sizeof *pd, GFP_KERNEL);
 815         if (!pd) {
 816                 ret = ERR_PTR(-ENOMEM);
 817                 goto bail;
 818         }
 819
 820         dev->n_pds_allocated++;
 821
 822         /* ib_alloc_pd() will initialize pd->ibpd. */
 823         pd->user = udata != NULL;
 824
 825         ret = &pd->ibpd;
 826
 827 bail:
 828         return ret;
 829 }
 830
 831 static int ipath_dealloc_pd(struct ib_pd *ibpd)
 832 {
 833         struct ipath_pd *pd = to_ipd(ibpd);
 834         struct ipath_ibdev *dev = to_idev(ibpd->device);
 835
 836         dev->n_pds_allocated--;
 837
 838         kfree(pd);
 839
 840         return 0;
 841 }
 842
 843 /**
 844  * ipath_create_ah - create an address handle
 845  * @pd: the protection domain
 846  * @ah_attr: the attributes of the AH
 847  *
 848  * This may be called from interrupt context.
 849  */
 850 static struct ib_ah *ipath_create_ah(struct ib_pd *pd,
 851                                      struct ib_ah_attr *ah_attr)
 852 {
 853         struct ipath_ah *ah;
 854         struct ib_ah *ret;
 855         struct ipath_ibdev *dev = to_idev(pd->device);
 856
 857         if (dev->n_ahs_allocated == ib_ipath_max_ahs) {
 858                 ret = ERR_PTR(-ENOMEM);
 859                 goto bail;
 860         }
 861
 862         /* A multicast address requires a GRH (see ch. 8.4.1). */
 863         if (ah_attr->dlid >= IPS_MULTICAST_LID_BASE &&
 864             ah_attr->dlid != IPS_PERMISSIVE_LID &&
 865             !(ah_attr->ah_flags & IB_AH_GRH)) {
 866                 ret = ERR_PTR(-EINVAL);
 867                 goto bail;
 868         }
 869
 870         if (ah_attr->dlid == 0) {
 871                 ret = ERR_PTR(-EINVAL);
 872                 goto bail;
 873         }
 874
 875         if (ah_attr->port_num < 1 ||
 876             ah_attr->port_num > pd->device->phys_port_cnt) {
 877                 ret = ERR_PTR(-EINVAL);
 878                 goto bail;
 879         }
 880
 881         ah = kmalloc(sizeof *ah, GFP_ATOMIC);
 882         if (!ah) {
 883                 ret = ERR_PTR(-ENOMEM);
 884                 goto bail;
 885         }
 886
 887         dev->n_ahs_allocated++;
 888
 889         /* ib_create_ah() will initialize ah->ibah. */
 890         ah->attr = *ah_attr;
 891
 892         ret = &ah->ibah;
 893
 894 bail:
 895         return ret;
 896 }
 897
 898 /**
 899  * ipath_destroy_ah - destroy an address handle
 900  * @ibah: the AH to destroy
 901  *
 902  * This may be called from interrupt context.
 903  */
 904 static int ipath_destroy_ah(struct ib_ah *ibah)
 905 {
 906         struct ipath_ibdev *dev = to_idev(ibah->device);
 907         struct ipath_ah *ah = to_iah(ibah);
 908
 909         dev->n_ahs_allocated--;
 910
 911         kfree(ah);
 912
 913         return 0;
 914 }
 915
 916 static int ipath_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr)
 917 {
 918         struct ipath_ah *ah = to_iah(ibah);
 919
 920         *ah_attr = ah->attr;
 921
 922         return 0;
 923 }
 924
 925 static int ipath_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
 926                             u16 *pkey)
 927 {
 928         struct ipath_ibdev *dev = to_idev(ibdev);
 929         int ret;
 930
 931         if (index >= ipath_layer_get_npkeys(dev->dd)) {
 932                 ret = -EINVAL;
 933                 goto bail;
 934         }
 935
 936         *pkey = ipath_layer_get_pkey(dev->dd, index);
 937         ret = 0;
 938
 939 bail:
 940         return ret;
 941 }
 942
 943
 944 /**
 945  * ipath_alloc_ucontext - allocate a ucontest
 946  * @ibdev: the infiniband device
 947  * @udata: not used by the InfiniPath driver
 948  */
 949
 950 static struct ib_ucontext *ipath_alloc_ucontext(struct ib_device *ibdev,
 951                                                 struct ib_udata *udata)
 952 {
 953         struct ipath_ucontext *context;
 954         struct ib_ucontext *ret;
 955
 956         context = kmalloc(sizeof *context, GFP_KERNEL);
 957         if (!context) {
 958                 ret = ERR_PTR(-ENOMEM);
 959                 goto bail;
 960         }
 961
 962         ret = &context->ibucontext;
 963
 964 bail:
 965         return ret;
 966 }
 967
 968 static int ipath_dealloc_ucontext(struct ib_ucontext *context)
 969 {
 970         kfree(to_iucontext(context));
 971         return 0;
 972 }
 973
 974 static int ipath_verbs_register_sysfs(struct ib_device *dev);
 975
 976 /**
 977  * ipath_register_ib_device - register our device with the infiniband core
 978  * @unit: the device number to register
 979  * @dd: the device data structure
 980  * Return the allocated ipath_ibdev pointer or NULL on error.
 981  */
 982 static void *ipath_register_ib_device(int unit, struct ipath_devdata *dd)
 983 {
 984         struct ipath_layer_counters cntrs;
 985         struct ipath_ibdev *idev;
 986         struct ib_device *dev;
 987         int ret;
 988
 989         idev = (struct ipath_ibdev *)ib_alloc_device(sizeof *idev);
 990         if (idev == NULL)
 991                 goto bail;
 992
 993         dev = &idev->ibdev;
 994
 995         /* Only need to initialize non-zero fields. */
 996         spin_lock_init(&idev->qp_table.lock);
 997         spin_lock_init(&idev->lk_table.lock);
 998         idev->sm_lid = __constant_be16_to_cpu(IB_LID_PERMISSIVE);
 999         /* Set the prefix to the default value (see ch. 4.1.1) */
1000         idev->gid_prefix = __constant_cpu_to_be64(0xfe80000000000000ULL);
1001
1002         ret = ipath_init_qp_table(idev, ib_ipath_qp_table_size);
1003         if (ret)
1004                 goto err_qp;
1005
1006         /*
1007          * The top ib_ipath_lkey_table_size bits are used to index the
1008          * table.  The lower 8 bits can be owned by the user (copied from
1009          * the LKEY).  The remaining bits act as a generation number or tag.
1010          */
1011         idev->lk_table.max = 1 << ib_ipath_lkey_table_size;
1012         idev->lk_table.table = kzalloc(idev->lk_table.max *
1013                                        sizeof(*idev->lk_table.table),
1014                                        GFP_KERNEL);
1015         if (idev->lk_table.table == NULL) {
1016                 ret = -ENOMEM;
1017                 goto err_lk;
1018         }
1019         spin_lock_init(&idev->pending_lock);
1020         INIT_LIST_HEAD(&idev->pending[0]);
1021         INIT_LIST_HEAD(&idev->pending[1]);
1022         INIT_LIST_HEAD(&idev->pending[2]);
1023         INIT_LIST_HEAD(&idev->piowait);
1024         INIT_LIST_HEAD(&idev->rnrwait);
1025         idev->pending_index = 0;
1026         idev->port_cap_flags =
1027                 IB_PORT_SYS_IMAGE_GUID_SUP | IB_PORT_CLIENT_REG_SUP;
1028         idev->pma_counter_select[0] = IB_PMA_PORT_XMIT_DATA;
1029         idev->pma_counter_select[1] = IB_PMA_PORT_RCV_DATA;
1030         idev->pma_counter_select[2] = IB_PMA_PORT_XMIT_PKTS;
1031         idev->pma_counter_select[3] = IB_PMA_PORT_RCV_PKTS;
1032         idev->pma_counter_select[5] = IB_PMA_PORT_XMIT_WAIT;
1033         idev->link_width_enabled = 3;   /* 1x or 4x */
1034
1035         /* Snapshot current HW counters to "clear" them. */
1036         ipath_layer_get_counters(dd, &cntrs);
1037         idev->z_symbol_error_counter = cntrs.symbol_error_counter;
1038         idev->z_link_error_recovery_counter =
1039                 cntrs.link_error_recovery_counter;
1040         idev->z_link_downed_counter = cntrs.link_downed_counter;
1041         idev->z_port_rcv_errors = cntrs.port_rcv_errors;
1042         idev->z_port_rcv_remphys_errors =
1043                 cntrs.port_rcv_remphys_errors;
1044         idev->z_port_xmit_discards = cntrs.port_xmit_discards;
1045         idev->z_port_xmit_data = cntrs.port_xmit_data;
1046         idev->z_port_rcv_data = cntrs.port_rcv_data;
1047         idev->z_port_xmit_packets = cntrs.port_xmit_packets;
1048         idev->z_port_rcv_packets = cntrs.port_rcv_packets;
1049         idev->z_local_link_integrity_errors =
1050                 cntrs.local_link_integrity_errors;
1051         idev->z_excessive_buffer_overrun_errors =
1052                 cntrs.excessive_buffer_overrun_errors;
1053
1054         /*
1055          * The system image GUID is supposed to be the same for all
1056          * IB HCAs in a single system but since there can be other
1057          * device types in the system, we can't be sure this is unique.
1058          */
1059         if (!sys_image_guid)
1060                 sys_image_guid = ipath_layer_get_guid(dd);
1061         idev->sys_image_guid = sys_image_guid;
1062         idev->ib_unit = unit;
1063         idev->dd = dd;
1064
1065         strlcpy(dev->name, "ipath%d", IB_DEVICE_NAME_MAX);
1066         dev->owner = THIS_MODULE;
1067         dev->node_guid = ipath_layer_get_guid(dd);
1068         dev->uverbs_abi_ver = IPATH_UVERBS_ABI_VERSION;
1069         dev->uverbs_cmd_mask =
1070                 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT)         |
1071                 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE)        |
1072                 (1ull << IB_USER_VERBS_CMD_QUERY_PORT)          |
1073                 (1ull << IB_USER_VERBS_CMD_ALLOC_PD)            |
1074                 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD)          |
1075                 (1ull << IB_USER_VERBS_CMD_CREATE_AH)           |
1076                 (1ull << IB_USER_VERBS_CMD_DESTROY_AH)          |
1077                 (1ull << IB_USER_VERBS_CMD_QUERY_AH)            |
1078                 (1ull << IB_USER_VERBS_CMD_REG_MR)              |
1079                 (1ull << IB_USER_VERBS_CMD_DEREG_MR)            |
1080                 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
1081                 (1ull << IB_USER_VERBS_CMD_CREATE_CQ)           |
1082                 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ)           |
1083                 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ)          |
1084                 (1ull << IB_USER_VERBS_CMD_POLL_CQ)             |
1085                 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ)       |
1086                 (1ull << IB_USER_VERBS_CMD_CREATE_QP)           |
1087                 (1ull << IB_USER_VERBS_CMD_QUERY_QP)            |
1088                 (1ull << IB_USER_VERBS_CMD_MODIFY_QP)           |
1089                 (1ull << IB_USER_VERBS_CMD_DESTROY_QP)          |
1090                 (1ull << IB_USER_VERBS_CMD_POST_SEND)           |
1091                 (1ull << IB_USER_VERBS_CMD_POST_RECV)           |
1092                 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST)        |
1093                 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST)        |
1094                 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ)          |
1095                 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ)          |
1096                 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ)           |
1097                 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ)         |
1098                 (1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV);
1099         dev->node_type = IB_NODE_CA;
1100         dev->phys_port_cnt = 1;
1101         dev->dma_device = ipath_layer_get_device(dd);
1102         dev->class_dev.dev = dev->dma_device;
1103         dev->query_device = ipath_query_device;
1104         dev->modify_device = ipath_modify_device;
1105         dev->query_port = ipath_query_port;
1106         dev->modify_port = ipath_modify_port;
1107         dev->query_pkey = ipath_query_pkey;
1108         dev->query_gid = ipath_query_gid;
1109         dev->alloc_ucontext = ipath_alloc_ucontext;
1110         dev->dealloc_ucontext = ipath_dealloc_ucontext;
1111         dev->alloc_pd = ipath_alloc_pd;
1112         dev->dealloc_pd = ipath_dealloc_pd;
1113         dev->create_ah = ipath_create_ah;
1114         dev->destroy_ah = ipath_destroy_ah;
1115         dev->query_ah = ipath_query_ah;
1116         dev->create_srq = ipath_create_srq;
1117         dev->modify_srq = ipath_modify_srq;
1118         dev->query_srq = ipath_query_srq;
1119         dev->destroy_srq = ipath_destroy_srq;
1120         dev->create_qp = ipath_create_qp;
1121         dev->modify_qp = ipath_modify_qp;
1122         dev->query_qp = ipath_query_qp;
1123         dev->destroy_qp = ipath_destroy_qp;
1124         dev->post_send = ipath_post_send;
1125         dev->post_recv = ipath_post_receive;
1126         dev->post_srq_recv = ipath_post_srq_receive;
1127         dev->create_cq = ipath_create_cq;
1128         dev->destroy_cq = ipath_destroy_cq;
1129         dev->resize_cq = ipath_resize_cq;
1130         dev->poll_cq = ipath_poll_cq;
1131         dev->req_notify_cq = ipath_req_notify_cq;
1132         dev->get_dma_mr = ipath_get_dma_mr;
1133         dev->reg_phys_mr = ipath_reg_phys_mr;
1134         dev->reg_user_mr = ipath_reg_user_mr;
1135         dev->dereg_mr = ipath_dereg_mr;
1136         dev->alloc_fmr = ipath_alloc_fmr;
1137         dev->map_phys_fmr = ipath_map_phys_fmr;
1138         dev->unmap_fmr = ipath_unmap_fmr;
1139         dev->dealloc_fmr = ipath_dealloc_fmr;
1140         dev->attach_mcast = ipath_multicast_attach;
1141         dev->detach_mcast = ipath_multicast_detach;
1142         dev->process_mad = ipath_process_mad;
1143
1144         snprintf(dev->node_desc, sizeof(dev->node_desc),
1145                  IPATH_IDSTR " %s kernel_SMA", system_utsname.nodename);
1146
1147         ret = ib_register_device(dev);
1148         if (ret)
1149                 goto err_reg;
1150
1151         if (ipath_verbs_register_sysfs(dev))
1152                 goto err_class;
1153
1154         ipath_layer_enable_timer(dd);
1155
1156         goto bail;
1157
1158 err_class:
1159         ib_unregister_device(dev);
1160 err_reg:
1161         kfree(idev->lk_table.table);
1162 err_lk:
1163         kfree(idev->qp_table.table);
1164 err_qp:
1165         ib_dealloc_device(dev);
1166         _VERBS_ERROR("ib_ipath%d cannot register verbs (%d)!\n",
1167                      unit, -ret);
1168         idev = NULL;
1169
1170 bail:
1171         return idev;
1172 }
1173
1174 static void ipath_unregister_ib_device(void *arg)
1175 {
1176         struct ipath_ibdev *dev = (struct ipath_ibdev *) arg;
1177         struct ib_device *ibdev = &dev->ibdev;
1178
1179         ipath_layer_disable_timer(dev->dd);
1180
1181         ib_unregister_device(ibdev);
1182
1183         if (!list_empty(&dev->pending[0]) ||
1184             !list_empty(&dev->pending[1]) ||
1185             !list_empty(&dev->pending[2]))
1186                 _VERBS_ERROR("ipath%d pending list not empty!\n",
1187                              dev->ib_unit);
1188         if (!list_empty(&dev->piowait))
1189                 _VERBS_ERROR("ipath%d piowait list not empty!\n",
1190                              dev->ib_unit);
1191         if (!list_empty(&dev->rnrwait))
1192                 _VERBS_ERROR("ipath%d rnrwait list not empty!\n",
1193                              dev->ib_unit);
1194         if (!ipath_mcast_tree_empty())
1195                 _VERBS_ERROR("ipath%d multicast table memory leak!\n",
1196                              dev->ib_unit);
1197         /*
1198          * Note that ipath_unregister_ib_device() can be called before all
1199          * the QPs are destroyed!
1200          */
1201         ipath_free_all_qps(&dev->qp_table);
1202         kfree(dev->qp_table.table);
1203         kfree(dev->lk_table.table);
1204         ib_dealloc_device(ibdev);
1205 }
1206
1207 static int __init ipath_verbs_init(void)
1208 {
1209         return ipath_verbs_register(ipath_register_ib_device,
1210                                     ipath_unregister_ib_device,
1211                                     ipath_ib_piobufavail, ipath_ib_rcv,
1212                                     ipath_ib_timer);
1213 }
1214
1215 static void __exit ipath_verbs_cleanup(void)
1216 {
1217         ipath_verbs_unregister();
1218 }
1219
1220 static ssize_t show_rev(struct class_device *cdev, char *buf)
1221 {
1222         struct ipath_ibdev *dev =
1223                 container_of(cdev, struct ipath_ibdev, ibdev.class_dev);
1224
1225         return sprintf(buf, "%x\n", ipath_layer_get_pcirev(dev->dd));
1226 }
1227
1228 static ssize_t show_hca(struct class_device *cdev, char *buf)
1229 {
1230         struct ipath_ibdev *dev =
1231                 container_of(cdev, struct ipath_ibdev, ibdev.class_dev);
1232         int ret;
1233
1234         ret = ipath_layer_get_boardname(dev->dd, buf, 128);
1235         if (ret < 0)
1236                 goto bail;
1237         strcat(buf, "\n");
1238         ret = strlen(buf);
1239
1240 bail:
1241         return ret;
1242 }
1243
1244 static ssize_t show_stats(struct class_device *cdev, char *buf)
1245 {
1246         struct ipath_ibdev *dev =
1247                 container_of(cdev, struct ipath_ibdev, ibdev.class_dev);
1248         int i;
1249         int len;
1250
1251         len = sprintf(buf,
1252                       "RC resends  %d\n"
1253                       "RC no QACK  %d\n"
1254                       "RC ACKs     %d\n"
1255                       "RC SEQ NAKs %d\n"
1256                       "RC RDMA seq %d\n"
1257                       "RC RNR NAKs %d\n"
1258                       "RC OTH NAKs %d\n"
1259                       "RC timeouts %d\n"
1260                       "RC RDMA dup %d\n"
1261                       "piobuf wait %d\n"
1262                       "no piobuf   %d\n"
1263                       "PKT drops   %d\n"
1264                       "WQE errs    %d\n",
1265                       dev->n_rc_resends, dev->n_rc_qacks, dev->n_rc_acks,
1266                       dev->n_seq_naks, dev->n_rdma_seq, dev->n_rnr_naks,
1267                       dev->n_other_naks, dev->n_timeouts,
1268                       dev->n_rdma_dup_busy, dev->n_piowait,
1269                       dev->n_no_piobuf, dev->n_pkt_drops, dev->n_wqe_errs);
1270         for (i = 0; i < ARRAY_SIZE(dev->opstats); i++) {
1271                 const struct ipath_opcode_stats *si = &dev->opstats[i];
1272
1273                 if (!si->n_packets && !si->n_bytes)
1274                         continue;
1275                 len += sprintf(buf + len, "%02x %llu/%llu\n", i,
1276                                (unsigned long long) si->n_packets,
1277                                (unsigned long long) si->n_bytes);
1278         }
1279         return len;
1280 }
1281
1282 static CLASS_DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
1283 static CLASS_DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL);
1284 static CLASS_DEVICE_ATTR(board_id, S_IRUGO, show_hca, NULL);
1285 static CLASS_DEVICE_ATTR(stats, S_IRUGO, show_stats, NULL);
1286
1287 static struct class_device_attribute *ipath_class_attributes[] = {
1288         &class_device_attr_hw_rev,
1289         &class_device_attr_hca_type,
1290         &class_device_attr_board_id,
1291         &class_device_attr_stats
1292 };
1293
1294 static int ipath_verbs_register_sysfs(struct ib_device *dev)
1295 {
1296         int i;
1297         int ret;
1298
1299         for (i = 0; i < ARRAY_SIZE(ipath_class_attributes); ++i)
1300                 if (class_device_create_file(&dev->class_dev,
1301                                              ipath_class_attributes[i])) {
1302                         ret = 1;
1303                         goto bail;
1304                 }
1305
1306         ret = 0;
1307
1308 bail:
1309         return ret;
1310 }
1311
1312 module_init(ipath_verbs_init);
1313 module_exit(ipath_verbs_cleanup);