drivers/infiniband/hw/amso1100/c2_rnic.c

   1 /*
   2  * Copyright (c) 2005 Ammasso, Inc. All rights reserved.
   3  * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  *
  33  */
  34
  35
  36 #include <linux/module.h>
  37 #include <linux/moduleparam.h>
  38 #include <linux/pci.h>
  39 #include <linux/netdevice.h>
  40 #include <linux/etherdevice.h>
  41 #include <linux/delay.h>
  42 #include <linux/ethtool.h>
  43 #include <linux/mii.h>
  44 #include <linux/if_vlan.h>
  45 #include <linux/crc32.h>
  46 #include <linux/in.h>
  47 #include <linux/ip.h>
  48 #include <linux/tcp.h>
  49 #include <linux/init.h>
  50 #include <linux/dma-mapping.h>
  51 #include <linux/mm.h>
  52 #include <linux/inet.h>
  53
  54 #include <linux/route.h>
  55
  56 #include <asm/io.h>
  57 #include <asm/irq.h>
  58 #include <asm/byteorder.h>
  59 #include <rdma/ib_smi.h>
  60 #include "c2.h"
  61 #include "c2_vq.h"
  62
  63 /* Device capabilities */
  64 #define C2_MIN_PAGESIZE  1024
  65
  66 #define C2_MAX_MRS       32768
  67 #define C2_MAX_QPS       16000
  68 #define C2_MAX_WQE_SZ    256
  69 #define C2_MAX_QP_WR     ((128*1024)/C2_MAX_WQE_SZ)
  70 #define C2_MAX_SGES      4
  71 #define C2_MAX_SGE_RD    1
  72 #define C2_MAX_CQS       32768
  73 #define C2_MAX_CQES      4096
  74 #define C2_MAX_PDS       16384
  75
  76 /*
  77  * Send the adapter INIT message to the amso1100
  78  */
  79 static int c2_adapter_init(struct c2_dev *c2dev)
  80 {
  81         struct c2wr_init_req wr;
  82         int err;
  83
  84         memset(&wr, 0, sizeof(wr));
  85         c2_wr_set_id(&wr, CCWR_INIT);
  86         wr.hdr.context = 0;
  87         wr.hint_count = cpu_to_be64(c2dev->hint_count_dma);
  88         wr.q0_host_shared = cpu_to_be64(c2dev->req_vq.shared_dma);
  89         wr.q1_host_shared = cpu_to_be64(c2dev->rep_vq.shared_dma);
  90         wr.q1_host_msg_pool = cpu_to_be64(c2dev->rep_vq.host_dma);
  91         wr.q2_host_shared = cpu_to_be64(c2dev->aeq.shared_dma);
  92         wr.q2_host_msg_pool = cpu_to_be64(c2dev->aeq.host_dma);
  93
  94         /* Post the init message */
  95         err = vq_send_wr(c2dev, (union c2wr *) & wr);
  96
  97         return err;
  98 }
  99
 100 /*
 101  * Send the adapter TERM message to the amso1100
 102  */
 103 static void c2_adapter_term(struct c2_dev *c2dev)
 104 {
 105         struct c2wr_init_req wr;
 106
 107         memset(&wr, 0, sizeof(wr));
 108         c2_wr_set_id(&wr, CCWR_TERM);
 109         wr.hdr.context = 0;
 110
 111         /* Post the init message */
 112         vq_send_wr(c2dev, (union c2wr *) & wr);
 113         c2dev->init = 0;
 114
 115         return;
 116 }
 117
 118 /*
 119  * Query the adapter
 120  */
 121 static int c2_rnic_query(struct c2_dev *c2dev, struct ib_device_attr *props)
 122 {
 123         struct c2_vq_req *vq_req;
 124         struct c2wr_rnic_query_req wr;
 125         struct c2wr_rnic_query_rep *reply;
 126         int err;
 127
 128         vq_req = vq_req_alloc(c2dev);
 129         if (!vq_req)
 130                 return -ENOMEM;
 131
 132         c2_wr_set_id(&wr, CCWR_RNIC_QUERY);
 133         wr.hdr.context = (unsigned long) vq_req;
 134         wr.rnic_handle = c2dev->adapter_handle;
 135
 136         vq_req_get(c2dev, vq_req);
 137
 138         err = vq_send_wr(c2dev, (union c2wr *) &wr);
 139         if (err) {
 140                 vq_req_put(c2dev, vq_req);
 141                 goto bail1;
 142         }
 143
 144         err = vq_wait_for_reply(c2dev, vq_req);
 145         if (err)
 146                 goto bail1;
 147
 148         reply =
 149             (struct c2wr_rnic_query_rep *) (unsigned long) (vq_req->reply_msg);
 150         if (!reply)
 151                 err = -ENOMEM;
 152
 153         err = c2_errno(reply);
 154         if (err)
 155                 goto bail2;
 156
 157         props->fw_ver =
 158                 ((u64)be32_to_cpu(reply->fw_ver_major) << 32) |
 159                 ((be32_to_cpu(reply->fw_ver_minor) && 0xFFFF) << 16) |
 160                 (be32_to_cpu(reply->fw_ver_patch) && 0xFFFF);
 161         memcpy(&props->sys_image_guid, c2dev->netdev->dev_addr, 6);
 162         props->max_mr_size         = 0xFFFFFFFF;
 163         props->page_size_cap       = ~(C2_MIN_PAGESIZE-1);
 164         props->vendor_id           = be32_to_cpu(reply->vendor_id);
 165         props->vendor_part_id      = be32_to_cpu(reply->part_number);
 166         props->hw_ver              = be32_to_cpu(reply->hw_version);
 167         props->max_qp              = be32_to_cpu(reply->max_qps);
 168         props->max_qp_wr           = be32_to_cpu(reply->max_qp_depth);
 169         props->device_cap_flags    = c2dev->device_cap_flags;
 170         props->max_sge             = C2_MAX_SGES;
 171         props->max_sge_rd          = C2_MAX_SGE_RD;
 172         props->max_cq              = be32_to_cpu(reply->max_cqs);
 173         props->max_cqe             = be32_to_cpu(reply->max_cq_depth);
 174         props->max_mr              = be32_to_cpu(reply->max_mrs);
 175         props->max_pd              = be32_to_cpu(reply->max_pds);
 176         props->max_qp_rd_atom      = be32_to_cpu(reply->max_qp_ird);
 177         props->max_ee_rd_atom      = 0;
 178         props->max_res_rd_atom     = be32_to_cpu(reply->max_global_ird);
 179         props->max_qp_init_rd_atom = be32_to_cpu(reply->max_qp_ord);
 180         props->max_ee_init_rd_atom = 0;
 181         props->atomic_cap          = IB_ATOMIC_NONE;
 182         props->max_ee              = 0;
 183         props->max_rdd             = 0;
 184         props->max_mw              = be32_to_cpu(reply->max_mws);
 185         props->max_raw_ipv6_qp     = 0;
 186         props->max_raw_ethy_qp     = 0;
 187         props->max_mcast_grp       = 0;
 188         props->max_mcast_qp_attach = 0;
 189         props->max_total_mcast_qp_attach = 0;
 190         props->max_ah              = 0;
 191         props->max_fmr             = 0;
 192         props->max_map_per_fmr     = 0;
 193         props->max_srq             = 0;
 194         props->max_srq_wr          = 0;
 195         props->max_srq_sge         = 0;
 196         props->max_pkeys           = 0;
 197         props->local_ca_ack_delay  = 0;
 198
 199  bail2:
 200         vq_repbuf_free(c2dev, reply);
 201
 202  bail1:
 203         vq_req_free(c2dev, vq_req);
 204         return err;
 205 }
 206
 207 /*
 208  * Add an IP address to the RNIC interface
 209  */
 210 int c2_add_addr(struct c2_dev *c2dev, u32 inaddr, u32 inmask)
 211 {
 212         struct c2_vq_req *vq_req;
 213         struct c2wr_rnic_setconfig_req *wr;
 214         struct c2wr_rnic_setconfig_rep *reply;
 215         struct c2_netaddr netaddr;
 216         int err, len;
 217
 218         vq_req = vq_req_alloc(c2dev);
 219         if (!vq_req)
 220                 return -ENOMEM;
 221
 222         len = sizeof(struct c2_netaddr);
 223         wr = kmalloc(c2dev->req_vq.msg_size, GFP_KERNEL);
 224         if (!wr) {
 225                 err = -ENOMEM;
 226                 goto bail0;
 227         }
 228
 229         c2_wr_set_id(wr, CCWR_RNIC_SETCONFIG);
 230         wr->hdr.context = (unsigned long) vq_req;
 231         wr->rnic_handle = c2dev->adapter_handle;
 232         wr->option = cpu_to_be32(C2_CFG_ADD_ADDR);
 233
 234         netaddr.ip_addr = inaddr;
 235         netaddr.netmask = inmask;
 236         netaddr.mtu = 0;
 237
 238         memcpy(wr->data, &netaddr, len);
 239
 240         vq_req_get(c2dev, vq_req);
 241
 242         err = vq_send_wr(c2dev, (union c2wr *) wr);
 243         if (err) {
 244                 vq_req_put(c2dev, vq_req);
 245                 goto bail1;
 246         }
 247
 248         err = vq_wait_for_reply(c2dev, vq_req);
 249         if (err)
 250                 goto bail1;
 251
 252         reply =
 253             (struct c2wr_rnic_setconfig_rep *) (unsigned long) (vq_req->reply_msg);
 254         if (!reply) {
 255                 err = -ENOMEM;
 256                 goto bail1;
 257         }
 258
 259         err = c2_errno(reply);
 260         vq_repbuf_free(c2dev, reply);
 261
 262       bail1:
 263         kfree(wr);
 264       bail0:
 265         vq_req_free(c2dev, vq_req);
 266         return err;
 267 }
 268
 269 /*
 270  * Delete an IP address from the RNIC interface
 271  */
 272 int c2_del_addr(struct c2_dev *c2dev, u32 inaddr, u32 inmask)
 273 {
 274         struct c2_vq_req *vq_req;
 275         struct c2wr_rnic_setconfig_req *wr;
 276         struct c2wr_rnic_setconfig_rep *reply;
 277         struct c2_netaddr netaddr;
 278         int err, len;
 279
 280         vq_req = vq_req_alloc(c2dev);
 281         if (!vq_req)
 282                 return -ENOMEM;
 283
 284         len = sizeof(struct c2_netaddr);
 285         wr = kmalloc(c2dev->req_vq.msg_size, GFP_KERNEL);
 286         if (!wr) {
 287                 err = -ENOMEM;
 288                 goto bail0;
 289         }
 290
 291         c2_wr_set_id(wr, CCWR_RNIC_SETCONFIG);
 292         wr->hdr.context = (unsigned long) vq_req;
 293         wr->rnic_handle = c2dev->adapter_handle;
 294         wr->option = cpu_to_be32(C2_CFG_DEL_ADDR);
 295
 296         netaddr.ip_addr = inaddr;
 297         netaddr.netmask = inmask;
 298         netaddr.mtu = 0;
 299
 300         memcpy(wr->data, &netaddr, len);
 301
 302         vq_req_get(c2dev, vq_req);
 303
 304         err = vq_send_wr(c2dev, (union c2wr *) wr);
 305         if (err) {
 306                 vq_req_put(c2dev, vq_req);
 307                 goto bail1;
 308         }
 309
 310         err = vq_wait_for_reply(c2dev, vq_req);
 311         if (err)
 312                 goto bail1;
 313
 314         reply =
 315             (struct c2wr_rnic_setconfig_rep *) (unsigned long) (vq_req->reply_msg);
 316         if (!reply) {
 317                 err = -ENOMEM;
 318                 goto bail1;
 319         }
 320
 321         err = c2_errno(reply);
 322         vq_repbuf_free(c2dev, reply);
 323
 324       bail1:
 325         kfree(wr);
 326       bail0:
 327         vq_req_free(c2dev, vq_req);
 328         return err;
 329 }
 330
 331 /*
 332  * Open a single RNIC instance to use with all
 333  * low level openib calls
 334  */
 335 static int c2_rnic_open(struct c2_dev *c2dev)
 336 {
 337         struct c2_vq_req *vq_req;
 338         union c2wr wr;
 339         struct c2wr_rnic_open_rep *reply;
 340         int err;
 341
 342         vq_req = vq_req_alloc(c2dev);
 343         if (vq_req == NULL) {
 344                 return -ENOMEM;
 345         }
 346
 347         memset(&wr, 0, sizeof(wr));
 348         c2_wr_set_id(&wr, CCWR_RNIC_OPEN);
 349         wr.rnic_open.req.hdr.context = (unsigned long) (vq_req);
 350         wr.rnic_open.req.flags = cpu_to_be16(RNIC_PRIV_MODE);
 351         wr.rnic_open.req.port_num = cpu_to_be16(0);
 352         wr.rnic_open.req.user_context = (unsigned long) c2dev;
 353
 354         vq_req_get(c2dev, vq_req);
 355
 356         err = vq_send_wr(c2dev, &wr);
 357         if (err) {
 358                 vq_req_put(c2dev, vq_req);
 359                 goto bail0;
 360         }
 361
 362         err = vq_wait_for_reply(c2dev, vq_req);
 363         if (err) {
 364                 goto bail0;
 365         }
 366
 367         reply = (struct c2wr_rnic_open_rep *) (unsigned long) (vq_req->reply_msg);
 368         if (!reply) {
 369                 err = -ENOMEM;
 370                 goto bail0;
 371         }
 372
 373         if ((err = c2_errno(reply)) != 0) {
 374                 goto bail1;
 375         }
 376
 377         c2dev->adapter_handle = reply->rnic_handle;
 378
 379       bail1:
 380         vq_repbuf_free(c2dev, reply);
 381       bail0:
 382         vq_req_free(c2dev, vq_req);
 383         return err;
 384 }
 385
 386 /*
 387  * Close the RNIC instance
 388  */
 389 static int c2_rnic_close(struct c2_dev *c2dev)
 390 {
 391         struct c2_vq_req *vq_req;
 392         union c2wr wr;
 393         struct c2wr_rnic_close_rep *reply;
 394         int err;
 395
 396         vq_req = vq_req_alloc(c2dev);
 397         if (vq_req == NULL) {
 398                 return -ENOMEM;
 399         }
 400
 401         memset(&wr, 0, sizeof(wr));
 402         c2_wr_set_id(&wr, CCWR_RNIC_CLOSE);
 403         wr.rnic_close.req.hdr.context = (unsigned long) vq_req;
 404         wr.rnic_close.req.rnic_handle = c2dev->adapter_handle;
 405
 406         vq_req_get(c2dev, vq_req);
 407
 408         err = vq_send_wr(c2dev, &wr);
 409         if (err) {
 410                 vq_req_put(c2dev, vq_req);
 411                 goto bail0;
 412         }
 413
 414         err = vq_wait_for_reply(c2dev, vq_req);
 415         if (err) {
 416                 goto bail0;
 417         }
 418
 419         reply = (struct c2wr_rnic_close_rep *) (unsigned long) (vq_req->reply_msg);
 420         if (!reply) {
 421                 err = -ENOMEM;
 422                 goto bail0;
 423         }
 424
 425         if ((err = c2_errno(reply)) != 0) {
 426                 goto bail1;
 427         }
 428
 429         c2dev->adapter_handle = 0;
 430
 431       bail1:
 432         vq_repbuf_free(c2dev, reply);
 433       bail0:
 434         vq_req_free(c2dev, vq_req);
 435         return err;
 436 }
 437
 438 /*
 439  * Called by c2_probe to initialize the RNIC. This principally
 440  * involves initalizing the various limits and resouce pools that
 441  * comprise the RNIC instance.
 442  */
 443 int c2_rnic_init(struct c2_dev *c2dev)
 444 {
 445         int err;
 446         u32 qsize, msgsize;
 447         void *q1_pages;
 448         void *q2_pages;
 449         void __iomem *mmio_regs;
 450
 451         /* Device capabilities */
 452         c2dev->device_cap_flags =
 453             (IB_DEVICE_RESIZE_MAX_WR |
 454              IB_DEVICE_CURR_QP_STATE_MOD |
 455              IB_DEVICE_SYS_IMAGE_GUID |
 456              IB_DEVICE_ZERO_STAG |
 457              IB_DEVICE_SEND_W_INV | IB_DEVICE_MEM_WINDOW);
 458
 459         /* Allocate the qptr_array */
 460         c2dev->qptr_array = vmalloc(C2_MAX_CQS * sizeof(void *));
 461         if (!c2dev->qptr_array) {
 462                 return -ENOMEM;
 463         }
 464
 465         /* Inialize the qptr_array */
 466         memset(c2dev->qptr_array, 0, C2_MAX_CQS * sizeof(void *));
 467         c2dev->qptr_array[0] = (void *) &c2dev->req_vq;
 468         c2dev->qptr_array[1] = (void *) &c2dev->rep_vq;
 469         c2dev->qptr_array[2] = (void *) &c2dev->aeq;
 470
 471         /* Initialize data structures */
 472         init_waitqueue_head(&c2dev->req_vq_wo);
 473         spin_lock_init(&c2dev->vqlock);
 474         spin_lock_init(&c2dev->lock);
 475
 476         /* Allocate MQ shared pointer pool for kernel clients. User
 477          * mode client pools are hung off the user context
 478          */
 479         err = c2_init_mqsp_pool(c2dev, GFP_KERNEL, &c2dev->kern_mqsp_pool);
 480         if (err) {
 481                 goto bail0;
 482         }
 483
 484         /* Allocate shared pointers for Q0, Q1, and Q2 from
 485          * the shared pointer pool.
 486          */
 487
 488         c2dev->hint_count = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
 489                                              &c2dev->hint_count_dma,
 490                                              GFP_KERNEL);
 491         c2dev->req_vq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
 492                                              &c2dev->req_vq.shared_dma,
 493                                              GFP_KERNEL);
 494         c2dev->rep_vq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
 495                                              &c2dev->rep_vq.shared_dma,
 496                                              GFP_KERNEL);
 497         c2dev->aeq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
 498                                           &c2dev->aeq.shared_dma, GFP_KERNEL);
 499         if (!c2dev->hint_count || !c2dev->req_vq.shared ||
 500             !c2dev->rep_vq.shared || !c2dev->aeq.shared) {
 501                 err = -ENOMEM;
 502                 goto bail1;
 503         }
 504
 505         mmio_regs = c2dev->kva;
 506         /* Initialize the Verbs Request Queue */
 507         c2_mq_req_init(&c2dev->req_vq, 0,
 508                        be32_to_cpu(readl(mmio_regs + C2_REGS_Q0_QSIZE)),
 509                        be32_to_cpu(readl(mmio_regs + C2_REGS_Q0_MSGSIZE)),
 510                        mmio_regs +
 511                        be32_to_cpu(readl(mmio_regs + C2_REGS_Q0_POOLSTART)),
 512                        mmio_regs +
 513                        be32_to_cpu(readl(mmio_regs + C2_REGS_Q0_SHARED)),
 514                        C2_MQ_ADAPTER_TARGET);
 515
 516         /* Initialize the Verbs Reply Queue */
 517         qsize = be32_to_cpu(readl(mmio_regs + C2_REGS_Q1_QSIZE));
 518         msgsize = be32_to_cpu(readl(mmio_regs + C2_REGS_Q1_MSGSIZE));
 519         q1_pages = kmalloc(qsize * msgsize, GFP_KERNEL);
 520         if (!q1_pages) {
 521                 err = -ENOMEM;
 522                 goto bail1;
 523         }
 524         c2dev->rep_vq.host_dma = dma_map_single(c2dev->ibdev.dma_device,
 525                                                 (void *)q1_pages, qsize * msgsize,
 526                                                 DMA_FROM_DEVICE);
 527         pci_unmap_addr_set(&c2dev->rep_vq, mapping, c2dev->rep_vq.host_dma);
 528         pr_debug("%s rep_vq va %p dma %llx\n", __FUNCTION__, q1_pages,
 529                  (u64)c2dev->rep_vq.host_dma);
 530         c2_mq_rep_init(&c2dev->rep_vq,
 531                    1,
 532                    qsize,
 533                    msgsize,
 534                    q1_pages,
 535                    mmio_regs +
 536                    be32_to_cpu(readl(mmio_regs + C2_REGS_Q1_SHARED)),
 537                    C2_MQ_HOST_TARGET);
 538
 539         /* Initialize the Asynchronus Event Queue */
 540         qsize = be32_to_cpu(readl(mmio_regs + C2_REGS_Q2_QSIZE));
 541         msgsize = be32_to_cpu(readl(mmio_regs + C2_REGS_Q2_MSGSIZE));
 542         q2_pages = kmalloc(qsize * msgsize, GFP_KERNEL);
 543         if (!q2_pages) {
 544                 err = -ENOMEM;
 545                 goto bail2;
 546         }
 547         c2dev->aeq.host_dma = dma_map_single(c2dev->ibdev.dma_device,
 548                                                 (void *)q2_pages, qsize * msgsize,
 549                                                 DMA_FROM_DEVICE);
 550         pci_unmap_addr_set(&c2dev->aeq, mapping, c2dev->aeq.host_dma);
 551         pr_debug("%s aeq va %p dma %llx\n", __FUNCTION__, q1_pages,
 552                  (u64)c2dev->rep_vq.host_dma);
 553         c2_mq_rep_init(&c2dev->aeq,
 554                        2,
 555                        qsize,
 556                        msgsize,
 557                        q2_pages,
 558                        mmio_regs +
 559                        be32_to_cpu(readl(mmio_regs + C2_REGS_Q2_SHARED)),
 560                        C2_MQ_HOST_TARGET);
 561
 562         /* Initialize the verbs request allocator */
 563         err = vq_init(c2dev);
 564         if (err)
 565                 goto bail3;
 566
 567         /* Enable interrupts on the adapter */
 568         writel(0, c2dev->regs + C2_IDIS);
 569
 570         /* create the WR init message */
 571         err = c2_adapter_init(c2dev);
 572         if (err)
 573                 goto bail4;
 574         c2dev->init++;
 575
 576         /* open an adapter instance */
 577         err = c2_rnic_open(c2dev);
 578         if (err)
 579                 goto bail4;
 580
 581         /* Initialize cached the adapter limits */
 582         if (c2_rnic_query(c2dev, &c2dev->props))
 583                 goto bail5;
 584
 585         /* Initialize the PD pool */
 586         err = c2_init_pd_table(c2dev);
 587         if (err)
 588                 goto bail5;
 589
 590         /* Initialize the QP pool */
 591         c2_init_qp_table(c2dev);
 592         return 0;
 593
 594       bail5:
 595         c2_rnic_close(c2dev);
 596       bail4:
 597         vq_term(c2dev);
 598       bail3:
 599         dma_unmap_single(c2dev->ibdev.dma_device,
 600                          pci_unmap_addr(&c2dev->aeq, mapping),
 601                          c2dev->aeq.q_size * c2dev->aeq.msg_size,
 602                          DMA_FROM_DEVICE);
 603         kfree(q2_pages);
 604       bail2:
 605         dma_unmap_single(c2dev->ibdev.dma_device,
 606                          pci_unmap_addr(&c2dev->rep_vq, mapping),
 607                          c2dev->rep_vq.q_size * c2dev->rep_vq.msg_size,
 608                          DMA_FROM_DEVICE);
 609         kfree(q1_pages);
 610       bail1:
 611         c2_free_mqsp_pool(c2dev, c2dev->kern_mqsp_pool);
 612       bail0:
 613         vfree(c2dev->qptr_array);
 614
 615         return err;
 616 }
 617
 618 /*
 619  * Called by c2_remove to cleanup the RNIC resources.
 620  */
 621 void c2_rnic_term(struct c2_dev *c2dev)
 622 {
 623
 624         /* Close the open adapter instance */
 625         c2_rnic_close(c2dev);
 626
 627         /* Send the TERM message to the adapter */
 628         c2_adapter_term(c2dev);
 629
 630         /* Disable interrupts on the adapter */
 631         writel(1, c2dev->regs + C2_IDIS);
 632
 633         /* Free the QP pool */
 634         c2_cleanup_qp_table(c2dev);
 635
 636         /* Free the PD pool */
 637         c2_cleanup_pd_table(c2dev);
 638
 639         /* Free the verbs request allocator */
 640         vq_term(c2dev);
 641
 642         /* Unmap and free the asynchronus event queue */
 643         dma_unmap_single(c2dev->ibdev.dma_device,
 644                          pci_unmap_addr(&c2dev->aeq, mapping),
 645                          c2dev->aeq.q_size * c2dev->aeq.msg_size,
 646                          DMA_FROM_DEVICE);
 647         kfree(c2dev->aeq.msg_pool.host);
 648
 649         /* Unmap and free the verbs reply queue */
 650         dma_unmap_single(c2dev->ibdev.dma_device,
 651                          pci_unmap_addr(&c2dev->rep_vq, mapping),
 652                          c2dev->rep_vq.q_size * c2dev->rep_vq.msg_size,
 653                          DMA_FROM_DEVICE);
 654         kfree(c2dev->rep_vq.msg_pool.host);
 655
 656         /* Free the MQ shared pointer pool */
 657         c2_free_mqsp_pool(c2dev, c2dev->kern_mqsp_pool);
 658
 659         /* Free the qptr_array */
 660         vfree(c2dev->qptr_array);
 661
 662         return;
 663 }