drivers/infiniband/core/cma.c

   1 /*
   2  * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
   3  * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
   4  * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
   5  * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * OpenIB.org BSD license below:
  12  *
  13  *     Redistribution and use in source and binary forms, with or
  14  *     without modification, are permitted provided that the following
  15  *     conditions are met:
  16  *
  17  *      - Redistributions of source code must retain the above
  18  *        copyright notice, this list of conditions and the following
  19  *        disclaimer.
  20  *
  21  *      - Redistributions in binary form must reproduce the above
  22  *        copyright notice, this list of conditions and the following
  23  *        disclaimer in the documentation and/or other materials
  24  *        provided with the distribution.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33  * SOFTWARE.
  34  */
  35
  36 #include <linux/completion.h>
  37 #include <linux/in.h>
  38 #include <linux/in6.h>
  39 #include <linux/mutex.h>
  40 #include <linux/random.h>
  41 #include <linux/idr.h>
  42 #include <linux/inetdevice.h>
  43 #include <linux/slab.h>
  44 #include <linux/module.h>
  45 #include <net/route.h>
  46
  47 #include <net/tcp.h>
  48 #include <net/ipv6.h>
  49
  50 #include <rdma/rdma_cm.h>
  51 #include <rdma/rdma_cm_ib.h>
  52 #include <rdma/rdma_netlink.h>
  53 #include <rdma/ib_cache.h>
  54 #include <rdma/ib_cm.h>
  55 #include <rdma/ib_sa.h>
  56 #include <rdma/iw_cm.h>
  57
  58 MODULE_AUTHOR("Sean Hefty");
  59 MODULE_DESCRIPTION("Generic RDMA CM Agent");
  60 MODULE_LICENSE("Dual BSD/GPL");
  61
  62 #define CMA_CM_RESPONSE_TIMEOUT 20
  63 #define CMA_MAX_CM_RETRIES 15
  64 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
  65 #define CMA_IBOE_PACKET_LIFETIME 18
  66
  67 static void cma_add_one(struct ib_device *device);
  68 static void cma_remove_one(struct ib_device *device);
  69
  70 static struct ib_client cma_client = {
  71         .name   = "cma",
  72         .add    = cma_add_one,
  73         .remove = cma_remove_one
  74 };
  75
  76 static struct ib_sa_client sa_client;
  77 static struct rdma_addr_client addr_client;
  78 static LIST_HEAD(dev_list);
  79 static LIST_HEAD(listen_any_list);
  80 static DEFINE_MUTEX(lock);
  81 static struct workqueue_struct *cma_wq;
  82 static DEFINE_IDR(sdp_ps);
  83 static DEFINE_IDR(tcp_ps);
  84 static DEFINE_IDR(udp_ps);
  85 static DEFINE_IDR(ipoib_ps);
  86 static DEFINE_IDR(ib_ps);
  87
  88 struct cma_device {
  89         struct list_head        list;
  90         struct ib_device        *device;
  91         struct completion       comp;
  92         atomic_t                refcount;
  93         struct list_head        id_list;
  94 };
  95
  96 struct rdma_bind_list {
  97         struct idr              *ps;
  98         struct hlist_head       owners;
  99         unsigned short          port;
 100 };
 101
 102 enum {
 103         CMA_OPTION_AFONLY,
 104 };
 105
 106 /*
 107  * Device removal can occur at anytime, so we need extra handling to
 108  * serialize notifying the user of device removal with other callbacks.
 109  * We do this by disabling removal notification while a callback is in process,
 110  * and reporting it after the callback completes.
 111  */
 112 struct rdma_id_private {
 113         struct rdma_cm_id       id;
 114
 115         struct rdma_bind_list   *bind_list;
 116         struct hlist_node       node;
 117         struct list_head        list; /* listen_any_list or cma_device.list */
 118         struct list_head        listen_list; /* per device listens */
 119         struct cma_device       *cma_dev;
 120         struct list_head        mc_list;
 121
 122         int                     internal_id;
 123         enum rdma_cm_state      state;
 124         spinlock_t              lock;
 125         struct mutex            qp_mutex;
 126
 127         struct completion       comp;
 128         atomic_t                refcount;
 129         struct mutex            handler_mutex;
 130
 131         int                     backlog;
 132         int                     timeout_ms;
 133         struct ib_sa_query      *query;
 134         int                     query_id;
 135         union {
 136                 struct ib_cm_id *ib;
 137                 struct iw_cm_id *iw;
 138         } cm_id;
 139
 140         u32                     seq_num;
 141         u32                     qkey;
 142         u32                     qp_num;
 143         pid_t                   owner;
 144         u32                     options;
 145         u8                      srq;
 146         u8                      tos;
 147         u8                      reuseaddr;
 148         u8                      afonly;
 149 };
 150
 151 struct cma_multicast {
 152         struct rdma_id_private *id_priv;
 153         union {
 154                 struct ib_sa_multicast *ib;
 155         } multicast;
 156         struct list_head        list;
 157         void                    *context;
 158         struct sockaddr_storage addr;
 159         struct kref             mcref;
 160 };
 161
 162 struct cma_work {
 163         struct work_struct      work;
 164         struct rdma_id_private  *id;
 165         enum rdma_cm_state      old_state;
 166         enum rdma_cm_state      new_state;
 167         struct rdma_cm_event    event;
 168 };
 169
 170 struct cma_ndev_work {
 171         struct work_struct      work;
 172         struct rdma_id_private  *id;
 173         struct rdma_cm_event    event;
 174 };
 175
 176 struct iboe_mcast_work {
 177         struct work_struct       work;
 178         struct rdma_id_private  *id;
 179         struct cma_multicast    *mc;
 180 };
 181
 182 union cma_ip_addr {
 183         struct in6_addr ip6;
 184         struct {
 185                 __be32 pad[3];
 186                 __be32 addr;
 187         } ip4;
 188 };
 189
 190 struct cma_hdr {
 191         u8 cma_version;
 192         u8 ip_version;  /* IP version: 7:4 */
 193         __be16 port;
 194         union cma_ip_addr src_addr;
 195         union cma_ip_addr dst_addr;
 196 };
 197
 198 struct sdp_hh {
 199         u8 bsdh[16];
 200         u8 sdp_version; /* Major version: 7:4 */
 201         u8 ip_version;  /* IP version: 7:4 */
 202         u8 sdp_specific1[10];
 203         __be16 port;
 204         __be16 sdp_specific2;
 205         union cma_ip_addr src_addr;
 206         union cma_ip_addr dst_addr;
 207 };
 208
 209 struct sdp_hah {
 210         u8 bsdh[16];
 211         u8 sdp_version;
 212 };
 213
 214 #define CMA_VERSION 0x00
 215 #define SDP_MAJ_VERSION 0x2
 216
 217 static int cma_comp(struct rdma_id_private *id_priv, enum rdma_cm_state comp)
 218 {
 219         unsigned long flags;
 220         int ret;
 221
 222         spin_lock_irqsave(&id_priv->lock, flags);
 223         ret = (id_priv->state == comp);
 224         spin_unlock_irqrestore(&id_priv->lock, flags);
 225         return ret;
 226 }
 227
 228 static int cma_comp_exch(struct rdma_id_private *id_priv,
 229                          enum rdma_cm_state comp, enum rdma_cm_state exch)
 230 {
 231         unsigned long flags;
 232         int ret;
 233
 234         spin_lock_irqsave(&id_priv->lock, flags);
 235         if ((ret = (id_priv->state == comp)))
 236                 id_priv->state = exch;
 237         spin_unlock_irqrestore(&id_priv->lock, flags);
 238         return ret;
 239 }
 240
 241 static enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv,
 242                                    enum rdma_cm_state exch)
 243 {
 244         unsigned long flags;
 245         enum rdma_cm_state old;
 246
 247         spin_lock_irqsave(&id_priv->lock, flags);
 248         old = id_priv->state;
 249         id_priv->state = exch;
 250         spin_unlock_irqrestore(&id_priv->lock, flags);
 251         return old;
 252 }
 253
 254 static inline u8 cma_get_ip_ver(struct cma_hdr *hdr)
 255 {
 256         return hdr->ip_version >> 4;
 257 }
 258
 259 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
 260 {
 261         hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
 262 }
 263
 264 static inline u8 sdp_get_majv(u8 sdp_version)
 265 {
 266         return sdp_version >> 4;
 267 }
 268
 269 static inline u8 sdp_get_ip_ver(struct sdp_hh *hh)
 270 {
 271         return hh->ip_version >> 4;
 272 }
 273
 274 static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver)
 275 {
 276         hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF);
 277 }
 278
 279 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
 280                               struct cma_device *cma_dev)
 281 {
 282         atomic_inc(&cma_dev->refcount);
 283         id_priv->cma_dev = cma_dev;
 284         id_priv->id.device = cma_dev->device;
 285         id_priv->id.route.addr.dev_addr.transport =
 286                 rdma_node_get_transport(cma_dev->device->node_type);
 287         list_add_tail(&id_priv->list, &cma_dev->id_list);
 288 }
 289
 290 static inline void cma_deref_dev(struct cma_device *cma_dev)
 291 {
 292         if (atomic_dec_and_test(&cma_dev->refcount))
 293                 complete(&cma_dev->comp);
 294 }
 295
 296 static inline void release_mc(struct kref *kref)
 297 {
 298         struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref);
 299
 300         kfree(mc->multicast.ib);
 301         kfree(mc);
 302 }
 303
 304 static void cma_release_dev(struct rdma_id_private *id_priv)
 305 {
 306         mutex_lock(&lock);
 307         list_del(&id_priv->list);
 308         cma_deref_dev(id_priv->cma_dev);
 309         id_priv->cma_dev = NULL;
 310         mutex_unlock(&lock);
 311 }
 312
 313 static int cma_set_qkey(struct rdma_id_private *id_priv)
 314 {
 315         struct ib_sa_mcmember_rec rec;
 316         int ret = 0;
 317
 318         if (id_priv->qkey)
 319                 return 0;
 320
 321         switch (id_priv->id.ps) {
 322         case RDMA_PS_UDP:
 323                 id_priv->qkey = RDMA_UDP_QKEY;
 324                 break;
 325         case RDMA_PS_IPOIB:
 326                 ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
 327                 ret = ib_sa_get_mcmember_rec(id_priv->id.device,
 328                                              id_priv->id.port_num, &rec.mgid,
 329                                              &rec);
 330                 if (!ret)
 331                         id_priv->qkey = be32_to_cpu(rec.qkey);
 332                 break;
 333         default:
 334                 break;
 335         }
 336         return ret;
 337 }
 338
 339 static int find_gid_port(struct ib_device *device, union ib_gid *gid, u8 port_num)
 340 {
 341         int i;
 342         int err;
 343         struct ib_port_attr props;
 344         union ib_gid tmp;
 345
 346         err = ib_query_port(device, port_num, &props);
 347         if (err)
 348                 return 1;
 349
 350         for (i = 0; i < props.gid_tbl_len; ++i) {
 351                 err = ib_query_gid(device, port_num, i, &tmp);
 352                 if (err)
 353                         return 1;
 354                 if (!memcmp(&tmp, gid, sizeof tmp))
 355                         return 0;
 356         }
 357
 358         return -EAGAIN;
 359 }
 360
 361 static int cma_acquire_dev(struct rdma_id_private *id_priv)
 362 {
 363         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 364         struct cma_device *cma_dev;
 365         union ib_gid gid, iboe_gid;
 366         int ret = -ENODEV;
 367         u8 port;
 368         enum rdma_link_layer dev_ll = dev_addr->dev_type == ARPHRD_INFINIBAND ?
 369                 IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET;
 370
 371         if (dev_ll != IB_LINK_LAYER_INFINIBAND &&
 372             id_priv->id.ps == RDMA_PS_IPOIB)
 373                 return -EINVAL;
 374
 375         mutex_lock(&lock);
 376         iboe_addr_get_sgid(dev_addr, &iboe_gid);
 377         memcpy(&gid, dev_addr->src_dev_addr +
 378                rdma_addr_gid_offset(dev_addr), sizeof gid);
 379         list_for_each_entry(cma_dev, &dev_list, list) {
 380                 for (port = 1; port <= cma_dev->device->phys_port_cnt; ++port) {
 381                         if (rdma_port_get_link_layer(cma_dev->device, port) == dev_ll) {
 382                                 if (rdma_node_get_transport(cma_dev->device->node_type) == RDMA_TRANSPORT_IB &&
 383                                     rdma_port_get_link_layer(cma_dev->device, port) == IB_LINK_LAYER_ETHERNET)
 384                                         ret = find_gid_port(cma_dev->device, &iboe_gid, port);
 385                                 else
 386                                         ret = find_gid_port(cma_dev->device, &gid, port);
 387
 388                                 if (!ret) {
 389                                         id_priv->id.port_num = port;
 390                                         goto out;
 391                                 } else if (ret == 1)
 392                                         break;
 393                         }
 394                 }
 395         }
 396
 397 out:
 398         if (!ret)
 399                 cma_attach_to_dev(id_priv, cma_dev);
 400
 401         mutex_unlock(&lock);
 402         return ret;
 403 }
 404
 405 static void cma_deref_id(struct rdma_id_private *id_priv)
 406 {
 407         if (atomic_dec_and_test(&id_priv->refcount))
 408                 complete(&id_priv->comp);
 409 }
 410
 411 static int cma_disable_callback(struct rdma_id_private *id_priv,
 412                                 enum rdma_cm_state state)
 413 {
 414         mutex_lock(&id_priv->handler_mutex);
 415         if (id_priv->state != state) {
 416                 mutex_unlock(&id_priv->handler_mutex);
 417                 return -EINVAL;
 418         }
 419         return 0;
 420 }
 421
 422 struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler,
 423                                   void *context, enum rdma_port_space ps,
 424                                   enum ib_qp_type qp_type)
 425 {
 426         struct rdma_id_private *id_priv;
 427
 428         id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
 429         if (!id_priv)
 430                 return ERR_PTR(-ENOMEM);
 431
 432         id_priv->owner = task_pid_nr(current);
 433         id_priv->state = RDMA_CM_IDLE;
 434         id_priv->id.context = context;
 435         id_priv->id.event_handler = event_handler;
 436         id_priv->id.ps = ps;
 437         id_priv->id.qp_type = qp_type;
 438         spin_lock_init(&id_priv->lock);
 439         mutex_init(&id_priv->qp_mutex);
 440         init_completion(&id_priv->comp);
 441         atomic_set(&id_priv->refcount, 1);
 442         mutex_init(&id_priv->handler_mutex);
 443         INIT_LIST_HEAD(&id_priv->listen_list);
 444         INIT_LIST_HEAD(&id_priv->mc_list);
 445         get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
 446
 447         return &id_priv->id;
 448 }
 449 EXPORT_SYMBOL(rdma_create_id);
 450
 451 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 452 {
 453         struct ib_qp_attr qp_attr;
 454         int qp_attr_mask, ret;
 455
 456         qp_attr.qp_state = IB_QPS_INIT;
 457         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 458         if (ret)
 459                 return ret;
 460
 461         ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 462         if (ret)
 463                 return ret;
 464
 465         qp_attr.qp_state = IB_QPS_RTR;
 466         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
 467         if (ret)
 468                 return ret;
 469
 470         qp_attr.qp_state = IB_QPS_RTS;
 471         qp_attr.sq_psn = 0;
 472         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
 473
 474         return ret;
 475 }
 476
 477 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 478 {
 479         struct ib_qp_attr qp_attr;
 480         int qp_attr_mask, ret;
 481
 482         qp_attr.qp_state = IB_QPS_INIT;
 483         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 484         if (ret)
 485                 return ret;
 486
 487         return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 488 }
 489
 490 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
 491                    struct ib_qp_init_attr *qp_init_attr)
 492 {
 493         struct rdma_id_private *id_priv;
 494         struct ib_qp *qp;
 495         int ret;
 496
 497         id_priv = container_of(id, struct rdma_id_private, id);
 498         if (id->device != pd->device)
 499                 return -EINVAL;
 500
 501         qp = ib_create_qp(pd, qp_init_attr);
 502         if (IS_ERR(qp))
 503                 return PTR_ERR(qp);
 504
 505         if (id->qp_type == IB_QPT_UD)
 506                 ret = cma_init_ud_qp(id_priv, qp);
 507         else
 508                 ret = cma_init_conn_qp(id_priv, qp);
 509         if (ret)
 510                 goto err;
 511
 512         id->qp = qp;
 513         id_priv->qp_num = qp->qp_num;
 514         id_priv->srq = (qp->srq != NULL);
 515         return 0;
 516 err:
 517         ib_destroy_qp(qp);
 518         return ret;
 519 }
 520 EXPORT_SYMBOL(rdma_create_qp);
 521
 522 void rdma_destroy_qp(struct rdma_cm_id *id)
 523 {
 524         struct rdma_id_private *id_priv;
 525
 526         id_priv = container_of(id, struct rdma_id_private, id);
 527         mutex_lock(&id_priv->qp_mutex);
 528         ib_destroy_qp(id_priv->id.qp);
 529         id_priv->id.qp = NULL;
 530         mutex_unlock(&id_priv->qp_mutex);
 531 }
 532 EXPORT_SYMBOL(rdma_destroy_qp);
 533
 534 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
 535                              struct rdma_conn_param *conn_param)
 536 {
 537         struct ib_qp_attr qp_attr;
 538         int qp_attr_mask, ret;
 539
 540         mutex_lock(&id_priv->qp_mutex);
 541         if (!id_priv->id.qp) {
 542                 ret = 0;
 543                 goto out;
 544         }
 545
 546         /* Need to update QP attributes from default values. */
 547         qp_attr.qp_state = IB_QPS_INIT;
 548         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 549         if (ret)
 550                 goto out;
 551
 552         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 553         if (ret)
 554                 goto out;
 555
 556         qp_attr.qp_state = IB_QPS_RTR;
 557         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 558         if (ret)
 559                 goto out;
 560
 561         if (conn_param)
 562                 qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
 563         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 564 out:
 565         mutex_unlock(&id_priv->qp_mutex);
 566         return ret;
 567 }
 568
 569 static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
 570                              struct rdma_conn_param *conn_param)
 571 {
 572         struct ib_qp_attr qp_attr;
 573         int qp_attr_mask, ret;
 574
 575         mutex_lock(&id_priv->qp_mutex);
 576         if (!id_priv->id.qp) {
 577                 ret = 0;
 578                 goto out;
 579         }
 580
 581         qp_attr.qp_state = IB_QPS_RTS;
 582         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 583         if (ret)
 584                 goto out;
 585
 586         if (conn_param)
 587                 qp_attr.max_rd_atomic = conn_param->initiator_depth;
 588         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 589 out:
 590         mutex_unlock(&id_priv->qp_mutex);
 591         return ret;
 592 }
 593
 594 static int cma_modify_qp_err(struct rdma_id_private *id_priv)
 595 {
 596         struct ib_qp_attr qp_attr;
 597         int ret;
 598
 599         mutex_lock(&id_priv->qp_mutex);
 600         if (!id_priv->id.qp) {
 601                 ret = 0;
 602                 goto out;
 603         }
 604
 605         qp_attr.qp_state = IB_QPS_ERR;
 606         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
 607 out:
 608         mutex_unlock(&id_priv->qp_mutex);
 609         return ret;
 610 }
 611
 612 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
 613                                struct ib_qp_attr *qp_attr, int *qp_attr_mask)
 614 {
 615         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 616         int ret;
 617         u16 pkey;
 618
 619         if (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num) ==
 620             IB_LINK_LAYER_INFINIBAND)
 621                 pkey = ib_addr_get_pkey(dev_addr);
 622         else
 623                 pkey = 0xffff;
 624
 625         ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
 626                                   pkey, &qp_attr->pkey_index);
 627         if (ret)
 628                 return ret;
 629
 630         qp_attr->port_num = id_priv->id.port_num;
 631         *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
 632
 633         if (id_priv->id.qp_type == IB_QPT_UD) {
 634                 ret = cma_set_qkey(id_priv);
 635                 if (ret)
 636                         return ret;
 637
 638                 qp_attr->qkey = id_priv->qkey;
 639                 *qp_attr_mask |= IB_QP_QKEY;
 640         } else {
 641                 qp_attr->qp_access_flags = 0;
 642                 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
 643         }
 644         return 0;
 645 }
 646
 647 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
 648                        int *qp_attr_mask)
 649 {
 650         struct rdma_id_private *id_priv;
 651         int ret = 0;
 652
 653         id_priv = container_of(id, struct rdma_id_private, id);
 654         switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
 655         case RDMA_TRANSPORT_IB:
 656                 if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD))
 657                         ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
 658                 else
 659                         ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
 660                                                  qp_attr_mask);
 661                 if (qp_attr->qp_state == IB_QPS_RTR)
 662                         qp_attr->rq_psn = id_priv->seq_num;
 663                 break;
 664         case RDMA_TRANSPORT_IWARP:
 665                 if (!id_priv->cm_id.iw) {
 666                         qp_attr->qp_access_flags = 0;
 667                         *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
 668                 } else
 669                         ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
 670                                                  qp_attr_mask);
 671                 break;
 672         default:
 673                 ret = -ENOSYS;
 674                 break;
 675         }
 676
 677         return ret;
 678 }
 679 EXPORT_SYMBOL(rdma_init_qp_attr);
 680
 681 static inline int cma_zero_addr(struct sockaddr *addr)
 682 {
 683         struct in6_addr *ip6;
 684
 685         if (addr->sa_family == AF_INET)
 686                 return ipv4_is_zeronet(
 687                         ((struct sockaddr_in *)addr)->sin_addr.s_addr);
 688         else {
 689                 ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr;
 690                 return (ip6->s6_addr32[0] | ip6->s6_addr32[1] |
 691                         ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0;
 692         }
 693 }
 694
 695 static inline int cma_loopback_addr(struct sockaddr *addr)
 696 {
 697         if (addr->sa_family == AF_INET)
 698                 return ipv4_is_loopback(
 699                         ((struct sockaddr_in *) addr)->sin_addr.s_addr);
 700         else
 701                 return ipv6_addr_loopback(
 702                         &((struct sockaddr_in6 *) addr)->sin6_addr);
 703 }
 704
 705 static inline int cma_any_addr(struct sockaddr *addr)
 706 {
 707         return cma_zero_addr(addr) || cma_loopback_addr(addr);
 708 }
 709
 710 static int cma_addr_cmp(struct sockaddr *src, struct sockaddr *dst)
 711 {
 712         if (src->sa_family != dst->sa_family)
 713                 return -1;
 714
 715         switch (src->sa_family) {
 716         case AF_INET:
 717                 return ((struct sockaddr_in *) src)->sin_addr.s_addr !=
 718                        ((struct sockaddr_in *) dst)->sin_addr.s_addr;
 719         default:
 720                 return ipv6_addr_cmp(&((struct sockaddr_in6 *) src)->sin6_addr,
 721                                      &((struct sockaddr_in6 *) dst)->sin6_addr);
 722         }
 723 }
 724
 725 static inline __be16 cma_port(struct sockaddr *addr)
 726 {
 727         if (addr->sa_family == AF_INET)
 728                 return ((struct sockaddr_in *) addr)->sin_port;
 729         else
 730                 return ((struct sockaddr_in6 *) addr)->sin6_port;
 731 }
 732
 733 static inline int cma_any_port(struct sockaddr *addr)
 734 {
 735         return !cma_port(addr);
 736 }
 737
 738 static int cma_get_net_info(void *hdr, enum rdma_port_space ps,
 739                             u8 *ip_ver, __be16 *port,
 740                             union cma_ip_addr **src, union cma_ip_addr **dst)
 741 {
 742         switch (ps) {
 743         case RDMA_PS_SDP:
 744                 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) !=
 745                     SDP_MAJ_VERSION)
 746                         return -EINVAL;
 747
 748                 *ip_ver = sdp_get_ip_ver(hdr);
 749                 *port   = ((struct sdp_hh *) hdr)->port;
 750                 *src    = &((struct sdp_hh *) hdr)->src_addr;
 751                 *dst    = &((struct sdp_hh *) hdr)->dst_addr;
 752                 break;
 753         default:
 754                 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION)
 755                         return -EINVAL;
 756
 757                 *ip_ver = cma_get_ip_ver(hdr);
 758                 *port   = ((struct cma_hdr *) hdr)->port;
 759                 *src    = &((struct cma_hdr *) hdr)->src_addr;
 760                 *dst    = &((struct cma_hdr *) hdr)->dst_addr;
 761                 break;
 762         }
 763
 764         if (*ip_ver != 4 && *ip_ver != 6)
 765                 return -EINVAL;
 766         return 0;
 767 }
 768
 769 static void cma_save_net_info(struct rdma_addr *addr,
 770                               struct rdma_addr *listen_addr,
 771                               u8 ip_ver, __be16 port,
 772                               union cma_ip_addr *src, union cma_ip_addr *dst)
 773 {
 774         struct sockaddr_in *listen4, *ip4;
 775         struct sockaddr_in6 *listen6, *ip6;
 776
 777         switch (ip_ver) {
 778         case 4:
 779                 listen4 = (struct sockaddr_in *) &listen_addr->src_addr;
 780                 ip4 = (struct sockaddr_in *) &addr->src_addr;
 781                 ip4->sin_family = listen4->sin_family;
 782                 ip4->sin_addr.s_addr = dst->ip4.addr;
 783                 ip4->sin_port = listen4->sin_port;
 784
 785                 ip4 = (struct sockaddr_in *) &addr->dst_addr;
 786                 ip4->sin_family = listen4->sin_family;
 787                 ip4->sin_addr.s_addr = src->ip4.addr;
 788                 ip4->sin_port = port;
 789                 break;
 790         case 6:
 791                 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr;
 792                 ip6 = (struct sockaddr_in6 *) &addr->src_addr;
 793                 ip6->sin6_family = listen6->sin6_family;
 794                 ip6->sin6_addr = dst->ip6;
 795                 ip6->sin6_port = listen6->sin6_port;
 796
 797                 ip6 = (struct sockaddr_in6 *) &addr->dst_addr;
 798                 ip6->sin6_family = listen6->sin6_family;
 799                 ip6->sin6_addr = src->ip6;
 800                 ip6->sin6_port = port;
 801                 break;
 802         default:
 803                 break;
 804         }
 805 }
 806
 807 static inline int cma_user_data_offset(enum rdma_port_space ps)
 808 {
 809         switch (ps) {
 810         case RDMA_PS_SDP:
 811                 return 0;
 812         default:
 813                 return sizeof(struct cma_hdr);
 814         }
 815 }
 816
 817 static void cma_cancel_route(struct rdma_id_private *id_priv)
 818 {
 819         switch (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num)) {
 820         case IB_LINK_LAYER_INFINIBAND:
 821                 if (id_priv->query)
 822                         ib_sa_cancel_query(id_priv->query_id, id_priv->query);
 823                 break;
 824         default:
 825                 break;
 826         }
 827 }
 828
 829 static void cma_cancel_listens(struct rdma_id_private *id_priv)
 830 {
 831         struct rdma_id_private *dev_id_priv;
 832
 833         /*
 834          * Remove from listen_any_list to prevent added devices from spawning
 835          * additional listen requests.
 836          */
 837         mutex_lock(&lock);
 838         list_del(&id_priv->list);
 839
 840         while (!list_empty(&id_priv->listen_list)) {
 841                 dev_id_priv = list_entry(id_priv->listen_list.next,
 842                                          struct rdma_id_private, listen_list);
 843                 /* sync with device removal to avoid duplicate destruction */
 844                 list_del_init(&dev_id_priv->list);
 845                 list_del(&dev_id_priv->listen_list);
 846                 mutex_unlock(&lock);
 847
 848                 rdma_destroy_id(&dev_id_priv->id);
 849                 mutex_lock(&lock);
 850         }
 851         mutex_unlock(&lock);
 852 }
 853
 854 static void cma_cancel_operation(struct rdma_id_private *id_priv,
 855                                  enum rdma_cm_state state)
 856 {
 857         switch (state) {
 858         case RDMA_CM_ADDR_QUERY:
 859                 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
 860                 break;
 861         case RDMA_CM_ROUTE_QUERY:
 862                 cma_cancel_route(id_priv);
 863                 break;
 864         case RDMA_CM_LISTEN:
 865                 if (cma_any_addr((struct sockaddr *) &id_priv->id.route.addr.src_addr)
 866                                 && !id_priv->cma_dev)
 867                         cma_cancel_listens(id_priv);
 868                 break;
 869         default:
 870                 break;
 871         }
 872 }
 873
 874 static void cma_release_port(struct rdma_id_private *id_priv)
 875 {
 876         struct rdma_bind_list *bind_list = id_priv->bind_list;
 877
 878         if (!bind_list)
 879                 return;
 880
 881         mutex_lock(&lock);
 882         hlist_del(&id_priv->node);
 883         if (hlist_empty(&bind_list->owners)) {
 884                 idr_remove(bind_list->ps, bind_list->port);
 885                 kfree(bind_list);
 886         }
 887         mutex_unlock(&lock);
 888 }
 889
 890 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
 891 {
 892         struct cma_multicast *mc;
 893
 894         while (!list_empty(&id_priv->mc_list)) {
 895                 mc = container_of(id_priv->mc_list.next,
 896                                   struct cma_multicast, list);
 897                 list_del(&mc->list);
 898                 switch (rdma_port_get_link_layer(id_priv->cma_dev->device, id_priv->id.port_num)) {
 899                 case IB_LINK_LAYER_INFINIBAND:
 900                         ib_sa_free_multicast(mc->multicast.ib);
 901                         kfree(mc);
 902                         break;
 903                 case IB_LINK_LAYER_ETHERNET:
 904                         kref_put(&mc->mcref, release_mc);
 905                         break;
 906                 default:
 907                         break;
 908                 }
 909         }
 910 }
 911
 912 void rdma_destroy_id(struct rdma_cm_id *id)
 913 {
 914         struct rdma_id_private *id_priv;
 915         enum rdma_cm_state state;
 916
 917         id_priv = container_of(id, struct rdma_id_private, id);
 918         state = cma_exch(id_priv, RDMA_CM_DESTROYING);
 919         cma_cancel_operation(id_priv, state);
 920
 921         /*
 922          * Wait for any active callback to finish.  New callbacks will find
 923          * the id_priv state set to destroying and abort.
 924          */
 925         mutex_lock(&id_priv->handler_mutex);
 926         mutex_unlock(&id_priv->handler_mutex);
 927
 928         if (id_priv->cma_dev) {
 929                 switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
 930                 case RDMA_TRANSPORT_IB:
 931                         if (id_priv->cm_id.ib)
 932                                 ib_destroy_cm_id(id_priv->cm_id.ib);
 933                         break;
 934                 case RDMA_TRANSPORT_IWARP:
 935                         if (id_priv->cm_id.iw)
 936                                 iw_destroy_cm_id(id_priv->cm_id.iw);
 937                         break;
 938                 default:
 939                         break;
 940                 }
 941                 cma_leave_mc_groups(id_priv);
 942                 cma_release_dev(id_priv);
 943         }
 944
 945         cma_release_port(id_priv);
 946         cma_deref_id(id_priv);
 947         wait_for_completion(&id_priv->comp);
 948
 949         if (id_priv->internal_id)
 950                 cma_deref_id(id_priv->id.context);
 951
 952         kfree(id_priv->id.route.path_rec);
 953         kfree(id_priv);
 954 }
 955 EXPORT_SYMBOL(rdma_destroy_id);
 956
 957 static int cma_rep_recv(struct rdma_id_private *id_priv)
 958 {
 959         int ret;
 960
 961         ret = cma_modify_qp_rtr(id_priv, NULL);
 962         if (ret)
 963                 goto reject;
 964
 965         ret = cma_modify_qp_rts(id_priv, NULL);
 966         if (ret)
 967                 goto reject;
 968
 969         ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
 970         if (ret)
 971                 goto reject;
 972
 973         return 0;
 974 reject:
 975         cma_modify_qp_err(id_priv);
 976         ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
 977                        NULL, 0, NULL, 0);
 978         return ret;
 979 }
 980
 981 static int cma_verify_rep(struct rdma_id_private *id_priv, void *data)
 982 {
 983         if (id_priv->id.ps == RDMA_PS_SDP &&
 984             sdp_get_majv(((struct sdp_hah *) data)->sdp_version) !=
 985             SDP_MAJ_VERSION)
 986                 return -EINVAL;
 987
 988         return 0;
 989 }
 990
 991 static void cma_set_rep_event_data(struct rdma_cm_event *event,
 992                                    struct ib_cm_rep_event_param *rep_data,
 993                                    void *private_data)
 994 {
 995         event->param.conn.private_data = private_data;
 996         event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
 997         event->param.conn.responder_resources = rep_data->responder_resources;
 998         event->param.conn.initiator_depth = rep_data->initiator_depth;
 999         event->param.conn.flow_control = rep_data->flow_control;
1000         event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
1001         event->param.conn.srq = rep_data->srq;
1002         event->param.conn.qp_num = rep_data->remote_qpn;
1003 }
1004
1005 static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1006 {
1007         struct rdma_id_private *id_priv = cm_id->context;
1008         struct rdma_cm_event event;
1009         int ret = 0;
1010
1011         if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
1012                 cma_disable_callback(id_priv, RDMA_CM_CONNECT)) ||
1013             (ib_event->event == IB_CM_TIMEWAIT_EXIT &&
1014                 cma_disable_callback(id_priv, RDMA_CM_DISCONNECT)))
1015                 return 0;
1016
1017         memset(&event, 0, sizeof event);
1018         switch (ib_event->event) {
1019         case IB_CM_REQ_ERROR:
1020         case IB_CM_REP_ERROR:
1021                 event.event = RDMA_CM_EVENT_UNREACHABLE;
1022                 event.status = -ETIMEDOUT;
1023                 break;
1024         case IB_CM_REP_RECEIVED:
1025                 event.status = cma_verify_rep(id_priv, ib_event->private_data);
1026                 if (event.status)
1027                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1028                 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) {
1029                         event.status = cma_rep_recv(id_priv);
1030                         event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
1031                                                      RDMA_CM_EVENT_ESTABLISHED;
1032                 } else
1033                         event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
1034                 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
1035                                        ib_event->private_data);
1036                 break;
1037         case IB_CM_RTU_RECEIVED:
1038         case IB_CM_USER_ESTABLISHED:
1039                 event.event = RDMA_CM_EVENT_ESTABLISHED;
1040                 break;
1041         case IB_CM_DREQ_ERROR:
1042                 event.status = -ETIMEDOUT; /* fall through */
1043         case IB_CM_DREQ_RECEIVED:
1044         case IB_CM_DREP_RECEIVED:
1045                 if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT,
1046                                    RDMA_CM_DISCONNECT))
1047                         goto out;
1048                 event.event = RDMA_CM_EVENT_DISCONNECTED;
1049                 break;
1050         case IB_CM_TIMEWAIT_EXIT:
1051                 event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT;
1052                 break;
1053         case IB_CM_MRA_RECEIVED:
1054                 /* ignore event */
1055                 goto out;
1056         case IB_CM_REJ_RECEIVED:
1057                 cma_modify_qp_err(id_priv);
1058                 event.status = ib_event->param.rej_rcvd.reason;
1059                 event.event = RDMA_CM_EVENT_REJECTED;
1060                 event.param.conn.private_data = ib_event->private_data;
1061                 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
1062                 break;
1063         default:
1064                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
1065                        ib_event->event);
1066                 goto out;
1067         }
1068
1069         ret = id_priv->id.event_handler(&id_priv->id, &event);
1070         if (ret) {
1071                 /* Destroy the CM ID by returning a non-zero value. */
1072                 id_priv->cm_id.ib = NULL;
1073                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1074                 mutex_unlock(&id_priv->handler_mutex);
1075                 rdma_destroy_id(&id_priv->id);
1076                 return ret;
1077         }
1078 out:
1079         mutex_unlock(&id_priv->handler_mutex);
1080         return ret;
1081 }
1082
1083 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
1084                                                struct ib_cm_event *ib_event)
1085 {
1086         struct rdma_id_private *id_priv;
1087         struct rdma_cm_id *id;
1088         struct rdma_route *rt;
1089         union cma_ip_addr *src, *dst;
1090         __be16 port;
1091         u8 ip_ver;
1092         int ret;
1093
1094         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1095                              &ip_ver, &port, &src, &dst))
1096                 return NULL;
1097
1098         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1099                             listen_id->ps, ib_event->param.req_rcvd.qp_type);
1100         if (IS_ERR(id))
1101                 return NULL;
1102
1103         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1104                           ip_ver, port, src, dst);
1105
1106         rt = &id->route;
1107         rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
1108         rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths,
1109                                GFP_KERNEL);
1110         if (!rt->path_rec)
1111                 goto err;
1112
1113         rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path;
1114         if (rt->num_paths == 2)
1115                 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
1116
1117         if (cma_any_addr((struct sockaddr *) &rt->addr.src_addr)) {
1118                 rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND;
1119                 rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
1120                 ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
1121         } else {
1122                 ret = rdma_translate_ip((struct sockaddr *) &rt->addr.src_addr,
1123                                         &rt->addr.dev_addr);
1124                 if (ret)
1125                         goto err;
1126         }
1127         rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
1128
1129         id_priv = container_of(id, struct rdma_id_private, id);
1130         id_priv->state = RDMA_CM_CONNECT;
1131         return id_priv;
1132
1133 err:
1134         rdma_destroy_id(id);
1135         return NULL;
1136 }
1137
1138 static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
1139                                               struct ib_cm_event *ib_event)
1140 {
1141         struct rdma_id_private *id_priv;
1142         struct rdma_cm_id *id;
1143         union cma_ip_addr *src, *dst;
1144         __be16 port;
1145         u8 ip_ver;
1146         int ret;
1147
1148         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1149                             listen_id->ps, IB_QPT_UD);
1150         if (IS_ERR(id))
1151                 return NULL;
1152
1153
1154         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1155                              &ip_ver, &port, &src, &dst))
1156                 goto err;
1157
1158         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1159                           ip_ver, port, src, dst);
1160
1161         if (!cma_any_addr((struct sockaddr *) &id->route.addr.src_addr)) {
1162                 ret = rdma_translate_ip((struct sockaddr *) &id->route.addr.src_addr,
1163                                         &id->route.addr.dev_addr);
1164                 if (ret)
1165                         goto err;
1166         }
1167
1168         id_priv = container_of(id, struct rdma_id_private, id);
1169         id_priv->state = RDMA_CM_CONNECT;
1170         return id_priv;
1171 err:
1172         rdma_destroy_id(id);
1173         return NULL;
1174 }
1175
1176 static void cma_set_req_event_data(struct rdma_cm_event *event,
1177                                    struct ib_cm_req_event_param *req_data,
1178                                    void *private_data, int offset)
1179 {
1180         event->param.conn.private_data = private_data + offset;
1181         event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
1182         event->param.conn.responder_resources = req_data->responder_resources;
1183         event->param.conn.initiator_depth = req_data->initiator_depth;
1184         event->param.conn.flow_control = req_data->flow_control;
1185         event->param.conn.retry_count = req_data->retry_count;
1186         event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
1187         event->param.conn.srq = req_data->srq;
1188         event->param.conn.qp_num = req_data->remote_qpn;
1189 }
1190
1191 static int cma_check_req_qp_type(struct rdma_cm_id *id, struct ib_cm_event *ib_event)
1192 {
1193         return (((ib_event->event == IB_CM_REQ_RECEIVED) &&
1194                  (ib_event->param.req_rcvd.qp_type == id->qp_type)) ||
1195                 ((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) &&
1196                  (id->qp_type == IB_QPT_UD)) ||
1197                 (!id->qp_type));
1198 }
1199
1200 static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1201 {
1202         struct rdma_id_private *listen_id, *conn_id;
1203         struct rdma_cm_event event;
1204         int offset, ret;
1205
1206         listen_id = cm_id->context;
1207         if (!cma_check_req_qp_type(&listen_id->id, ib_event))
1208                 return -EINVAL;
1209
1210         if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
1211                 return -ECONNABORTED;
1212
1213         memset(&event, 0, sizeof event);
1214         offset = cma_user_data_offset(listen_id->id.ps);
1215         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1216         if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) {
1217                 conn_id = cma_new_udp_id(&listen_id->id, ib_event);
1218                 event.param.ud.private_data = ib_event->private_data + offset;
1219                 event.param.ud.private_data_len =
1220                                 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
1221         } else {
1222                 conn_id = cma_new_conn_id(&listen_id->id, ib_event);
1223                 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
1224                                        ib_event->private_data, offset);
1225         }
1226         if (!conn_id) {
1227                 ret = -ENOMEM;
1228                 goto err1;
1229         }
1230
1231         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1232         ret = cma_acquire_dev(conn_id);
1233         if (ret)
1234                 goto err2;
1235
1236         conn_id->cm_id.ib = cm_id;
1237         cm_id->context = conn_id;
1238         cm_id->cm_handler = cma_ib_handler;
1239
1240         /*
1241          * Protect against the user destroying conn_id from another thread
1242          * until we're done accessing it.
1243          */
1244         atomic_inc(&conn_id->refcount);
1245         ret = conn_id->id.event_handler(&conn_id->id, &event);
1246         if (ret)
1247                 goto err3;
1248
1249         /*
1250          * Acquire mutex to prevent user executing rdma_destroy_id()
1251          * while we're accessing the cm_id.
1252          */
1253         mutex_lock(&lock);
1254         if (cma_comp(conn_id, RDMA_CM_CONNECT) && (conn_id->id.qp_type != IB_QPT_UD))
1255                 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
1256         mutex_unlock(&lock);
1257         mutex_unlock(&conn_id->handler_mutex);
1258         mutex_unlock(&listen_id->handler_mutex);
1259         cma_deref_id(conn_id);
1260         return 0;
1261
1262 err3:
1263         cma_deref_id(conn_id);
1264         /* Destroy the CM ID by returning a non-zero value. */
1265         conn_id->cm_id.ib = NULL;
1266 err2:
1267         cma_exch(conn_id, RDMA_CM_DESTROYING);
1268         mutex_unlock(&conn_id->handler_mutex);
1269 err1:
1270         mutex_unlock(&listen_id->handler_mutex);
1271         if (conn_id)
1272                 rdma_destroy_id(&conn_id->id);
1273         return ret;
1274 }
1275
1276 static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr)
1277 {
1278         return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr)));
1279 }
1280
1281 static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr,
1282                                  struct ib_cm_compare_data *compare)
1283 {
1284         struct cma_hdr *cma_data, *cma_mask;
1285         struct sdp_hh *sdp_data, *sdp_mask;
1286         __be32 ip4_addr;
1287         struct in6_addr ip6_addr;
1288
1289         memset(compare, 0, sizeof *compare);
1290         cma_data = (void *) compare->data;
1291         cma_mask = (void *) compare->mask;
1292         sdp_data = (void *) compare->data;
1293         sdp_mask = (void *) compare->mask;
1294
1295         switch (addr->sa_family) {
1296         case AF_INET:
1297                 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
1298                 if (ps == RDMA_PS_SDP) {
1299                         sdp_set_ip_ver(sdp_data, 4);
1300                         sdp_set_ip_ver(sdp_mask, 0xF);
1301                         sdp_data->dst_addr.ip4.addr = ip4_addr;
1302                         sdp_mask->dst_addr.ip4.addr = htonl(~0);
1303                 } else {
1304                         cma_set_ip_ver(cma_data, 4);
1305                         cma_set_ip_ver(cma_mask, 0xF);
1306                         if (!cma_any_addr(addr)) {
1307                                 cma_data->dst_addr.ip4.addr = ip4_addr;
1308                                 cma_mask->dst_addr.ip4.addr = htonl(~0);
1309                         }
1310                 }
1311                 break;
1312         case AF_INET6:
1313                 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr;
1314                 if (ps == RDMA_PS_SDP) {
1315                         sdp_set_ip_ver(sdp_data, 6);
1316                         sdp_set_ip_ver(sdp_mask, 0xF);
1317                         sdp_data->dst_addr.ip6 = ip6_addr;
1318                         memset(&sdp_mask->dst_addr.ip6, 0xFF,
1319                                sizeof sdp_mask->dst_addr.ip6);
1320                 } else {
1321                         cma_set_ip_ver(cma_data, 6);
1322                         cma_set_ip_ver(cma_mask, 0xF);
1323                         if (!cma_any_addr(addr)) {
1324                                 cma_data->dst_addr.ip6 = ip6_addr;
1325                                 memset(&cma_mask->dst_addr.ip6, 0xFF,
1326                                        sizeof cma_mask->dst_addr.ip6);
1327                         }
1328                 }
1329                 break;
1330         default:
1331                 break;
1332         }
1333 }
1334
1335 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
1336 {
1337         struct rdma_id_private *id_priv = iw_id->context;
1338         struct rdma_cm_event event;
1339         struct sockaddr_in *sin;
1340         int ret = 0;
1341
1342         if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
1343                 return 0;
1344
1345         memset(&event, 0, sizeof event);
1346         switch (iw_event->event) {
1347         case IW_CM_EVENT_CLOSE:
1348                 event.event = RDMA_CM_EVENT_DISCONNECTED;
1349                 break;
1350         case IW_CM_EVENT_CONNECT_REPLY:
1351                 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1352                 *sin = iw_event->local_addr;
1353                 sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr;
1354                 *sin = iw_event->remote_addr;
1355                 switch (iw_event->status) {
1356                 case 0:
1357                         event.event = RDMA_CM_EVENT_ESTABLISHED;
1358                         event.param.conn.initiator_depth = iw_event->ird;
1359                         event.param.conn.responder_resources = iw_event->ord;
1360                         break;
1361                 case -ECONNRESET:
1362                 case -ECONNREFUSED:
1363                         event.event = RDMA_CM_EVENT_REJECTED;
1364                         break;
1365                 case -ETIMEDOUT:
1366                         event.event = RDMA_CM_EVENT_UNREACHABLE;
1367                         break;
1368                 default:
1369                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1370                         break;
1371                 }
1372                 break;
1373         case IW_CM_EVENT_ESTABLISHED:
1374                 event.event = RDMA_CM_EVENT_ESTABLISHED;
1375                 event.param.conn.initiator_depth = iw_event->ird;
1376                 event.param.conn.responder_resources = iw_event->ord;
1377                 break;
1378         default:
1379                 BUG_ON(1);
1380         }
1381
1382         event.status = iw_event->status;
1383         event.param.conn.private_data = iw_event->private_data;
1384         event.param.conn.private_data_len = iw_event->private_data_len;
1385         ret = id_priv->id.event_handler(&id_priv->id, &event);
1386         if (ret) {
1387                 /* Destroy the CM ID by returning a non-zero value. */
1388                 id_priv->cm_id.iw = NULL;
1389                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1390                 mutex_unlock(&id_priv->handler_mutex);
1391                 rdma_destroy_id(&id_priv->id);
1392                 return ret;
1393         }
1394
1395         mutex_unlock(&id_priv->handler_mutex);
1396         return ret;
1397 }
1398
1399 static int iw_conn_req_handler(struct iw_cm_id *cm_id,
1400                                struct iw_cm_event *iw_event)
1401 {
1402         struct rdma_cm_id *new_cm_id;
1403         struct rdma_id_private *listen_id, *conn_id;
1404         struct sockaddr_in *sin;
1405         struct net_device *dev = NULL;
1406         struct rdma_cm_event event;
1407         int ret;
1408         struct ib_device_attr attr;
1409
1410         listen_id = cm_id->context;
1411         if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
1412                 return -ECONNABORTED;
1413
1414         /* Create a new RDMA id for the new IW CM ID */
1415         new_cm_id = rdma_create_id(listen_id->id.event_handler,
1416                                    listen_id->id.context,
1417                                    RDMA_PS_TCP, IB_QPT_RC);
1418         if (IS_ERR(new_cm_id)) {
1419                 ret = -ENOMEM;
1420                 goto out;
1421         }
1422         conn_id = container_of(new_cm_id, struct rdma_id_private, id);
1423         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1424         conn_id->state = RDMA_CM_CONNECT;
1425
1426         dev = ip_dev_find(&init_net, iw_event->local_addr.sin_addr.s_addr);
1427         if (!dev) {
1428                 ret = -EADDRNOTAVAIL;
1429                 mutex_unlock(&conn_id->handler_mutex);
1430                 rdma_destroy_id(new_cm_id);
1431                 goto out;
1432         }
1433         ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
1434         if (ret) {
1435                 mutex_unlock(&conn_id->handler_mutex);
1436                 rdma_destroy_id(new_cm_id);
1437                 goto out;
1438         }
1439
1440         ret = cma_acquire_dev(conn_id);
1441         if (ret) {
1442                 mutex_unlock(&conn_id->handler_mutex);
1443                 rdma_destroy_id(new_cm_id);
1444                 goto out;
1445         }
1446
1447         conn_id->cm_id.iw = cm_id;
1448         cm_id->context = conn_id;
1449         cm_id->cm_handler = cma_iw_handler;
1450
1451         sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr;
1452         *sin = iw_event->local_addr;
1453         sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr;
1454         *sin = iw_event->remote_addr;
1455
1456         ret = ib_query_device(conn_id->id.device, &attr);
1457         if (ret) {
1458                 mutex_unlock(&conn_id->handler_mutex);
1459                 rdma_destroy_id(new_cm_id);
1460                 goto out;
1461         }
1462
1463         memset(&event, 0, sizeof event);
1464         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1465         event.param.conn.private_data = iw_event->private_data;
1466         event.param.conn.private_data_len = iw_event->private_data_len;
1467         event.param.conn.initiator_depth = iw_event->ird;
1468         event.param.conn.responder_resources = iw_event->ord;
1469
1470         /*
1471          * Protect against the user destroying conn_id from another thread
1472          * until we're done accessing it.
1473          */
1474         atomic_inc(&conn_id->refcount);
1475         ret = conn_id->id.event_handler(&conn_id->id, &event);
1476         if (ret) {
1477                 /* User wants to destroy the CM ID */
1478                 conn_id->cm_id.iw = NULL;
1479                 cma_exch(conn_id, RDMA_CM_DESTROYING);
1480                 mutex_unlock(&conn_id->handler_mutex);
1481                 cma_deref_id(conn_id);
1482                 rdma_destroy_id(&conn_id->id);
1483                 goto out;
1484         }
1485
1486         mutex_unlock(&conn_id->handler_mutex);
1487         cma_deref_id(conn_id);
1488
1489 out:
1490         if (dev)
1491                 dev_put(dev);
1492         mutex_unlock(&listen_id->handler_mutex);
1493         return ret;
1494 }
1495
1496 static int cma_ib_listen(struct rdma_id_private *id_priv)
1497 {
1498         struct ib_cm_compare_data compare_data;
1499         struct sockaddr *addr;
1500         struct ib_cm_id *id;
1501         __be64 svc_id;
1502         int ret;
1503
1504         id = ib_create_cm_id(id_priv->id.device, cma_req_handler, id_priv);
1505         if (IS_ERR(id))
1506                 return PTR_ERR(id);
1507
1508         id_priv->cm_id.ib = id;
1509
1510         addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
1511         svc_id = cma_get_service_id(id_priv->id.ps, addr);
1512         if (cma_any_addr(addr) && !id_priv->afonly)
1513                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL);
1514         else {
1515                 cma_set_compare_data(id_priv->id.ps, addr, &compare_data);
1516                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data);
1517         }
1518
1519         if (ret) {
1520                 ib_destroy_cm_id(id_priv->cm_id.ib);
1521                 id_priv->cm_id.ib = NULL;
1522         }
1523
1524         return ret;
1525 }
1526
1527 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
1528 {
1529         int ret;
1530         struct sockaddr_in *sin;
1531         struct iw_cm_id *id;
1532
1533         id = iw_create_cm_id(id_priv->id.device,
1534                              iw_conn_req_handler,
1535                              id_priv);
1536         if (IS_ERR(id))
1537                 return PTR_ERR(id);
1538
1539         id_priv->cm_id.iw = id;
1540
1541         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1542         id_priv->cm_id.iw->local_addr = *sin;
1543
1544         ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
1545
1546         if (ret) {
1547                 iw_destroy_cm_id(id_priv->cm_id.iw);
1548                 id_priv->cm_id.iw = NULL;
1549         }
1550
1551         return ret;
1552 }
1553
1554 static int cma_listen_handler(struct rdma_cm_id *id,
1555                               struct rdma_cm_event *event)
1556 {
1557         struct rdma_id_private *id_priv = id->context;
1558
1559         id->context = id_priv->id.context;
1560         id->event_handler = id_priv->id.event_handler;
1561         return id_priv->id.event_handler(id, event);
1562 }
1563
1564 static void cma_listen_on_dev(struct rdma_id_private *id_priv,
1565                               struct cma_device *cma_dev)
1566 {
1567         struct rdma_id_private *dev_id_priv;
1568         struct rdma_cm_id *id;
1569         int ret;
1570
1571         id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps,
1572                             id_priv->id.qp_type);
1573         if (IS_ERR(id))
1574                 return;
1575
1576         dev_id_priv = container_of(id, struct rdma_id_private, id);
1577
1578         dev_id_priv->state = RDMA_CM_ADDR_BOUND;
1579         memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr,
1580                ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr));
1581
1582         cma_attach_to_dev(dev_id_priv, cma_dev);
1583         list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
1584         atomic_inc(&id_priv->refcount);
1585         dev_id_priv->internal_id = 1;
1586         dev_id_priv->afonly = id_priv->afonly;
1587
1588         ret = rdma_listen(id, id_priv->backlog);
1589         if (ret)
1590                 printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, "
1591                        "listening on device %s\n", ret, cma_dev->device->name);
1592 }
1593
1594 static void cma_listen_on_all(struct rdma_id_private *id_priv)
1595 {
1596         struct cma_device *cma_dev;
1597
1598         mutex_lock(&lock);
1599         list_add_tail(&id_priv->list, &listen_any_list);
1600         list_for_each_entry(cma_dev, &dev_list, list)
1601                 cma_listen_on_dev(id_priv, cma_dev);
1602         mutex_unlock(&lock);
1603 }
1604
1605 void rdma_set_service_type(struct rdma_cm_id *id, int tos)
1606 {
1607         struct rdma_id_private *id_priv;
1608
1609         id_priv = container_of(id, struct rdma_id_private, id);
1610         id_priv->tos = (u8) tos;
1611 }
1612 EXPORT_SYMBOL(rdma_set_service_type);
1613
1614 static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec,
1615                               void *context)
1616 {
1617         struct cma_work *work = context;
1618         struct rdma_route *route;
1619
1620         route = &work->id->id.route;
1621
1622         if (!status) {
1623                 route->num_paths = 1;
1624                 *route->path_rec = *path_rec;
1625         } else {
1626                 work->old_state = RDMA_CM_ROUTE_QUERY;
1627                 work->new_state = RDMA_CM_ADDR_RESOLVED;
1628                 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
1629                 work->event.status = status;
1630         }
1631
1632         queue_work(cma_wq, &work->work);
1633 }
1634
1635 static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
1636                               struct cma_work *work)
1637 {
1638         struct rdma_addr *addr = &id_priv->id.route.addr;
1639         struct ib_sa_path_rec path_rec;
1640         ib_sa_comp_mask comp_mask;
1641         struct sockaddr_in6 *sin6;
1642
1643         memset(&path_rec, 0, sizeof path_rec);
1644         rdma_addr_get_sgid(&addr->dev_addr, &path_rec.sgid);
1645         rdma_addr_get_dgid(&addr->dev_addr, &path_rec.dgid);
1646         path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr));
1647         path_rec.numb_path = 1;
1648         path_rec.reversible = 1;
1649         path_rec.service_id = cma_get_service_id(id_priv->id.ps,
1650                                                         (struct sockaddr *) &addr->dst_addr);
1651
1652         comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
1653                     IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
1654                     IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
1655
1656         if (addr->src_addr.ss_family == AF_INET) {
1657                 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
1658                 comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
1659         } else {
1660                 sin6 = (struct sockaddr_in6 *) &addr->src_addr;
1661                 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
1662                 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
1663         }
1664
1665         id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
1666                                                id_priv->id.port_num, &path_rec,
1667                                                comp_mask, timeout_ms,
1668                                                GFP_KERNEL, cma_query_handler,
1669                                                work, &id_priv->query);
1670
1671         return (id_priv->query_id < 0) ? id_priv->query_id : 0;
1672 }
1673
1674 static void cma_work_handler(struct work_struct *_work)
1675 {
1676         struct cma_work *work = container_of(_work, struct cma_work, work);
1677         struct rdma_id_private *id_priv = work->id;
1678         int destroy = 0;
1679
1680         mutex_lock(&id_priv->handler_mutex);
1681         if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
1682                 goto out;
1683
1684         if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1685                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1686                 destroy = 1;
1687         }
1688 out:
1689         mutex_unlock(&id_priv->handler_mutex);
1690         cma_deref_id(id_priv);
1691         if (destroy)
1692                 rdma_destroy_id(&id_priv->id);
1693         kfree(work);
1694 }
1695
1696 static void cma_ndev_work_handler(struct work_struct *_work)
1697 {
1698         struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work);
1699         struct rdma_id_private *id_priv = work->id;
1700         int destroy = 0;
1701
1702         mutex_lock(&id_priv->handler_mutex);
1703         if (id_priv->state == RDMA_CM_DESTROYING ||
1704             id_priv->state == RDMA_CM_DEVICE_REMOVAL)
1705                 goto out;
1706
1707         if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1708                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1709                 destroy = 1;
1710         }
1711
1712 out:
1713         mutex_unlock(&id_priv->handler_mutex);
1714         cma_deref_id(id_priv);
1715         if (destroy)
1716                 rdma_destroy_id(&id_priv->id);
1717         kfree(work);
1718 }
1719
1720 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
1721 {
1722         struct rdma_route *route = &id_priv->id.route;
1723         struct cma_work *work;
1724         int ret;
1725
1726         work = kzalloc(sizeof *work, GFP_KERNEL);
1727         if (!work)
1728                 return -ENOMEM;
1729
1730         work->id = id_priv;
1731         INIT_WORK(&work->work, cma_work_handler);
1732         work->old_state = RDMA_CM_ROUTE_QUERY;
1733         work->new_state = RDMA_CM_ROUTE_RESOLVED;
1734         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1735
1736         route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
1737         if (!route->path_rec) {
1738                 ret = -ENOMEM;
1739                 goto err1;
1740         }
1741
1742         ret = cma_query_ib_route(id_priv, timeout_ms, work);
1743         if (ret)
1744                 goto err2;
1745
1746         return 0;
1747 err2:
1748         kfree(route->path_rec);
1749         route->path_rec = NULL;
1750 err1:
1751         kfree(work);
1752         return ret;
1753 }
1754
1755 int rdma_set_ib_paths(struct rdma_cm_id *id,
1756                       struct ib_sa_path_rec *path_rec, int num_paths)
1757 {
1758         struct rdma_id_private *id_priv;
1759         int ret;
1760
1761         id_priv = container_of(id, struct rdma_id_private, id);
1762         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
1763                            RDMA_CM_ROUTE_RESOLVED))
1764                 return -EINVAL;
1765
1766         id->route.path_rec = kmemdup(path_rec, sizeof *path_rec * num_paths,
1767                                      GFP_KERNEL);
1768         if (!id->route.path_rec) {
1769                 ret = -ENOMEM;
1770                 goto err;
1771         }
1772
1773         id->route.num_paths = num_paths;
1774         return 0;
1775 err:
1776         cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED);
1777         return ret;
1778 }
1779 EXPORT_SYMBOL(rdma_set_ib_paths);
1780
1781 static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
1782 {
1783         struct cma_work *work;
1784
1785         work = kzalloc(sizeof *work, GFP_KERNEL);
1786         if (!work)
1787                 return -ENOMEM;
1788
1789         work->id = id_priv;
1790         INIT_WORK(&work->work, cma_work_handler);
1791         work->old_state = RDMA_CM_ROUTE_QUERY;
1792         work->new_state = RDMA_CM_ROUTE_RESOLVED;
1793         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1794         queue_work(cma_wq, &work->work);
1795         return 0;
1796 }
1797
1798 static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
1799 {
1800         struct rdma_route *route = &id_priv->id.route;
1801         struct rdma_addr *addr = &route->addr;
1802         struct cma_work *work;
1803         int ret;
1804         struct sockaddr_in *src_addr = (struct sockaddr_in *)&route->addr.src_addr;
1805         struct sockaddr_in *dst_addr = (struct sockaddr_in *)&route->addr.dst_addr;
1806         struct net_device *ndev = NULL;
1807         u16 vid;
1808
1809         if (src_addr->sin_family != dst_addr->sin_family)
1810                 return -EINVAL;
1811
1812         work = kzalloc(sizeof *work, GFP_KERNEL);
1813         if (!work)
1814                 return -ENOMEM;
1815
1816         work->id = id_priv;
1817         INIT_WORK(&work->work, cma_work_handler);
1818
1819         route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL);
1820         if (!route->path_rec) {
1821                 ret = -ENOMEM;
1822                 goto err1;
1823         }
1824
1825         route->num_paths = 1;
1826
1827         if (addr->dev_addr.bound_dev_if)
1828                 ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if);
1829         if (!ndev) {
1830                 ret = -ENODEV;
1831                 goto err2;
1832         }
1833
1834         vid = rdma_vlan_dev_vlan_id(ndev);
1835
1836         iboe_mac_vlan_to_ll(&route->path_rec->sgid, addr->dev_addr.src_dev_addr, vid);
1837         iboe_mac_vlan_to_ll(&route->path_rec->dgid, addr->dev_addr.dst_dev_addr, vid);
1838
1839         route->path_rec->hop_limit = 1;
1840         route->path_rec->reversible = 1;
1841         route->path_rec->pkey = cpu_to_be16(0xffff);
1842         route->path_rec->mtu_selector = IB_SA_EQ;
1843         route->path_rec->sl = netdev_get_prio_tc_map(
1844                         ndev->priv_flags & IFF_802_1Q_VLAN ?
1845                                 vlan_dev_real_dev(ndev) : ndev,
1846                         rt_tos2priority(id_priv->tos));
1847
1848         route->path_rec->mtu = iboe_get_mtu(ndev->mtu);
1849         route->path_rec->rate_selector = IB_SA_EQ;
1850         route->path_rec->rate = iboe_get_rate(ndev);
1851         dev_put(ndev);
1852         route->path_rec->packet_life_time_selector = IB_SA_EQ;
1853         route->path_rec->packet_life_time = CMA_IBOE_PACKET_LIFETIME;
1854         if (!route->path_rec->mtu) {
1855                 ret = -EINVAL;
1856                 goto err2;
1857         }
1858
1859         work->old_state = RDMA_CM_ROUTE_QUERY;
1860         work->new_state = RDMA_CM_ROUTE_RESOLVED;
1861         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1862         work->event.status = 0;
1863
1864         queue_work(cma_wq, &work->work);
1865
1866         return 0;
1867
1868 err2:
1869         kfree(route->path_rec);
1870         route->path_rec = NULL;
1871 err1:
1872         kfree(work);
1873         return ret;
1874 }
1875
1876 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
1877 {
1878         struct rdma_id_private *id_priv;
1879         int ret;
1880
1881         id_priv = container_of(id, struct rdma_id_private, id);
1882         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY))
1883                 return -EINVAL;
1884
1885         atomic_inc(&id_priv->refcount);
1886         switch (rdma_node_get_transport(id->device->node_type)) {
1887         case RDMA_TRANSPORT_IB:
1888                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
1889                 case IB_LINK_LAYER_INFINIBAND:
1890                         ret = cma_resolve_ib_route(id_priv, timeout_ms);
1891                         break;
1892                 case IB_LINK_LAYER_ETHERNET:
1893                         ret = cma_resolve_iboe_route(id_priv);
1894                         break;
1895                 default:
1896                         ret = -ENOSYS;
1897                 }
1898                 break;
1899         case RDMA_TRANSPORT_IWARP:
1900                 ret = cma_resolve_iw_route(id_priv, timeout_ms);
1901                 break;
1902         default:
1903                 ret = -ENOSYS;
1904                 break;
1905         }
1906         if (ret)
1907                 goto err;
1908
1909         return 0;
1910 err:
1911         cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED);
1912         cma_deref_id(id_priv);
1913         return ret;
1914 }
1915 EXPORT_SYMBOL(rdma_resolve_route);
1916
1917 static int cma_bind_loopback(struct rdma_id_private *id_priv)
1918 {
1919         struct cma_device *cma_dev;
1920         struct ib_port_attr port_attr;
1921         union ib_gid gid;
1922         u16 pkey;
1923         int ret;
1924         u8 p;
1925
1926         mutex_lock(&lock);
1927         if (list_empty(&dev_list)) {
1928                 ret = -ENODEV;
1929                 goto out;
1930         }
1931         list_for_each_entry(cma_dev, &dev_list, list)
1932                 for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p)
1933                         if (!ib_query_port(cma_dev->device, p, &port_attr) &&
1934                             port_attr.state == IB_PORT_ACTIVE)
1935                                 goto port_found;
1936
1937         p = 1;
1938         cma_dev = list_entry(dev_list.next, struct cma_device, list);
1939
1940 port_found:
1941         ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid);
1942         if (ret)
1943                 goto out;
1944
1945         ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
1946         if (ret)
1947                 goto out;
1948
1949         id_priv->id.route.addr.dev_addr.dev_type =
1950                 (rdma_port_get_link_layer(cma_dev->device, p) == IB_LINK_LAYER_INFINIBAND) ?
1951                 ARPHRD_INFINIBAND : ARPHRD_ETHER;
1952
1953         rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
1954         ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
1955         id_priv->id.port_num = p;
1956         cma_attach_to_dev(id_priv, cma_dev);
1957 out:
1958         mutex_unlock(&lock);
1959         return ret;
1960 }
1961
1962 static void addr_handler(int status, struct sockaddr *src_addr,
1963                          struct rdma_dev_addr *dev_addr, void *context)
1964 {
1965         struct rdma_id_private *id_priv = context;
1966         struct rdma_cm_event event;
1967
1968         memset(&event, 0, sizeof event);
1969         mutex_lock(&id_priv->handler_mutex);
1970         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY,
1971                            RDMA_CM_ADDR_RESOLVED))
1972                 goto out;
1973
1974         if (!status && !id_priv->cma_dev)
1975                 status = cma_acquire_dev(id_priv);
1976
1977         if (status) {
1978                 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
1979                                    RDMA_CM_ADDR_BOUND))
1980                         goto out;
1981                 event.event = RDMA_CM_EVENT_ADDR_ERROR;
1982                 event.status = status;
1983         } else {
1984                 memcpy(&id_priv->id.route.addr.src_addr, src_addr,
1985                        ip_addr_size(src_addr));
1986                 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
1987         }
1988
1989         if (id_priv->id.event_handler(&id_priv->id, &event)) {
1990                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1991                 mutex_unlock(&id_priv->handler_mutex);
1992                 cma_deref_id(id_priv);
1993                 rdma_destroy_id(&id_priv->id);
1994                 return;
1995         }
1996 out:
1997         mutex_unlock(&id_priv->handler_mutex);
1998         cma_deref_id(id_priv);
1999 }
2000
2001 static int cma_resolve_loopback(struct rdma_id_private *id_priv)
2002 {
2003         struct cma_work *work;
2004         struct sockaddr *src, *dst;
2005         union ib_gid gid;
2006         int ret;
2007
2008         work = kzalloc(sizeof *work, GFP_KERNEL);
2009         if (!work)
2010                 return -ENOMEM;
2011
2012         if (!id_priv->cma_dev) {
2013                 ret = cma_bind_loopback(id_priv);
2014                 if (ret)
2015                         goto err;
2016         }
2017
2018         rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
2019         rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
2020
2021         src = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
2022         if (cma_zero_addr(src)) {
2023                 dst = (struct sockaddr *) &id_priv->id.route.addr.dst_addr;
2024                 if ((src->sa_family = dst->sa_family) == AF_INET) {
2025                         ((struct sockaddr_in *)src)->sin_addr =
2026                                 ((struct sockaddr_in *)dst)->sin_addr;
2027                 } else {
2028                         ((struct sockaddr_in6 *)src)->sin6_addr =
2029                                 ((struct sockaddr_in6 *)dst)->sin6_addr;
2030                 }
2031         }
2032
2033         work->id = id_priv;
2034         INIT_WORK(&work->work, cma_work_handler);
2035         work->old_state = RDMA_CM_ADDR_QUERY;
2036         work->new_state = RDMA_CM_ADDR_RESOLVED;
2037         work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2038         queue_work(cma_wq, &work->work);
2039         return 0;
2040 err:
2041         kfree(work);
2042         return ret;
2043 }
2044
2045 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
2046                          struct sockaddr *dst_addr)
2047 {
2048         if (!src_addr || !src_addr->sa_family) {
2049                 src_addr = (struct sockaddr *) &id->route.addr.src_addr;
2050                 if ((src_addr->sa_family = dst_addr->sa_family) == AF_INET6) {
2051                         ((struct sockaddr_in6 *) src_addr)->sin6_scope_id =
2052                                 ((struct sockaddr_in6 *) dst_addr)->sin6_scope_id;
2053                 }
2054         }
2055         return rdma_bind_addr(id, src_addr);
2056 }
2057
2058 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
2059                       struct sockaddr *dst_addr, int timeout_ms)
2060 {
2061         struct rdma_id_private *id_priv;
2062         int ret;
2063
2064         id_priv = container_of(id, struct rdma_id_private, id);
2065         if (id_priv->state == RDMA_CM_IDLE) {
2066                 ret = cma_bind_addr(id, src_addr, dst_addr);
2067                 if (ret)
2068                         return ret;
2069         }
2070
2071         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY))
2072                 return -EINVAL;
2073
2074         atomic_inc(&id_priv->refcount);
2075         memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr));
2076         if (cma_any_addr(dst_addr))
2077                 ret = cma_resolve_loopback(id_priv);
2078         else
2079                 ret = rdma_resolve_ip(&addr_client, (struct sockaddr *) &id->route.addr.src_addr,
2080                                       dst_addr, &id->route.addr.dev_addr,
2081                                       timeout_ms, addr_handler, id_priv);
2082         if (ret)
2083                 goto err;
2084
2085         return 0;
2086 err:
2087         cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
2088         cma_deref_id(id_priv);
2089         return ret;
2090 }
2091 EXPORT_SYMBOL(rdma_resolve_addr);
2092
2093 int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)
2094 {
2095         struct rdma_id_private *id_priv;
2096         unsigned long flags;
2097         int ret;
2098
2099         id_priv = container_of(id, struct rdma_id_private, id);
2100         spin_lock_irqsave(&id_priv->lock, flags);
2101         if (id_priv->state == RDMA_CM_IDLE) {
2102                 id_priv->reuseaddr = reuse;
2103                 ret = 0;
2104         } else {
2105                 ret = -EINVAL;
2106         }
2107         spin_unlock_irqrestore(&id_priv->lock, flags);
2108         return ret;
2109 }
2110 EXPORT_SYMBOL(rdma_set_reuseaddr);
2111
2112 int rdma_set_afonly(struct rdma_cm_id *id, int afonly)
2113 {
2114         struct rdma_id_private *id_priv;
2115         unsigned long flags;
2116         int ret;
2117
2118         id_priv = container_of(id, struct rdma_id_private, id);
2119         spin_lock_irqsave(&id_priv->lock, flags);
2120         if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) {
2121                 id_priv->options |= (1 << CMA_OPTION_AFONLY);
2122                 id_priv->afonly = afonly;
2123                 ret = 0;
2124         } else {
2125                 ret = -EINVAL;
2126         }
2127         spin_unlock_irqrestore(&id_priv->lock, flags);
2128         return ret;
2129 }
2130 EXPORT_SYMBOL(rdma_set_afonly);
2131
2132 static void cma_bind_port(struct rdma_bind_list *bind_list,
2133                           struct rdma_id_private *id_priv)
2134 {
2135         struct sockaddr_in *sin;
2136
2137         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
2138         sin->sin_port = htons(bind_list->port);
2139         id_priv->bind_list = bind_list;
2140         hlist_add_head(&id_priv->node, &bind_list->owners);
2141 }
2142
2143 static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv,
2144                           unsigned short snum)
2145 {
2146         struct rdma_bind_list *bind_list;
2147         int port, ret;
2148
2149         bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
2150         if (!bind_list)
2151                 return -ENOMEM;
2152
2153         do {
2154                 ret = idr_get_new_above(ps, bind_list, snum, &port);
2155         } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
2156
2157         if (ret)
2158                 goto err1;
2159
2160         if (port != snum) {
2161                 ret = -EADDRNOTAVAIL;
2162                 goto err2;
2163         }
2164
2165         bind_list->ps = ps;
2166         bind_list->port = (unsigned short) port;
2167         cma_bind_port(bind_list, id_priv);
2168         return 0;
2169 err2:
2170         idr_remove(ps, port);
2171 err1:
2172         kfree(bind_list);
2173         return ret;
2174 }
2175
2176 static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv)
2177 {
2178         static unsigned int last_used_port;
2179         int low, high, remaining;
2180         unsigned int rover;
2181
2182         inet_get_local_port_range(&low, &high);
2183         remaining = (high - low) + 1;
2184         rover = net_random() % remaining + low;
2185 retry:
2186         if (last_used_port != rover &&
2187             !idr_find(ps, (unsigned short) rover)) {
2188                 int ret = cma_alloc_port(ps, id_priv, rover);
2189                 /*
2190                  * Remember previously used port number in order to avoid
2191                  * re-using same port immediately after it is closed.
2192                  */
2193                 if (!ret)
2194                         last_used_port = rover;
2195                 if (ret != -EADDRNOTAVAIL)
2196                         return ret;
2197         }
2198         if (--remaining) {
2199                 rover++;
2200                 if ((rover < low) || (rover > high))
2201                         rover = low;
2202                 goto retry;
2203         }
2204         return -EADDRNOTAVAIL;
2205 }
2206
2207 /*
2208  * Check that the requested port is available.  This is called when trying to
2209  * bind to a specific port, or when trying to listen on a bound port.  In
2210  * the latter case, the provided id_priv may already be on the bind_list, but
2211  * we still need to check that it's okay to start listening.
2212  */
2213 static int cma_check_port(struct rdma_bind_list *bind_list,
2214                           struct rdma_id_private *id_priv, uint8_t reuseaddr)
2215 {
2216         struct rdma_id_private *cur_id;
2217         struct sockaddr *addr, *cur_addr;
2218         struct hlist_node *node;
2219
2220         addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
2221         hlist_for_each_entry(cur_id, node, &bind_list->owners, node) {
2222                 if (id_priv == cur_id)
2223                         continue;
2224
2225                 if ((cur_id->state != RDMA_CM_LISTEN) && reuseaddr &&
2226                     cur_id->reuseaddr)
2227                         continue;
2228
2229                 cur_addr = (struct sockaddr *) &cur_id->id.route.addr.src_addr;
2230                 if (id_priv->afonly && cur_id->afonly &&
2231                     (addr->sa_family != cur_addr->sa_family))
2232                         continue;
2233
2234                 if (cma_any_addr(addr) || cma_any_addr(cur_addr))
2235                         return -EADDRNOTAVAIL;
2236
2237                 if (!cma_addr_cmp(addr, cur_addr))
2238                         return -EADDRINUSE;
2239         }
2240         return 0;
2241 }
2242
2243 static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv)
2244 {
2245         struct rdma_bind_list *bind_list;
2246         unsigned short snum;
2247         int ret;
2248
2249         snum = ntohs(cma_port((struct sockaddr *) &id_priv->id.route.addr.src_addr));
2250         if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
2251                 return -EACCES;
2252
2253         bind_list = idr_find(ps, snum);
2254         if (!bind_list) {
2255                 ret = cma_alloc_port(ps, id_priv, snum);
2256         } else {
2257                 ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr);
2258                 if (!ret)
2259                         cma_bind_port(bind_list, id_priv);
2260         }
2261         return ret;
2262 }
2263
2264 static int cma_bind_listen(struct rdma_id_private *id_priv)
2265 {
2266         struct rdma_bind_list *bind_list = id_priv->bind_list;
2267         int ret = 0;
2268
2269         mutex_lock(&lock);
2270         if (bind_list->owners.first->next)
2271                 ret = cma_check_port(bind_list, id_priv, 0);
2272         mutex_unlock(&lock);
2273         return ret;
2274 }
2275
2276 static int cma_get_port(struct rdma_id_private *id_priv)
2277 {
2278         struct idr *ps;
2279         int ret;
2280
2281         switch (id_priv->id.ps) {
2282         case RDMA_PS_SDP:
2283                 ps = &sdp_ps;
2284                 break;
2285         case RDMA_PS_TCP:
2286                 ps = &tcp_ps;
2287                 break;
2288         case RDMA_PS_UDP:
2289                 ps = &udp_ps;
2290                 break;
2291         case RDMA_PS_IPOIB:
2292                 ps = &ipoib_ps;
2293                 break;
2294         case RDMA_PS_IB:
2295                 ps = &ib_ps;
2296                 break;
2297         default:
2298                 return -EPROTONOSUPPORT;
2299         }
2300
2301         mutex_lock(&lock);
2302         if (cma_any_port((struct sockaddr *) &id_priv->id.route.addr.src_addr))
2303                 ret = cma_alloc_any_port(ps, id_priv);
2304         else
2305                 ret = cma_use_port(ps, id_priv);
2306         mutex_unlock(&lock);
2307
2308         return ret;
2309 }
2310
2311 static int cma_check_linklocal(struct rdma_dev_addr *dev_addr,
2312                                struct sockaddr *addr)
2313 {
2314 #if IS_ENABLED(CONFIG_IPV6)
2315         struct sockaddr_in6 *sin6;
2316
2317         if (addr->sa_family != AF_INET6)
2318                 return 0;
2319
2320         sin6 = (struct sockaddr_in6 *) addr;
2321         if ((ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL) &&
2322             !sin6->sin6_scope_id)
2323                         return -EINVAL;
2324
2325         dev_addr->bound_dev_if = sin6->sin6_scope_id;
2326 #endif
2327         return 0;
2328 }
2329
2330 int rdma_listen(struct rdma_cm_id *id, int backlog)
2331 {
2332         struct rdma_id_private *id_priv;
2333         int ret;
2334
2335         id_priv = container_of(id, struct rdma_id_private, id);
2336         if (id_priv->state == RDMA_CM_IDLE) {
2337                 ((struct sockaddr *) &id->route.addr.src_addr)->sa_family = AF_INET;
2338                 ret = rdma_bind_addr(id, (struct sockaddr *) &id->route.addr.src_addr);
2339                 if (ret)
2340                         return ret;
2341         }
2342
2343         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN))
2344                 return -EINVAL;
2345
2346         if (id_priv->reuseaddr) {
2347                 ret = cma_bind_listen(id_priv);
2348                 if (ret)
2349                         goto err;
2350         }
2351
2352         id_priv->backlog = backlog;
2353         if (id->device) {
2354                 switch (rdma_node_get_transport(id->device->node_type)) {
2355                 case RDMA_TRANSPORT_IB:
2356                         ret = cma_ib_listen(id_priv);
2357                         if (ret)
2358                                 goto err;
2359                         break;
2360                 case RDMA_TRANSPORT_IWARP:
2361                         ret = cma_iw_listen(id_priv, backlog);
2362                         if (ret)
2363                                 goto err;
2364                         break;
2365                 default:
2366                         ret = -ENOSYS;
2367                         goto err;
2368                 }
2369         } else
2370                 cma_listen_on_all(id_priv);
2371
2372         return 0;
2373 err:
2374         id_priv->backlog = 0;
2375         cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND);
2376         return ret;
2377 }
2378 EXPORT_SYMBOL(rdma_listen);
2379
2380 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
2381 {
2382         struct rdma_id_private *id_priv;
2383         int ret;
2384
2385         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
2386                 return -EAFNOSUPPORT;
2387
2388         id_priv = container_of(id, struct rdma_id_private, id);
2389         if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))
2390                 return -EINVAL;
2391
2392         ret = cma_check_linklocal(&id->route.addr.dev_addr, addr);
2393         if (ret)
2394                 goto err1;
2395
2396         if (!cma_any_addr(addr)) {
2397                 ret = rdma_translate_ip(addr, &id->route.addr.dev_addr);
2398                 if (ret)
2399                         goto err1;
2400
2401                 ret = cma_acquire_dev(id_priv);
2402                 if (ret)
2403                         goto err1;
2404         }
2405
2406         memcpy(&id->route.addr.src_addr, addr, ip_addr_size(addr));
2407         if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) {
2408                 if (addr->sa_family == AF_INET)
2409                         id_priv->afonly = 1;
2410 #if IS_ENABLED(CONFIG_IPV6)
2411                 else if (addr->sa_family == AF_INET6)
2412                         id_priv->afonly = init_net.ipv6.sysctl.bindv6only;
2413 #endif
2414         }
2415         ret = cma_get_port(id_priv);
2416         if (ret)
2417                 goto err2;
2418
2419         return 0;
2420 err2:
2421         if (id_priv->cma_dev)
2422                 cma_release_dev(id_priv);
2423 err1:
2424         cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
2425         return ret;
2426 }
2427 EXPORT_SYMBOL(rdma_bind_addr);
2428
2429 static int cma_format_hdr(void *hdr, enum rdma_port_space ps,
2430                           struct rdma_route *route)
2431 {
2432         struct cma_hdr *cma_hdr;
2433         struct sdp_hh *sdp_hdr;
2434
2435         if (route->addr.src_addr.ss_family == AF_INET) {
2436                 struct sockaddr_in *src4, *dst4;
2437
2438                 src4 = (struct sockaddr_in *) &route->addr.src_addr;
2439                 dst4 = (struct sockaddr_in *) &route->addr.dst_addr;
2440
2441                 switch (ps) {
2442                 case RDMA_PS_SDP:
2443                         sdp_hdr = hdr;
2444                         if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2445                                 return -EINVAL;
2446                         sdp_set_ip_ver(sdp_hdr, 4);
2447                         sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2448                         sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2449                         sdp_hdr->port = src4->sin_port;
2450                         break;
2451                 default:
2452                         cma_hdr = hdr;
2453                         cma_hdr->cma_version = CMA_VERSION;
2454                         cma_set_ip_ver(cma_hdr, 4);
2455                         cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2456                         cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2457                         cma_hdr->port = src4->sin_port;
2458                         break;
2459                 }
2460         } else {
2461                 struct sockaddr_in6 *src6, *dst6;
2462
2463                 src6 = (struct sockaddr_in6 *) &route->addr.src_addr;
2464                 dst6 = (struct sockaddr_in6 *) &route->addr.dst_addr;
2465
2466                 switch (ps) {
2467                 case RDMA_PS_SDP:
2468                         sdp_hdr = hdr;
2469                         if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2470                                 return -EINVAL;
2471                         sdp_set_ip_ver(sdp_hdr, 6);
2472                         sdp_hdr->src_addr.ip6 = src6->sin6_addr;
2473                         sdp_hdr->dst_addr.ip6 = dst6->sin6_addr;
2474                         sdp_hdr->port = src6->sin6_port;
2475                         break;
2476                 default:
2477                         cma_hdr = hdr;
2478                         cma_hdr->cma_version = CMA_VERSION;
2479                         cma_set_ip_ver(cma_hdr, 6);
2480                         cma_hdr->src_addr.ip6 = src6->sin6_addr;
2481                         cma_hdr->dst_addr.ip6 = dst6->sin6_addr;
2482                         cma_hdr->port = src6->sin6_port;
2483                         break;
2484                 }
2485         }
2486         return 0;
2487 }
2488
2489 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
2490                                 struct ib_cm_event *ib_event)
2491 {
2492         struct rdma_id_private *id_priv = cm_id->context;
2493         struct rdma_cm_event event;
2494         struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd;
2495         int ret = 0;
2496
2497         if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
2498                 return 0;
2499
2500         memset(&event, 0, sizeof event);
2501         switch (ib_event->event) {
2502         case IB_CM_SIDR_REQ_ERROR:
2503                 event.event = RDMA_CM_EVENT_UNREACHABLE;
2504                 event.status = -ETIMEDOUT;
2505                 break;
2506         case IB_CM_SIDR_REP_RECEIVED:
2507                 event.param.ud.private_data = ib_event->private_data;
2508                 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
2509                 if (rep->status != IB_SIDR_SUCCESS) {
2510                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2511                         event.status = ib_event->param.sidr_rep_rcvd.status;
2512                         break;
2513                 }
2514                 ret = cma_set_qkey(id_priv);
2515                 if (ret) {
2516                         event.event = RDMA_CM_EVENT_ADDR_ERROR;
2517                         event.status = -EINVAL;
2518                         break;
2519                 }
2520                 if (id_priv->qkey != rep->qkey) {
2521                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2522                         event.status = -EINVAL;
2523                         break;
2524                 }
2525                 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
2526                                      id_priv->id.route.path_rec,
2527                                      &event.param.ud.ah_attr);
2528                 event.param.ud.qp_num = rep->qpn;
2529                 event.param.ud.qkey = rep->qkey;
2530                 event.event = RDMA_CM_EVENT_ESTABLISHED;
2531                 event.status = 0;
2532                 break;
2533         default:
2534                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
2535                        ib_event->event);
2536                 goto out;
2537         }
2538
2539         ret = id_priv->id.event_handler(&id_priv->id, &event);
2540         if (ret) {
2541                 /* Destroy the CM ID by returning a non-zero value. */
2542                 id_priv->cm_id.ib = NULL;
2543                 cma_exch(id_priv, RDMA_CM_DESTROYING);
2544                 mutex_unlock(&id_priv->handler_mutex);
2545                 rdma_destroy_id(&id_priv->id);
2546                 return ret;
2547         }
2548 out:
2549         mutex_unlock(&id_priv->handler_mutex);
2550         return ret;
2551 }
2552
2553 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
2554                               struct rdma_conn_param *conn_param)
2555 {
2556         struct ib_cm_sidr_req_param req;
2557         struct rdma_route *route;
2558         struct ib_cm_id *id;
2559         int ret;
2560
2561         req.private_data_len = sizeof(struct cma_hdr) +
2562                                conn_param->private_data_len;
2563         if (req.private_data_len < conn_param->private_data_len)
2564                 return -EINVAL;
2565
2566         req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2567         if (!req.private_data)
2568                 return -ENOMEM;
2569
2570         if (conn_param->private_data && conn_param->private_data_len)
2571                 memcpy((void *) req.private_data + sizeof(struct cma_hdr),
2572                        conn_param->private_data, conn_param->private_data_len);
2573
2574         route = &id_priv->id.route;
2575         ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route);
2576         if (ret)
2577                 goto out;
2578
2579         id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler,
2580                              id_priv);
2581         if (IS_ERR(id)) {
2582                 ret = PTR_ERR(id);
2583                 goto out;
2584         }
2585         id_priv->cm_id.ib = id;
2586
2587         req.path = route->path_rec;
2588         req.service_id = cma_get_service_id(id_priv->id.ps,
2589                                             (struct sockaddr *) &route->addr.dst_addr);
2590         req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
2591         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2592
2593         ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
2594         if (ret) {
2595                 ib_destroy_cm_id(id_priv->cm_id.ib);
2596                 id_priv->cm_id.ib = NULL;
2597         }
2598 out:
2599         kfree(req.private_data);
2600         return ret;
2601 }
2602
2603 static int cma_connect_ib(struct rdma_id_private *id_priv,
2604                           struct rdma_conn_param *conn_param)
2605 {
2606         struct ib_cm_req_param req;
2607         struct rdma_route *route;
2608         void *private_data;
2609         struct ib_cm_id *id;
2610         int offset, ret;
2611
2612         memset(&req, 0, sizeof req);
2613         offset = cma_user_data_offset(id_priv->id.ps);
2614         req.private_data_len = offset + conn_param->private_data_len;
2615         if (req.private_data_len < conn_param->private_data_len)
2616                 return -EINVAL;
2617
2618         private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2619         if (!private_data)
2620                 return -ENOMEM;
2621
2622         if (conn_param->private_data && conn_param->private_data_len)
2623                 memcpy(private_data + offset, conn_param->private_data,
2624                        conn_param->private_data_len);
2625
2626         id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv);
2627         if (IS_ERR(id)) {
2628                 ret = PTR_ERR(id);
2629                 goto out;
2630         }
2631         id_priv->cm_id.ib = id;
2632
2633         route = &id_priv->id.route;
2634         ret = cma_format_hdr(private_data, id_priv->id.ps, route);
2635         if (ret)
2636                 goto out;
2637         req.private_data = private_data;
2638
2639         req.primary_path = &route->path_rec[0];
2640         if (route->num_paths == 2)
2641                 req.alternate_path = &route->path_rec[1];
2642
2643         req.service_id = cma_get_service_id(id_priv->id.ps,
2644                                             (struct sockaddr *) &route->addr.dst_addr);
2645         req.qp_num = id_priv->qp_num;
2646         req.qp_type = id_priv->id.qp_type;
2647         req.starting_psn = id_priv->seq_num;
2648         req.responder_resources = conn_param->responder_resources;
2649         req.initiator_depth = conn_param->initiator_depth;
2650         req.flow_control = conn_param->flow_control;
2651         req.retry_count = conn_param->retry_count;
2652         req.rnr_retry_count = conn_param->rnr_retry_count;
2653         req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2654         req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2655         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2656         req.srq = id_priv->srq ? 1 : 0;
2657
2658         ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
2659 out:
2660         if (ret && !IS_ERR(id)) {
2661                 ib_destroy_cm_id(id);
2662                 id_priv->cm_id.ib = NULL;
2663         }
2664
2665         kfree(private_data);
2666         return ret;
2667 }
2668
2669 static int cma_connect_iw(struct rdma_id_private *id_priv,
2670                           struct rdma_conn_param *conn_param)
2671 {
2672         struct iw_cm_id *cm_id;
2673         struct sockaddr_in* sin;
2674         int ret;
2675         struct iw_cm_conn_param iw_param;
2676
2677         cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
2678         if (IS_ERR(cm_id))
2679                 return PTR_ERR(cm_id);
2680
2681         id_priv->cm_id.iw = cm_id;
2682
2683         sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr;
2684         cm_id->local_addr = *sin;
2685
2686         sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr;
2687         cm_id->remote_addr = *sin;
2688
2689         ret = cma_modify_qp_rtr(id_priv, conn_param);
2690         if (ret)
2691                 goto out;
2692
2693         if (conn_param) {
2694                 iw_param.ord = conn_param->initiator_depth;
2695                 iw_param.ird = conn_param->responder_resources;
2696                 iw_param.private_data = conn_param->private_data;
2697                 iw_param.private_data_len = conn_param->private_data_len;
2698                 iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num;
2699         } else {
2700                 memset(&iw_param, 0, sizeof iw_param);
2701                 iw_param.qpn = id_priv->qp_num;
2702         }
2703         ret = iw_cm_connect(cm_id, &iw_param);
2704 out:
2705         if (ret) {
2706                 iw_destroy_cm_id(cm_id);
2707                 id_priv->cm_id.iw = NULL;
2708         }
2709         return ret;
2710 }
2711
2712 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2713 {
2714         struct rdma_id_private *id_priv;
2715         int ret;
2716
2717         id_priv = container_of(id, struct rdma_id_private, id);
2718         if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT))
2719                 return -EINVAL;
2720
2721         if (!id->qp) {
2722                 id_priv->qp_num = conn_param->qp_num;
2723                 id_priv->srq = conn_param->srq;
2724         }
2725
2726         switch (rdma_node_get_transport(id->device->node_type)) {
2727         case RDMA_TRANSPORT_IB:
2728                 if (id->qp_type == IB_QPT_UD)
2729                         ret = cma_resolve_ib_udp(id_priv, conn_param);
2730                 else
2731                         ret = cma_connect_ib(id_priv, conn_param);
2732                 break;
2733         case RDMA_TRANSPORT_IWARP:
2734                 ret = cma_connect_iw(id_priv, conn_param);
2735                 break;
2736         default:
2737                 ret = -ENOSYS;
2738                 break;
2739         }
2740         if (ret)
2741                 goto err;
2742
2743         return 0;
2744 err:
2745         cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED);
2746         return ret;
2747 }
2748 EXPORT_SYMBOL(rdma_connect);
2749
2750 static int cma_accept_ib(struct rdma_id_private *id_priv,
2751                          struct rdma_conn_param *conn_param)
2752 {
2753         struct ib_cm_rep_param rep;
2754         int ret;
2755
2756         ret = cma_modify_qp_rtr(id_priv, conn_param);
2757         if (ret)
2758                 goto out;
2759
2760         ret = cma_modify_qp_rts(id_priv, conn_param);
2761         if (ret)
2762                 goto out;
2763
2764         memset(&rep, 0, sizeof rep);
2765         rep.qp_num = id_priv->qp_num;
2766         rep.starting_psn = id_priv->seq_num;
2767         rep.private_data = conn_param->private_data;
2768         rep.private_data_len = conn_param->private_data_len;
2769         rep.responder_resources = conn_param->responder_resources;
2770         rep.initiator_depth = conn_param->initiator_depth;
2771         rep.failover_accepted = 0;
2772         rep.flow_control = conn_param->flow_control;
2773         rep.rnr_retry_count = conn_param->rnr_retry_count;
2774         rep.srq = id_priv->srq ? 1 : 0;
2775
2776         ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
2777 out:
2778         return ret;
2779 }
2780
2781 static int cma_accept_iw(struct rdma_id_private *id_priv,
2782                   struct rdma_conn_param *conn_param)
2783 {
2784         struct iw_cm_conn_param iw_param;
2785         int ret;
2786
2787         ret = cma_modify_qp_rtr(id_priv, conn_param);
2788         if (ret)
2789                 return ret;
2790
2791         iw_param.ord = conn_param->initiator_depth;
2792         iw_param.ird = conn_param->responder_resources;
2793         iw_param.private_data = conn_param->private_data;
2794         iw_param.private_data_len = conn_param->private_data_len;
2795         if (id_priv->id.qp) {
2796                 iw_param.qpn = id_priv->qp_num;
2797         } else
2798                 iw_param.qpn = conn_param->qp_num;
2799
2800         return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
2801 }
2802
2803 static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
2804                              enum ib_cm_sidr_status status,
2805                              const void *private_data, int private_data_len)
2806 {
2807         struct ib_cm_sidr_rep_param rep;
2808         int ret;
2809
2810         memset(&rep, 0, sizeof rep);
2811         rep.status = status;
2812         if (status == IB_SIDR_SUCCESS) {
2813                 ret = cma_set_qkey(id_priv);
2814                 if (ret)
2815                         return ret;
2816                 rep.qp_num = id_priv->qp_num;
2817                 rep.qkey = id_priv->qkey;
2818         }
2819         rep.private_data = private_data;
2820         rep.private_data_len = private_data_len;
2821
2822         return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
2823 }
2824
2825 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2826 {
2827         struct rdma_id_private *id_priv;
2828         int ret;
2829
2830         id_priv = container_of(id, struct rdma_id_private, id);
2831
2832         id_priv->owner = task_pid_nr(current);
2833
2834         if (!cma_comp(id_priv, RDMA_CM_CONNECT))
2835                 return -EINVAL;
2836
2837         if (!id->qp && conn_param) {
2838                 id_priv->qp_num = conn_param->qp_num;
2839                 id_priv->srq = conn_param->srq;
2840         }
2841
2842         switch (rdma_node_get_transport(id->device->node_type)) {
2843         case RDMA_TRANSPORT_IB:
2844                 if (id->qp_type == IB_QPT_UD) {
2845                         if (conn_param)
2846                                 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
2847                                                         conn_param->private_data,
2848                                                         conn_param->private_data_len);
2849                         else
2850                                 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
2851                                                         NULL, 0);
2852                 } else {
2853                         if (conn_param)
2854                                 ret = cma_accept_ib(id_priv, conn_param);
2855                         else
2856                                 ret = cma_rep_recv(id_priv);
2857                 }
2858                 break;
2859         case RDMA_TRANSPORT_IWARP:
2860                 ret = cma_accept_iw(id_priv, conn_param);
2861                 break;
2862         default:
2863                 ret = -ENOSYS;
2864                 break;
2865         }
2866
2867         if (ret)
2868                 goto reject;
2869
2870         return 0;
2871 reject:
2872         cma_modify_qp_err(id_priv);
2873         rdma_reject(id, NULL, 0);
2874         return ret;
2875 }
2876 EXPORT_SYMBOL(rdma_accept);
2877
2878 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
2879 {
2880         struct rdma_id_private *id_priv;
2881         int ret;
2882
2883         id_priv = container_of(id, struct rdma_id_private, id);
2884         if (!id_priv->cm_id.ib)
2885                 return -EINVAL;
2886
2887         switch (id->device->node_type) {
2888         case RDMA_NODE_IB_CA:
2889                 ret = ib_cm_notify(id_priv->cm_id.ib, event);
2890                 break;
2891         default:
2892                 ret = 0;
2893                 break;
2894         }
2895         return ret;
2896 }
2897 EXPORT_SYMBOL(rdma_notify);
2898
2899 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
2900                 u8 private_data_len)
2901 {
2902         struct rdma_id_private *id_priv;
2903         int ret;
2904
2905         id_priv = container_of(id, struct rdma_id_private, id);
2906         if (!id_priv->cm_id.ib)
2907                 return -EINVAL;
2908
2909         switch (rdma_node_get_transport(id->device->node_type)) {
2910         case RDMA_TRANSPORT_IB:
2911                 if (id->qp_type == IB_QPT_UD)
2912                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
2913                                                 private_data, private_data_len);
2914                 else
2915                         ret = ib_send_cm_rej(id_priv->cm_id.ib,
2916                                              IB_CM_REJ_CONSUMER_DEFINED, NULL,
2917                                              0, private_data, private_data_len);
2918                 break;
2919         case RDMA_TRANSPORT_IWARP:
2920                 ret = iw_cm_reject(id_priv->cm_id.iw,
2921                                    private_data, private_data_len);
2922                 break;
2923         default:
2924                 ret = -ENOSYS;
2925                 break;
2926         }
2927         return ret;
2928 }
2929 EXPORT_SYMBOL(rdma_reject);
2930
2931 int rdma_disconnect(struct rdma_cm_id *id)
2932 {
2933         struct rdma_id_private *id_priv;
2934         int ret;
2935
2936         id_priv = container_of(id, struct rdma_id_private, id);
2937         if (!id_priv->cm_id.ib)
2938                 return -EINVAL;
2939
2940         switch (rdma_node_get_transport(id->device->node_type)) {
2941         case RDMA_TRANSPORT_IB:
2942                 ret = cma_modify_qp_err(id_priv);
2943                 if (ret)
2944                         goto out;
2945                 /* Initiate or respond to a disconnect. */
2946                 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0))
2947                         ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
2948                 break;
2949         case RDMA_TRANSPORT_IWARP:
2950                 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
2951                 break;
2952         default:
2953                 ret = -EINVAL;
2954                 break;
2955         }
2956 out:
2957         return ret;
2958 }
2959 EXPORT_SYMBOL(rdma_disconnect);
2960
2961 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
2962 {
2963         struct rdma_id_private *id_priv;
2964         struct cma_multicast *mc = multicast->context;
2965         struct rdma_cm_event event;
2966         int ret;
2967
2968         id_priv = mc->id_priv;
2969         if (cma_disable_callback(id_priv, RDMA_CM_ADDR_BOUND) &&
2970             cma_disable_callback(id_priv, RDMA_CM_ADDR_RESOLVED))
2971                 return 0;
2972
2973         mutex_lock(&id_priv->qp_mutex);
2974         if (!status && id_priv->id.qp)
2975                 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
2976                                          be16_to_cpu(multicast->rec.mlid));
2977         mutex_unlock(&id_priv->qp_mutex);
2978
2979         memset(&event, 0, sizeof event);
2980         event.status = status;
2981         event.param.ud.private_data = mc->context;
2982         if (!status) {
2983                 event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
2984                 ib_init_ah_from_mcmember(id_priv->id.device,
2985                                          id_priv->id.port_num, &multicast->rec,
2986                                          &event.param.ud.ah_attr);
2987                 event.param.ud.qp_num = 0xFFFFFF;
2988                 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
2989         } else
2990                 event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
2991
2992         ret = id_priv->id.event_handler(&id_priv->id, &event);
2993         if (ret) {
2994                 cma_exch(id_priv, RDMA_CM_DESTROYING);
2995                 mutex_unlock(&id_priv->handler_mutex);
2996                 rdma_destroy_id(&id_priv->id);
2997                 return 0;
2998         }
2999
3000         mutex_unlock(&id_priv->handler_mutex);
3001         return 0;
3002 }
3003
3004 static void cma_set_mgid(struct rdma_id_private *id_priv,
3005                          struct sockaddr *addr, union ib_gid *mgid)
3006 {
3007         unsigned char mc_map[MAX_ADDR_LEN];
3008         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3009         struct sockaddr_in *sin = (struct sockaddr_in *) addr;
3010         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
3011
3012         if (cma_any_addr(addr)) {
3013                 memset(mgid, 0, sizeof *mgid);
3014         } else if ((addr->sa_family == AF_INET6) &&
3015                    ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) ==
3016                                                                  0xFF10A01B)) {
3017                 /* IPv6 address is an SA assigned MGID. */
3018                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
3019         } else if ((addr->sa_family == AF_INET6)) {
3020                 ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map);
3021                 if (id_priv->id.ps == RDMA_PS_UDP)
3022                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
3023                 *mgid = *(union ib_gid *) (mc_map + 4);
3024         } else {
3025                 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
3026                 if (id_priv->id.ps == RDMA_PS_UDP)
3027                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
3028                 *mgid = *(union ib_gid *) (mc_map + 4);
3029         }
3030 }
3031
3032 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
3033                                  struct cma_multicast *mc)
3034 {
3035         struct ib_sa_mcmember_rec rec;
3036         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3037         ib_sa_comp_mask comp_mask;
3038         int ret;
3039
3040         ib_addr_get_mgid(dev_addr, &rec.mgid);
3041         ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
3042                                      &rec.mgid, &rec);
3043         if (ret)
3044                 return ret;
3045
3046         cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
3047         if (id_priv->id.ps == RDMA_PS_UDP)
3048                 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
3049         rdma_addr_get_sgid(dev_addr, &rec.port_gid);
3050         rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
3051         rec.join_state = 1;
3052
3053         comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
3054                     IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
3055                     IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
3056                     IB_SA_MCMEMBER_REC_FLOW_LABEL |
3057                     IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
3058
3059         if (id_priv->id.ps == RDMA_PS_IPOIB)
3060                 comp_mask |= IB_SA_MCMEMBER_REC_RATE |
3061                              IB_SA_MCMEMBER_REC_RATE_SELECTOR |
3062                              IB_SA_MCMEMBER_REC_MTU_SELECTOR |
3063                              IB_SA_MCMEMBER_REC_MTU |
3064                              IB_SA_MCMEMBER_REC_HOP_LIMIT;
3065
3066         mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
3067                                                 id_priv->id.port_num, &rec,
3068                                                 comp_mask, GFP_KERNEL,
3069                                                 cma_ib_mc_handler, mc);
3070         return PTR_RET(mc->multicast.ib);
3071 }
3072
3073 static void iboe_mcast_work_handler(struct work_struct *work)
3074 {
3075         struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work);
3076         struct cma_multicast *mc = mw->mc;
3077         struct ib_sa_multicast *m = mc->multicast.ib;
3078
3079         mc->multicast.ib->context = mc;
3080         cma_ib_mc_handler(0, m);
3081         kref_put(&mc->mcref, release_mc);
3082         kfree(mw);
3083 }
3084
3085 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid)
3086 {
3087         struct sockaddr_in *sin = (struct sockaddr_in *)addr;
3088         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
3089
3090         if (cma_any_addr(addr)) {
3091                 memset(mgid, 0, sizeof *mgid);
3092         } else if (addr->sa_family == AF_INET6) {
3093                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
3094         } else {
3095                 mgid->raw[0] = 0xff;
3096                 mgid->raw[1] = 0x0e;
3097                 mgid->raw[2] = 0;
3098                 mgid->raw[3] = 0;
3099                 mgid->raw[4] = 0;
3100                 mgid->raw[5] = 0;
3101                 mgid->raw[6] = 0;
3102                 mgid->raw[7] = 0;
3103                 mgid->raw[8] = 0;
3104                 mgid->raw[9] = 0;
3105                 mgid->raw[10] = 0xff;
3106                 mgid->raw[11] = 0xff;
3107                 *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr;
3108         }
3109 }
3110
3111 static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
3112                                    struct cma_multicast *mc)
3113 {
3114         struct iboe_mcast_work *work;
3115         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3116         int err;
3117         struct sockaddr *addr = (struct sockaddr *)&mc->addr;
3118         struct net_device *ndev = NULL;
3119
3120         if (cma_zero_addr((struct sockaddr *)&mc->addr))
3121                 return -EINVAL;
3122
3123         work = kzalloc(sizeof *work, GFP_KERNEL);
3124         if (!work)
3125                 return -ENOMEM;
3126
3127         mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL);
3128         if (!mc->multicast.ib) {
3129                 err = -ENOMEM;
3130                 goto out1;
3131         }
3132
3133         cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid);
3134
3135         mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff);
3136         if (id_priv->id.ps == RDMA_PS_UDP)
3137                 mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
3138
3139         if (dev_addr->bound_dev_if)
3140                 ndev = dev_get_by_index(&init_net, dev_addr->bound_dev_if);
3141         if (!ndev) {
3142                 err = -ENODEV;
3143                 goto out2;
3144         }
3145         mc->multicast.ib->rec.rate = iboe_get_rate(ndev);
3146         mc->multicast.ib->rec.hop_limit = 1;
3147         mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu);
3148         dev_put(ndev);
3149         if (!mc->multicast.ib->rec.mtu) {
3150                 err = -EINVAL;
3151                 goto out2;
3152         }
3153         iboe_addr_get_sgid(dev_addr, &mc->multicast.ib->rec.port_gid);
3154         work->id = id_priv;
3155         work->mc = mc;
3156         INIT_WORK(&work->work, iboe_mcast_work_handler);
3157         kref_get(&mc->mcref);
3158         queue_work(cma_wq, &work->work);
3159
3160         return 0;
3161
3162 out2:
3163         kfree(mc->multicast.ib);
3164 out1:
3165         kfree(work);
3166         return err;
3167 }
3168
3169 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
3170                         void *context)
3171 {
3172         struct rdma_id_private *id_priv;
3173         struct cma_multicast *mc;
3174         int ret;
3175
3176         id_priv = container_of(id, struct rdma_id_private, id);
3177         if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) &&
3178             !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED))
3179                 return -EINVAL;
3180
3181         mc = kmalloc(sizeof *mc, GFP_KERNEL);
3182         if (!mc)
3183                 return -ENOMEM;
3184
3185         memcpy(&mc->addr, addr, ip_addr_size(addr));
3186         mc->context = context;
3187         mc->id_priv = id_priv;
3188
3189         spin_lock(&id_priv->lock);
3190         list_add(&mc->list, &id_priv->mc_list);
3191         spin_unlock(&id_priv->lock);
3192
3193         switch (rdma_node_get_transport(id->device->node_type)) {
3194         case RDMA_TRANSPORT_IB:
3195                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
3196                 case IB_LINK_LAYER_INFINIBAND:
3197                         ret = cma_join_ib_multicast(id_priv, mc);
3198                         break;
3199                 case IB_LINK_LAYER_ETHERNET:
3200                         kref_init(&mc->mcref);
3201                         ret = cma_iboe_join_multicast(id_priv, mc);
3202                         break;
3203                 default:
3204                         ret = -EINVAL;
3205                 }
3206                 break;
3207         default:
3208                 ret = -ENOSYS;
3209                 break;
3210         }
3211
3212         if (ret) {
3213                 spin_lock_irq(&id_priv->lock);
3214                 list_del(&mc->list);
3215                 spin_unlock_irq(&id_priv->lock);
3216                 kfree(mc);
3217         }
3218         return ret;
3219 }
3220 EXPORT_SYMBOL(rdma_join_multicast);
3221
3222 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
3223 {
3224         struct rdma_id_private *id_priv;
3225         struct cma_multicast *mc;
3226
3227         id_priv = container_of(id, struct rdma_id_private, id);
3228         spin_lock_irq(&id_priv->lock);
3229         list_for_each_entry(mc, &id_priv->mc_list, list) {
3230                 if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) {
3231                         list_del(&mc->list);
3232                         spin_unlock_irq(&id_priv->lock);
3233
3234                         if (id->qp)
3235                                 ib_detach_mcast(id->qp,
3236                                                 &mc->multicast.ib->rec.mgid,
3237                                                 be16_to_cpu(mc->multicast.ib->rec.mlid));
3238                         if (rdma_node_get_transport(id_priv->cma_dev->device->node_type) == RDMA_TRANSPORT_IB) {
3239                                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
3240                                 case IB_LINK_LAYER_INFINIBAND:
3241                                         ib_sa_free_multicast(mc->multicast.ib);
3242                                         kfree(mc);
3243                                         break;
3244                                 case IB_LINK_LAYER_ETHERNET:
3245                                         kref_put(&mc->mcref, release_mc);
3246                                         break;
3247                                 default:
3248                                         break;
3249                                 }
3250                         }
3251                         return;
3252                 }
3253         }
3254         spin_unlock_irq(&id_priv->lock);
3255 }
3256 EXPORT_SYMBOL(rdma_leave_multicast);
3257
3258 static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)
3259 {
3260         struct rdma_dev_addr *dev_addr;
3261         struct cma_ndev_work *work;
3262
3263         dev_addr = &id_priv->id.route.addr.dev_addr;
3264
3265         if ((dev_addr->bound_dev_if == ndev->ifindex) &&
3266             memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) {
3267                 printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n",
3268                        ndev->name, &id_priv->id);
3269                 work = kzalloc(sizeof *work, GFP_KERNEL);
3270                 if (!work)
3271                         return -ENOMEM;
3272
3273                 INIT_WORK(&work->work, cma_ndev_work_handler);
3274                 work->id = id_priv;
3275                 work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
3276                 atomic_inc(&id_priv->refcount);
3277                 queue_work(cma_wq, &work->work);
3278         }
3279
3280         return 0;
3281 }
3282
3283 static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
3284                                void *ctx)
3285 {
3286         struct net_device *ndev = (struct net_device *)ctx;
3287         struct cma_device *cma_dev;
3288         struct rdma_id_private *id_priv;
3289         int ret = NOTIFY_DONE;
3290
3291         if (dev_net(ndev) != &init_net)
3292                 return NOTIFY_DONE;
3293
3294         if (event != NETDEV_BONDING_FAILOVER)
3295                 return NOTIFY_DONE;
3296
3297         if (!(ndev->flags & IFF_MASTER) || !(ndev->priv_flags & IFF_BONDING))
3298                 return NOTIFY_DONE;
3299
3300         mutex_lock(&lock);
3301         list_for_each_entry(cma_dev, &dev_list, list)
3302                 list_for_each_entry(id_priv, &cma_dev->id_list, list) {
3303                         ret = cma_netdev_change(ndev, id_priv);
3304                         if (ret)
3305                                 goto out;
3306                 }
3307
3308 out:
3309         mutex_unlock(&lock);
3310         return ret;
3311 }
3312
3313 static struct notifier_block cma_nb = {
3314         .notifier_call = cma_netdev_callback
3315 };
3316
3317 static void cma_add_one(struct ib_device *device)
3318 {
3319         struct cma_device *cma_dev;
3320         struct rdma_id_private *id_priv;
3321
3322         cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
3323         if (!cma_dev)
3324                 return;
3325
3326         cma_dev->device = device;
3327
3328         init_completion(&cma_dev->comp);
3329         atomic_set(&cma_dev->refcount, 1);
3330         INIT_LIST_HEAD(&cma_dev->id_list);
3331         ib_set_client_data(device, &cma_client, cma_dev);
3332
3333         mutex_lock(&lock);
3334         list_add_tail(&cma_dev->list, &dev_list);
3335         list_for_each_entry(id_priv, &listen_any_list, list)
3336                 cma_listen_on_dev(id_priv, cma_dev);
3337         mutex_unlock(&lock);
3338 }
3339
3340 static int cma_remove_id_dev(struct rdma_id_private *id_priv)
3341 {
3342         struct rdma_cm_event event;
3343         enum rdma_cm_state state;
3344         int ret = 0;
3345
3346         /* Record that we want to remove the device */
3347         state = cma_exch(id_priv, RDMA_CM_DEVICE_REMOVAL);
3348         if (state == RDMA_CM_DESTROYING)
3349                 return 0;
3350
3351         cma_cancel_operation(id_priv, state);
3352         mutex_lock(&id_priv->handler_mutex);
3353
3354         /* Check for destruction from another callback. */
3355         if (!cma_comp(id_priv, RDMA_CM_DEVICE_REMOVAL))
3356                 goto out;
3357
3358         memset(&event, 0, sizeof event);
3359         event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
3360         ret = id_priv->id.event_handler(&id_priv->id, &event);
3361 out:
3362         mutex_unlock(&id_priv->handler_mutex);
3363         return ret;
3364 }
3365
3366 static void cma_process_remove(struct cma_device *cma_dev)
3367 {
3368         struct rdma_id_private *id_priv;
3369         int ret;
3370
3371         mutex_lock(&lock);
3372         while (!list_empty(&cma_dev->id_list)) {
3373                 id_priv = list_entry(cma_dev->id_list.next,
3374                                      struct rdma_id_private, list);
3375
3376                 list_del(&id_priv->listen_list);
3377                 list_del_init(&id_priv->list);
3378                 atomic_inc(&id_priv->refcount);
3379                 mutex_unlock(&lock);
3380
3381                 ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);
3382                 cma_deref_id(id_priv);
3383                 if (ret)
3384                         rdma_destroy_id(&id_priv->id);
3385
3386                 mutex_lock(&lock);
3387         }
3388         mutex_unlock(&lock);
3389
3390         cma_deref_dev(cma_dev);
3391         wait_for_completion(&cma_dev->comp);
3392 }
3393
3394 static void cma_remove_one(struct ib_device *device)
3395 {
3396         struct cma_device *cma_dev;
3397
3398         cma_dev = ib_get_client_data(device, &cma_client);
3399         if (!cma_dev)
3400                 return;
3401
3402         mutex_lock(&lock);
3403         list_del(&cma_dev->list);
3404         mutex_unlock(&lock);
3405
3406         cma_process_remove(cma_dev);
3407         kfree(cma_dev);
3408 }
3409
3410 static int cma_get_id_stats(struct sk_buff *skb, struct netlink_callback *cb)
3411 {
3412         struct nlmsghdr *nlh;
3413         struct rdma_cm_id_stats *id_stats;
3414         struct rdma_id_private *id_priv;
3415         struct rdma_cm_id *id = NULL;
3416         struct cma_device *cma_dev;
3417         int i_dev = 0, i_id = 0;
3418
3419         /*
3420          * We export all of the IDs as a sequence of messages.  Each
3421          * ID gets its own netlink message.
3422          */
3423         mutex_lock(&lock);
3424
3425         list_for_each_entry(cma_dev, &dev_list, list) {
3426                 if (i_dev < cb->args[0]) {
3427                         i_dev++;
3428                         continue;
3429                 }
3430
3431                 i_id = 0;
3432                 list_for_each_entry(id_priv, &cma_dev->id_list, list) {
3433                         if (i_id < cb->args[1]) {
3434                                 i_id++;
3435                                 continue;
3436                         }
3437
3438                         id_stats = ibnl_put_msg(skb, &nlh, cb->nlh->nlmsg_seq,
3439                                                 sizeof *id_stats, RDMA_NL_RDMA_CM,
3440                                                 RDMA_NL_RDMA_CM_ID_STATS);
3441                         if (!id_stats)
3442                                 goto out;
3443
3444                         memset(id_stats, 0, sizeof *id_stats);
3445                         id = &id_priv->id;
3446                         id_stats->node_type = id->route.addr.dev_addr.dev_type;
3447                         id_stats->port_num = id->port_num;
3448                         id_stats->bound_dev_if =
3449                                 id->route.addr.dev_addr.bound_dev_if;
3450
3451                         if (id->route.addr.src_addr.ss_family == AF_INET) {
3452                                 if (ibnl_put_attr(skb, nlh,
3453                                                   sizeof(struct sockaddr_in),
3454                                                   &id->route.addr.src_addr,
3455                                                   RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {
3456                                         goto out;
3457                                 }
3458                                 if (ibnl_put_attr(skb, nlh,
3459                                                   sizeof(struct sockaddr_in),
3460                                                   &id->route.addr.dst_addr,
3461                                                   RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {
3462                                         goto out;
3463                                 }
3464                         } else if (id->route.addr.src_addr.ss_family == AF_INET6) {
3465                                 if (ibnl_put_attr(skb, nlh,
3466                                                   sizeof(struct sockaddr_in6),
3467                                                   &id->route.addr.src_addr,
3468                                                   RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {
3469                                         goto out;
3470                                 }
3471                                 if (ibnl_put_attr(skb, nlh,
3472                                                   sizeof(struct sockaddr_in6),
3473                                                   &id->route.addr.dst_addr,
3474                                                   RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {
3475                                         goto out;
3476                                 }
3477                         }
3478
3479                         id_stats->pid           = id_priv->owner;
3480                         id_stats->port_space    = id->ps;
3481                         id_stats->cm_state      = id_priv->state;
3482                         id_stats->qp_num        = id_priv->qp_num;
3483                         id_stats->qp_type       = id->qp_type;
3484
3485                         i_id++;
3486                 }
3487
3488                 cb->args[1] = 0;
3489                 i_dev++;
3490         }
3491
3492 out:
3493         mutex_unlock(&lock);
3494         cb->args[0] = i_dev;
3495         cb->args[1] = i_id;
3496
3497         return skb->len;
3498 }
3499
3500 static const struct ibnl_client_cbs cma_cb_table[] = {
3501         [RDMA_NL_RDMA_CM_ID_STATS] = { .dump = cma_get_id_stats },
3502 };
3503
3504 static int __init cma_init(void)
3505 {
3506         int ret;
3507
3508         cma_wq = create_singlethread_workqueue("rdma_cm");
3509         if (!cma_wq)
3510                 return -ENOMEM;
3511
3512         ib_sa_register_client(&sa_client);
3513         rdma_addr_register_client(&addr_client);
3514         register_netdevice_notifier(&cma_nb);
3515
3516         ret = ib_register_client(&cma_client);
3517         if (ret)
3518                 goto err;
3519
3520         if (ibnl_add_client(RDMA_NL_RDMA_CM, RDMA_NL_RDMA_CM_NUM_OPS, cma_cb_table))
3521                 printk(KERN_WARNING "RDMA CMA: failed to add netlink callback\n");
3522
3523         return 0;
3524
3525 err:
3526         unregister_netdevice_notifier(&cma_nb);
3527         rdma_addr_unregister_client(&addr_client);
3528         ib_sa_unregister_client(&sa_client);
3529         destroy_workqueue(cma_wq);
3530         return ret;
3531 }
3532
3533 static void __exit cma_cleanup(void)
3534 {
3535         ibnl_remove_client(RDMA_NL_RDMA_CM);
3536         ib_unregister_client(&cma_client);
3537         unregister_netdevice_notifier(&cma_nb);
3538         rdma_addr_unregister_client(&addr_client);
3539         ib_sa_unregister_client(&sa_client);
3540         destroy_workqueue(cma_wq);
3541         idr_destroy(&sdp_ps);
3542         idr_destroy(&tcp_ps);
3543         idr_destroy(&udp_ps);
3544         idr_destroy(&ipoib_ps);
3545         idr_destroy(&ib_ps);
3546 }
3547
3548 module_init(cma_init);
3549 module_exit(cma_cleanup);