drivers/infiniband/ulp/srp/ib_srp.c

   1 /*
   2  * Copyright (c) 2005 Cisco Systems.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/module.h>
  36 #include <linux/init.h>
  37 #include <linux/slab.h>
  38 #include <linux/err.h>
  39 #include <linux/string.h>
  40 #include <linux/parser.h>
  41 #include <linux/random.h>
  42 #include <linux/jiffies.h>
  43 #include <linux/lockdep.h>
  44 #include <linux/inet.h>
  45 #include <rdma/ib_cache.h>
  46
  47 #include <linux/atomic.h>
  48
  49 #include <scsi/scsi.h>
  50 #include <scsi/scsi_device.h>
  51 #include <scsi/scsi_dbg.h>
  52 #include <scsi/scsi_tcq.h>
  53 #include <scsi/srp.h>
  54 #include <scsi/scsi_transport_srp.h>
  55
  56 #include "ib_srp.h"
  57
  58 #define DRV_NAME        "ib_srp"
  59 #define PFX             DRV_NAME ": "
  60
  61 MODULE_AUTHOR("Roland Dreier");
  62 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator");
  63 MODULE_LICENSE("Dual BSD/GPL");
  64
  65 #if !defined(CONFIG_DYNAMIC_DEBUG)
  66 #define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt)
  67 #define DYNAMIC_DEBUG_BRANCH(descriptor) false
  68 #endif
  69
  70 static unsigned int srp_sg_tablesize;
  71 static unsigned int cmd_sg_entries;
  72 static unsigned int indirect_sg_entries;
  73 static bool allow_ext_sg;
  74 static bool prefer_fr = true;
  75 static bool register_always = true;
  76 static bool never_register;
  77 static int topspin_workarounds = 1;
  78
  79 module_param(srp_sg_tablesize, uint, 0444);
  80 MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
  81
  82 module_param(cmd_sg_entries, uint, 0444);
  83 MODULE_PARM_DESC(cmd_sg_entries,
  84                  "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
  85
  86 module_param(indirect_sg_entries, uint, 0444);
  87 MODULE_PARM_DESC(indirect_sg_entries,
  88                  "Default max number of gather/scatter entries (default is 12, max is " __stringify(SG_MAX_SEGMENTS) ")");
  89
  90 module_param(allow_ext_sg, bool, 0444);
  91 MODULE_PARM_DESC(allow_ext_sg,
  92                   "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");
  93
  94 module_param(topspin_workarounds, int, 0444);
  95 MODULE_PARM_DESC(topspin_workarounds,
  96                  "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
  97
  98 module_param(prefer_fr, bool, 0444);
  99 MODULE_PARM_DESC(prefer_fr,
 100 "Whether to use fast registration if both FMR and fast registration are supported");
 101
 102 module_param(register_always, bool, 0444);
 103 MODULE_PARM_DESC(register_always,
 104                  "Use memory registration even for contiguous memory regions");
 105
 106 module_param(never_register, bool, 0444);
 107 MODULE_PARM_DESC(never_register, "Never register memory");
 108
 109 static const struct kernel_param_ops srp_tmo_ops;
 110
 111 static int srp_reconnect_delay = 10;
 112 module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
 113                 S_IRUGO | S_IWUSR);
 114 MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");
 115
 116 static int srp_fast_io_fail_tmo = 15;
 117 module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
 118                 S_IRUGO | S_IWUSR);
 119 MODULE_PARM_DESC(fast_io_fail_tmo,
 120                  "Number of seconds between the observation of a transport"
 121                  " layer error and failing all I/O. \"off\" means that this"
 122                  " functionality is disabled.");
 123
 124 static int srp_dev_loss_tmo = 600;
 125 module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
 126                 S_IRUGO | S_IWUSR);
 127 MODULE_PARM_DESC(dev_loss_tmo,
 128                  "Maximum number of seconds that the SRP transport should"
 129                  " insulate transport layer errors. After this time has been"
 130                  " exceeded the SCSI host is removed. Should be"
 131                  " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
 132                  " if fast_io_fail_tmo has not been set. \"off\" means that"
 133                  " this functionality is disabled.");
 134
 135 static bool srp_use_imm_data = true;
 136 module_param_named(use_imm_data, srp_use_imm_data, bool, 0644);
 137 MODULE_PARM_DESC(use_imm_data,
 138                  "Whether or not to request permission to use immediate data during SRP login.");
 139
 140 static unsigned int srp_max_imm_data = 8 * 1024;
 141 module_param_named(max_imm_data, srp_max_imm_data, uint, 0644);
 142 MODULE_PARM_DESC(max_imm_data, "Maximum immediate data size.");
 143
 144 static unsigned ch_count;
 145 module_param(ch_count, uint, 0444);
 146 MODULE_PARM_DESC(ch_count,
 147                  "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");
 148
 149 static void srp_add_one(struct ib_device *device);
 150 static void srp_remove_one(struct ib_device *device, void *client_data);
 151 static void srp_rename_dev(struct ib_device *device, void *client_data);
 152 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc);
 153 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
 154                 const char *opname);
 155 static int srp_ib_cm_handler(struct ib_cm_id *cm_id,
 156                              const struct ib_cm_event *event);
 157 static int srp_rdma_cm_handler(struct rdma_cm_id *cm_id,
 158                                struct rdma_cm_event *event);
 159
 160 static struct scsi_transport_template *ib_srp_transport_template;
 161 static struct workqueue_struct *srp_remove_wq;
 162
 163 static struct ib_client srp_client = {
 164         .name   = "srp",
 165         .add    = srp_add_one,
 166         .remove = srp_remove_one,
 167         .rename = srp_rename_dev
 168 };
 169
 170 static struct ib_sa_client srp_sa_client;
 171
 172 static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
 173 {
 174         int tmo = *(int *)kp->arg;
 175
 176         if (tmo >= 0)
 177                 return sprintf(buffer, "%d", tmo);
 178         else
 179                 return sprintf(buffer, "off");
 180 }
 181
 182 static int srp_tmo_set(const char *val, const struct kernel_param *kp)
 183 {
 184         int tmo, res;
 185
 186         res = srp_parse_tmo(&tmo, val);
 187         if (res)
 188                 goto out;
 189
 190         if (kp->arg == &srp_reconnect_delay)
 191                 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
 192                                     srp_dev_loss_tmo);
 193         else if (kp->arg == &srp_fast_io_fail_tmo)
 194                 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
 195         else
 196                 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
 197                                     tmo);
 198         if (res)
 199                 goto out;
 200         *(int *)kp->arg = tmo;
 201
 202 out:
 203         return res;
 204 }
 205
 206 static const struct kernel_param_ops srp_tmo_ops = {
 207         .get = srp_tmo_get,
 208         .set = srp_tmo_set,
 209 };
 210
 211 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
 212 {
 213         return (struct srp_target_port *) host->hostdata;
 214 }
 215
 216 static const char *srp_target_info(struct Scsi_Host *host)
 217 {
 218         return host_to_target(host)->target_name;
 219 }
 220
 221 static int srp_target_is_topspin(struct srp_target_port *target)
 222 {
 223         static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
 224         static const u8 cisco_oui[3]   = { 0x00, 0x1b, 0x0d };
 225
 226         return topspin_workarounds &&
 227                 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
 228                  !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
 229 }
 230
 231 static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
 232                                    gfp_t gfp_mask,
 233                                    enum dma_data_direction direction)
 234 {
 235         struct srp_iu *iu;
 236
 237         iu = kmalloc(sizeof *iu, gfp_mask);
 238         if (!iu)
 239                 goto out;
 240
 241         iu->buf = kzalloc(size, gfp_mask);
 242         if (!iu->buf)
 243                 goto out_free_iu;
 244
 245         iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
 246                                     direction);
 247         if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
 248                 goto out_free_buf;
 249
 250         iu->size      = size;
 251         iu->direction = direction;
 252
 253         return iu;
 254
 255 out_free_buf:
 256         kfree(iu->buf);
 257 out_free_iu:
 258         kfree(iu);
 259 out:
 260         return NULL;
 261 }
 262
 263 static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
 264 {
 265         if (!iu)
 266                 return;
 267
 268         ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
 269                             iu->direction);
 270         kfree(iu->buf);
 271         kfree(iu);
 272 }
 273
 274 static void srp_qp_event(struct ib_event *event, void *context)
 275 {
 276         pr_debug("QP event %s (%d)\n",
 277                  ib_event_msg(event->event), event->event);
 278 }
 279
 280 static int srp_init_ib_qp(struct srp_target_port *target,
 281                           struct ib_qp *qp)
 282 {
 283         struct ib_qp_attr *attr;
 284         int ret;
 285
 286         attr = kmalloc(sizeof *attr, GFP_KERNEL);
 287         if (!attr)
 288                 return -ENOMEM;
 289
 290         ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
 291                                   target->srp_host->port,
 292                                   be16_to_cpu(target->ib_cm.pkey),
 293                                   &attr->pkey_index);
 294         if (ret)
 295                 goto out;
 296
 297         attr->qp_state        = IB_QPS_INIT;
 298         attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
 299                                     IB_ACCESS_REMOTE_WRITE);
 300         attr->port_num        = target->srp_host->port;
 301
 302         ret = ib_modify_qp(qp, attr,
 303                            IB_QP_STATE          |
 304                            IB_QP_PKEY_INDEX     |
 305                            IB_QP_ACCESS_FLAGS   |
 306                            IB_QP_PORT);
 307
 308 out:
 309         kfree(attr);
 310         return ret;
 311 }
 312
 313 static int srp_new_ib_cm_id(struct srp_rdma_ch *ch)
 314 {
 315         struct srp_target_port *target = ch->target;
 316         struct ib_cm_id *new_cm_id;
 317
 318         new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
 319                                     srp_ib_cm_handler, ch);
 320         if (IS_ERR(new_cm_id))
 321                 return PTR_ERR(new_cm_id);
 322
 323         if (ch->ib_cm.cm_id)
 324                 ib_destroy_cm_id(ch->ib_cm.cm_id);
 325         ch->ib_cm.cm_id = new_cm_id;
 326         if (rdma_cap_opa_ah(target->srp_host->srp_dev->dev,
 327                             target->srp_host->port))
 328                 ch->ib_cm.path.rec_type = SA_PATH_REC_TYPE_OPA;
 329         else
 330                 ch->ib_cm.path.rec_type = SA_PATH_REC_TYPE_IB;
 331         ch->ib_cm.path.sgid = target->sgid;
 332         ch->ib_cm.path.dgid = target->ib_cm.orig_dgid;
 333         ch->ib_cm.path.pkey = target->ib_cm.pkey;
 334         ch->ib_cm.path.service_id = target->ib_cm.service_id;
 335
 336         return 0;
 337 }
 338
 339 static int srp_new_rdma_cm_id(struct srp_rdma_ch *ch)
 340 {
 341         struct srp_target_port *target = ch->target;
 342         struct rdma_cm_id *new_cm_id;
 343         int ret;
 344
 345         new_cm_id = rdma_create_id(target->net, srp_rdma_cm_handler, ch,
 346                                    RDMA_PS_TCP, IB_QPT_RC);
 347         if (IS_ERR(new_cm_id)) {
 348                 ret = PTR_ERR(new_cm_id);
 349                 new_cm_id = NULL;
 350                 goto out;
 351         }
 352
 353         init_completion(&ch->done);
 354         ret = rdma_resolve_addr(new_cm_id, target->rdma_cm.src_specified ?
 355                                 (struct sockaddr *)&target->rdma_cm.src : NULL,
 356                                 (struct sockaddr *)&target->rdma_cm.dst,
 357                                 SRP_PATH_REC_TIMEOUT_MS);
 358         if (ret) {
 359                 pr_err("No route available from %pIS to %pIS (%d)\n",
 360                        &target->rdma_cm.src, &target->rdma_cm.dst, ret);
 361                 goto out;
 362         }
 363         ret = wait_for_completion_interruptible(&ch->done);
 364         if (ret < 0)
 365                 goto out;
 366
 367         ret = ch->status;
 368         if (ret) {
 369                 pr_err("Resolving address %pIS failed (%d)\n",
 370                        &target->rdma_cm.dst, ret);
 371                 goto out;
 372         }
 373
 374         swap(ch->rdma_cm.cm_id, new_cm_id);
 375
 376 out:
 377         if (new_cm_id)
 378                 rdma_destroy_id(new_cm_id);
 379
 380         return ret;
 381 }
 382
 383 static int srp_new_cm_id(struct srp_rdma_ch *ch)
 384 {
 385         struct srp_target_port *target = ch->target;
 386
 387         return target->using_rdma_cm ? srp_new_rdma_cm_id(ch) :
 388                 srp_new_ib_cm_id(ch);
 389 }
 390
 391 static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
 392 {
 393         struct srp_device *dev = target->srp_host->srp_dev;
 394         struct ib_fmr_pool_param fmr_param;
 395
 396         memset(&fmr_param, 0, sizeof(fmr_param));
 397         fmr_param.pool_size         = target->mr_pool_size;
 398         fmr_param.dirty_watermark   = fmr_param.pool_size / 4;
 399         fmr_param.cache             = 1;
 400         fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
 401         fmr_param.page_shift        = ilog2(dev->mr_page_size);
 402         fmr_param.access            = (IB_ACCESS_LOCAL_WRITE |
 403                                        IB_ACCESS_REMOTE_WRITE |
 404                                        IB_ACCESS_REMOTE_READ);
 405
 406         return ib_create_fmr_pool(dev->pd, &fmr_param);
 407 }
 408
 409 /**
 410  * srp_destroy_fr_pool() - free the resources owned by a pool
 411  * @pool: Fast registration pool to be destroyed.
 412  */
 413 static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
 414 {
 415         int i;
 416         struct srp_fr_desc *d;
 417
 418         if (!pool)
 419                 return;
 420
 421         for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
 422                 if (d->mr)
 423                         ib_dereg_mr(d->mr);
 424         }
 425         kfree(pool);
 426 }
 427
 428 /**
 429  * srp_create_fr_pool() - allocate and initialize a pool for fast registration
 430  * @device:            IB device to allocate fast registration descriptors for.
 431  * @pd:                Protection domain associated with the FR descriptors.
 432  * @pool_size:         Number of descriptors to allocate.
 433  * @max_page_list_len: Maximum fast registration work request page list length.
 434  */
 435 static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
 436                                               struct ib_pd *pd, int pool_size,
 437                                               int max_page_list_len)
 438 {
 439         struct srp_fr_pool *pool;
 440         struct srp_fr_desc *d;
 441         struct ib_mr *mr;
 442         int i, ret = -EINVAL;
 443         enum ib_mr_type mr_type;
 444
 445         if (pool_size <= 0)
 446                 goto err;
 447         ret = -ENOMEM;
 448         pool = kzalloc(struct_size(pool, desc, pool_size), GFP_KERNEL);
 449         if (!pool)
 450                 goto err;
 451         pool->size = pool_size;
 452         pool->max_page_list_len = max_page_list_len;
 453         spin_lock_init(&pool->lock);
 454         INIT_LIST_HEAD(&pool->free_list);
 455
 456         if (device->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG)
 457                 mr_type = IB_MR_TYPE_SG_GAPS;
 458         else
 459                 mr_type = IB_MR_TYPE_MEM_REG;
 460
 461         for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
 462                 mr = ib_alloc_mr(pd, mr_type, max_page_list_len);
 463                 if (IS_ERR(mr)) {
 464                         ret = PTR_ERR(mr);
 465                         if (ret == -ENOMEM)
 466                                 pr_info("%s: ib_alloc_mr() failed. Try to reduce max_cmd_per_lun, max_sect or ch_count\n",
 467                                         dev_name(&device->dev));
 468                         goto destroy_pool;
 469                 }
 470                 d->mr = mr;
 471                 list_add_tail(&d->entry, &pool->free_list);
 472         }
 473
 474 out:
 475         return pool;
 476
 477 destroy_pool:
 478         srp_destroy_fr_pool(pool);
 479
 480 err:
 481         pool = ERR_PTR(ret);
 482         goto out;
 483 }
 484
 485 /**
 486  * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
 487  * @pool: Pool to obtain descriptor from.
 488  */
 489 static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
 490 {
 491         struct srp_fr_desc *d = NULL;
 492         unsigned long flags;
 493
 494         spin_lock_irqsave(&pool->lock, flags);
 495         if (!list_empty(&pool->free_list)) {
 496                 d = list_first_entry(&pool->free_list, typeof(*d), entry);
 497                 list_del(&d->entry);
 498         }
 499         spin_unlock_irqrestore(&pool->lock, flags);
 500
 501         return d;
 502 }
 503
 504 /**
 505  * srp_fr_pool_put() - put an FR descriptor back in the free list
 506  * @pool: Pool the descriptor was allocated from.
 507  * @desc: Pointer to an array of fast registration descriptor pointers.
 508  * @n:    Number of descriptors to put back.
 509  *
 510  * Note: The caller must already have queued an invalidation request for
 511  * desc->mr->rkey before calling this function.
 512  */
 513 static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
 514                             int n)
 515 {
 516         unsigned long flags;
 517         int i;
 518
 519         spin_lock_irqsave(&pool->lock, flags);
 520         for (i = 0; i < n; i++)
 521                 list_add(&desc[i]->entry, &pool->free_list);
 522         spin_unlock_irqrestore(&pool->lock, flags);
 523 }
 524
 525 static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
 526 {
 527         struct srp_device *dev = target->srp_host->srp_dev;
 528
 529         return srp_create_fr_pool(dev->dev, dev->pd, target->mr_pool_size,
 530                                   dev->max_pages_per_mr);
 531 }
 532
 533 /**
 534  * srp_destroy_qp() - destroy an RDMA queue pair
 535  * @ch: SRP RDMA channel.
 536  *
 537  * Drain the qp before destroying it.  This avoids that the receive
 538  * completion handler can access the queue pair while it is
 539  * being destroyed.
 540  */
 541 static void srp_destroy_qp(struct srp_rdma_ch *ch)
 542 {
 543         spin_lock_irq(&ch->lock);
 544         ib_process_cq_direct(ch->send_cq, -1);
 545         spin_unlock_irq(&ch->lock);
 546
 547         ib_drain_qp(ch->qp);
 548         ib_destroy_qp(ch->qp);
 549 }
 550
 551 static int srp_create_ch_ib(struct srp_rdma_ch *ch)
 552 {
 553         struct srp_target_port *target = ch->target;
 554         struct srp_device *dev = target->srp_host->srp_dev;
 555         struct ib_qp_init_attr *init_attr;
 556         struct ib_cq *recv_cq, *send_cq;
 557         struct ib_qp *qp;
 558         struct ib_fmr_pool *fmr_pool = NULL;
 559         struct srp_fr_pool *fr_pool = NULL;
 560         const int m = 1 + dev->use_fast_reg * target->mr_per_cmd * 2;
 561         int ret;
 562
 563         init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
 564         if (!init_attr)
 565                 return -ENOMEM;
 566
 567         /* queue_size + 1 for ib_drain_rq() */
 568         recv_cq = ib_alloc_cq(dev->dev, ch, target->queue_size + 1,
 569                                 ch->comp_vector, IB_POLL_SOFTIRQ);
 570         if (IS_ERR(recv_cq)) {
 571                 ret = PTR_ERR(recv_cq);
 572                 goto err;
 573         }
 574
 575         send_cq = ib_alloc_cq(dev->dev, ch, m * target->queue_size,
 576                                 ch->comp_vector, IB_POLL_DIRECT);
 577         if (IS_ERR(send_cq)) {
 578                 ret = PTR_ERR(send_cq);
 579                 goto err_recv_cq;
 580         }
 581
 582         init_attr->event_handler       = srp_qp_event;
 583         init_attr->cap.max_send_wr     = m * target->queue_size;
 584         init_attr->cap.max_recv_wr     = target->queue_size + 1;
 585         init_attr->cap.max_recv_sge    = 1;
 586         init_attr->cap.max_send_sge    = SRP_MAX_SGE;
 587         init_attr->sq_sig_type         = IB_SIGNAL_REQ_WR;
 588         init_attr->qp_type             = IB_QPT_RC;
 589         init_attr->send_cq             = send_cq;
 590         init_attr->recv_cq             = recv_cq;
 591
 592         if (target->using_rdma_cm) {
 593                 ret = rdma_create_qp(ch->rdma_cm.cm_id, dev->pd, init_attr);
 594                 qp = ch->rdma_cm.cm_id->qp;
 595         } else {
 596                 qp = ib_create_qp(dev->pd, init_attr);
 597                 if (!IS_ERR(qp)) {
 598                         ret = srp_init_ib_qp(target, qp);
 599                         if (ret)
 600                                 ib_destroy_qp(qp);
 601                 } else {
 602                         ret = PTR_ERR(qp);
 603                 }
 604         }
 605         if (ret) {
 606                 pr_err("QP creation failed for dev %s: %d\n",
 607                        dev_name(&dev->dev->dev), ret);
 608                 goto err_send_cq;
 609         }
 610
 611         if (dev->use_fast_reg) {
 612                 fr_pool = srp_alloc_fr_pool(target);
 613                 if (IS_ERR(fr_pool)) {
 614                         ret = PTR_ERR(fr_pool);
 615                         shost_printk(KERN_WARNING, target->scsi_host, PFX
 616                                      "FR pool allocation failed (%d)\n", ret);
 617                         goto err_qp;
 618                 }
 619         } else if (dev->use_fmr) {
 620                 fmr_pool = srp_alloc_fmr_pool(target);
 621                 if (IS_ERR(fmr_pool)) {
 622                         ret = PTR_ERR(fmr_pool);
 623                         shost_printk(KERN_WARNING, target->scsi_host, PFX
 624                                      "FMR pool allocation failed (%d)\n", ret);
 625                         goto err_qp;
 626                 }
 627         }
 628
 629         if (ch->qp)
 630                 srp_destroy_qp(ch);
 631         if (ch->recv_cq)
 632                 ib_free_cq(ch->recv_cq);
 633         if (ch->send_cq)
 634                 ib_free_cq(ch->send_cq);
 635
 636         ch->qp = qp;
 637         ch->recv_cq = recv_cq;
 638         ch->send_cq = send_cq;
 639
 640         if (dev->use_fast_reg) {
 641                 if (ch->fr_pool)
 642                         srp_destroy_fr_pool(ch->fr_pool);
 643                 ch->fr_pool = fr_pool;
 644         } else if (dev->use_fmr) {
 645                 if (ch->fmr_pool)
 646                         ib_destroy_fmr_pool(ch->fmr_pool);
 647                 ch->fmr_pool = fmr_pool;
 648         }
 649
 650         kfree(init_attr);
 651         return 0;
 652
 653 err_qp:
 654         if (target->using_rdma_cm)
 655                 rdma_destroy_qp(ch->rdma_cm.cm_id);
 656         else
 657                 ib_destroy_qp(qp);
 658
 659 err_send_cq:
 660         ib_free_cq(send_cq);
 661
 662 err_recv_cq:
 663         ib_free_cq(recv_cq);
 664
 665 err:
 666         kfree(init_attr);
 667         return ret;
 668 }
 669
 670 /*
 671  * Note: this function may be called without srp_alloc_iu_bufs() having been
 672  * invoked. Hence the ch->[rt]x_ring checks.
 673  */
 674 static void srp_free_ch_ib(struct srp_target_port *target,
 675                            struct srp_rdma_ch *ch)
 676 {
 677         struct srp_device *dev = target->srp_host->srp_dev;
 678         int i;
 679
 680         if (!ch->target)
 681                 return;
 682
 683         if (target->using_rdma_cm) {
 684                 if (ch->rdma_cm.cm_id) {
 685                         rdma_destroy_id(ch->rdma_cm.cm_id);
 686                         ch->rdma_cm.cm_id = NULL;
 687                 }
 688         } else {
 689                 if (ch->ib_cm.cm_id) {
 690                         ib_destroy_cm_id(ch->ib_cm.cm_id);
 691                         ch->ib_cm.cm_id = NULL;
 692                 }
 693         }
 694
 695         /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
 696         if (!ch->qp)
 697                 return;
 698
 699         if (dev->use_fast_reg) {
 700                 if (ch->fr_pool)
 701                         srp_destroy_fr_pool(ch->fr_pool);
 702         } else if (dev->use_fmr) {
 703                 if (ch->fmr_pool)
 704                         ib_destroy_fmr_pool(ch->fmr_pool);
 705         }
 706
 707         srp_destroy_qp(ch);
 708         ib_free_cq(ch->send_cq);
 709         ib_free_cq(ch->recv_cq);
 710
 711         /*
 712          * Avoid that the SCSI error handler tries to use this channel after
 713          * it has been freed. The SCSI error handler can namely continue
 714          * trying to perform recovery actions after scsi_remove_host()
 715          * returned.
 716          */
 717         ch->target = NULL;
 718
 719         ch->qp = NULL;
 720         ch->send_cq = ch->recv_cq = NULL;
 721
 722         if (ch->rx_ring) {
 723                 for (i = 0; i < target->queue_size; ++i)
 724                         srp_free_iu(target->srp_host, ch->rx_ring[i]);
 725                 kfree(ch->rx_ring);
 726                 ch->rx_ring = NULL;
 727         }
 728         if (ch->tx_ring) {
 729                 for (i = 0; i < target->queue_size; ++i)
 730                         srp_free_iu(target->srp_host, ch->tx_ring[i]);
 731                 kfree(ch->tx_ring);
 732                 ch->tx_ring = NULL;
 733         }
 734 }
 735
 736 static void srp_path_rec_completion(int status,
 737                                     struct sa_path_rec *pathrec,
 738                                     void *ch_ptr)
 739 {
 740         struct srp_rdma_ch *ch = ch_ptr;
 741         struct srp_target_port *target = ch->target;
 742
 743         ch->status = status;
 744         if (status)
 745                 shost_printk(KERN_ERR, target->scsi_host,
 746                              PFX "Got failed path rec status %d\n", status);
 747         else
 748                 ch->ib_cm.path = *pathrec;
 749         complete(&ch->done);
 750 }
 751
 752 static int srp_ib_lookup_path(struct srp_rdma_ch *ch)
 753 {
 754         struct srp_target_port *target = ch->target;
 755         int ret;
 756
 757         ch->ib_cm.path.numb_path = 1;
 758
 759         init_completion(&ch->done);
 760
 761         ch->ib_cm.path_query_id = ib_sa_path_rec_get(&srp_sa_client,
 762                                                target->srp_host->srp_dev->dev,
 763                                                target->srp_host->port,
 764                                                &ch->ib_cm.path,
 765                                                IB_SA_PATH_REC_SERVICE_ID |
 766                                                IB_SA_PATH_REC_DGID       |
 767                                                IB_SA_PATH_REC_SGID       |
 768                                                IB_SA_PATH_REC_NUMB_PATH  |
 769                                                IB_SA_PATH_REC_PKEY,
 770                                                SRP_PATH_REC_TIMEOUT_MS,
 771                                                GFP_KERNEL,
 772                                                srp_path_rec_completion,
 773                                                ch, &ch->ib_cm.path_query);
 774         if (ch->ib_cm.path_query_id < 0)
 775                 return ch->ib_cm.path_query_id;
 776
 777         ret = wait_for_completion_interruptible(&ch->done);
 778         if (ret < 0)
 779                 return ret;
 780
 781         if (ch->status < 0)
 782                 shost_printk(KERN_WARNING, target->scsi_host,
 783                              PFX "Path record query failed: sgid %pI6, dgid %pI6, pkey %#04x, service_id %#16llx\n",
 784                              ch->ib_cm.path.sgid.raw, ch->ib_cm.path.dgid.raw,
 785                              be16_to_cpu(target->ib_cm.pkey),
 786                              be64_to_cpu(target->ib_cm.service_id));
 787
 788         return ch->status;
 789 }
 790
 791 static int srp_rdma_lookup_path(struct srp_rdma_ch *ch)
 792 {
 793         struct srp_target_port *target = ch->target;
 794         int ret;
 795
 796         init_completion(&ch->done);
 797
 798         ret = rdma_resolve_route(ch->rdma_cm.cm_id, SRP_PATH_REC_TIMEOUT_MS);
 799         if (ret)
 800                 return ret;
 801
 802         wait_for_completion_interruptible(&ch->done);
 803
 804         if (ch->status != 0)
 805                 shost_printk(KERN_WARNING, target->scsi_host,
 806                              PFX "Path resolution failed\n");
 807
 808         return ch->status;
 809 }
 810
 811 static int srp_lookup_path(struct srp_rdma_ch *ch)
 812 {
 813         struct srp_target_port *target = ch->target;
 814
 815         return target->using_rdma_cm ? srp_rdma_lookup_path(ch) :
 816                 srp_ib_lookup_path(ch);
 817 }
 818
 819 static u8 srp_get_subnet_timeout(struct srp_host *host)
 820 {
 821         struct ib_port_attr attr;
 822         int ret;
 823         u8 subnet_timeout = 18;
 824
 825         ret = ib_query_port(host->srp_dev->dev, host->port, &attr);
 826         if (ret == 0)
 827                 subnet_timeout = attr.subnet_timeout;
 828
 829         if (unlikely(subnet_timeout < 15))
 830                 pr_warn("%s: subnet timeout %d may cause SRP login to fail.\n",
 831                         dev_name(&host->srp_dev->dev->dev), subnet_timeout);
 832
 833         return subnet_timeout;
 834 }
 835
 836 static int srp_send_req(struct srp_rdma_ch *ch, uint32_t max_iu_len,
 837                         bool multich)
 838 {
 839         struct srp_target_port *target = ch->target;
 840         struct {
 841                 struct rdma_conn_param    rdma_param;
 842                 struct srp_login_req_rdma rdma_req;
 843                 struct ib_cm_req_param    ib_param;
 844                 struct srp_login_req      ib_req;
 845         } *req = NULL;
 846         char *ipi, *tpi;
 847         int status;
 848
 849         req = kzalloc(sizeof *req, GFP_KERNEL);
 850         if (!req)
 851                 return -ENOMEM;
 852
 853         req->ib_param.flow_control = 1;
 854         req->ib_param.retry_count = target->tl_retry_count;
 855
 856         /*
 857          * Pick some arbitrary defaults here; we could make these
 858          * module parameters if anyone cared about setting them.
 859          */
 860         req->ib_param.responder_resources = 4;
 861         req->ib_param.rnr_retry_count = 7;
 862         req->ib_param.max_cm_retries = 15;
 863
 864         req->ib_req.opcode = SRP_LOGIN_REQ;
 865         req->ib_req.tag = 0;
 866         req->ib_req.req_it_iu_len = cpu_to_be32(max_iu_len);
 867         req->ib_req.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
 868                                               SRP_BUF_FORMAT_INDIRECT);
 869         req->ib_req.req_flags = (multich ? SRP_MULTICHAN_MULTI :
 870                                  SRP_MULTICHAN_SINGLE);
 871         if (srp_use_imm_data) {
 872                 req->ib_req.req_flags |= SRP_IMMED_REQUESTED;
 873                 req->ib_req.imm_data_offset = cpu_to_be16(SRP_IMM_DATA_OFFSET);
 874         }
 875
 876         if (target->using_rdma_cm) {
 877                 req->rdma_param.flow_control = req->ib_param.flow_control;
 878                 req->rdma_param.responder_resources =
 879                         req->ib_param.responder_resources;
 880                 req->rdma_param.initiator_depth = req->ib_param.initiator_depth;
 881                 req->rdma_param.retry_count = req->ib_param.retry_count;
 882                 req->rdma_param.rnr_retry_count = req->ib_param.rnr_retry_count;
 883                 req->rdma_param.private_data = &req->rdma_req;
 884                 req->rdma_param.private_data_len = sizeof(req->rdma_req);
 885
 886                 req->rdma_req.opcode = req->ib_req.opcode;
 887                 req->rdma_req.tag = req->ib_req.tag;
 888                 req->rdma_req.req_it_iu_len = req->ib_req.req_it_iu_len;
 889                 req->rdma_req.req_buf_fmt = req->ib_req.req_buf_fmt;
 890                 req->rdma_req.req_flags = req->ib_req.req_flags;
 891                 req->rdma_req.imm_data_offset = req->ib_req.imm_data_offset;
 892
 893                 ipi = req->rdma_req.initiator_port_id;
 894                 tpi = req->rdma_req.target_port_id;
 895         } else {
 896                 u8 subnet_timeout;
 897
 898                 subnet_timeout = srp_get_subnet_timeout(target->srp_host);
 899
 900                 req->ib_param.primary_path = &ch->ib_cm.path;
 901                 req->ib_param.alternate_path = NULL;
 902                 req->ib_param.service_id = target->ib_cm.service_id;
 903                 get_random_bytes(&req->ib_param.starting_psn, 4);
 904                 req->ib_param.starting_psn &= 0xffffff;
 905                 req->ib_param.qp_num = ch->qp->qp_num;
 906                 req->ib_param.qp_type = ch->qp->qp_type;
 907                 req->ib_param.local_cm_response_timeout = subnet_timeout + 2;
 908                 req->ib_param.remote_cm_response_timeout = subnet_timeout + 2;
 909                 req->ib_param.private_data = &req->ib_req;
 910                 req->ib_param.private_data_len = sizeof(req->ib_req);
 911
 912                 ipi = req->ib_req.initiator_port_id;
 913                 tpi = req->ib_req.target_port_id;
 914         }
 915
 916         /*
 917          * In the published SRP specification (draft rev. 16a), the
 918          * port identifier format is 8 bytes of ID extension followed
 919          * by 8 bytes of GUID.  Older drafts put the two halves in the
 920          * opposite order, so that the GUID comes first.
 921          *
 922          * Targets conforming to these obsolete drafts can be
 923          * recognized by the I/O Class they report.
 924          */
 925         if (target->io_class == SRP_REV10_IB_IO_CLASS) {
 926                 memcpy(ipi,     &target->sgid.global.interface_id, 8);
 927                 memcpy(ipi + 8, &target->initiator_ext, 8);
 928                 memcpy(tpi,     &target->ioc_guid, 8);
 929                 memcpy(tpi + 8, &target->id_ext, 8);
 930         } else {
 931                 memcpy(ipi,     &target->initiator_ext, 8);
 932                 memcpy(ipi + 8, &target->sgid.global.interface_id, 8);
 933                 memcpy(tpi,     &target->id_ext, 8);
 934                 memcpy(tpi + 8, &target->ioc_guid, 8);
 935         }
 936
 937         /*
 938          * Topspin/Cisco SRP targets will reject our login unless we
 939          * zero out the first 8 bytes of our initiator port ID and set
 940          * the second 8 bytes to the local node GUID.
 941          */
 942         if (srp_target_is_topspin(target)) {
 943                 shost_printk(KERN_DEBUG, target->scsi_host,
 944                              PFX "Topspin/Cisco initiator port ID workaround "
 945                              "activated for target GUID %016llx\n",
 946                              be64_to_cpu(target->ioc_guid));
 947                 memset(ipi, 0, 8);
 948                 memcpy(ipi + 8, &target->srp_host->srp_dev->dev->node_guid, 8);
 949         }
 950
 951         if (target->using_rdma_cm)
 952                 status = rdma_connect(ch->rdma_cm.cm_id, &req->rdma_param);
 953         else
 954                 status = ib_send_cm_req(ch->ib_cm.cm_id, &req->ib_param);
 955
 956         kfree(req);
 957
 958         return status;
 959 }
 960
 961 static bool srp_queue_remove_work(struct srp_target_port *target)
 962 {
 963         bool changed = false;
 964
 965         spin_lock_irq(&target->lock);
 966         if (target->state != SRP_TARGET_REMOVED) {
 967                 target->state = SRP_TARGET_REMOVED;
 968                 changed = true;
 969         }
 970         spin_unlock_irq(&target->lock);
 971
 972         if (changed)
 973                 queue_work(srp_remove_wq, &target->remove_work);
 974
 975         return changed;
 976 }
 977
 978 static void srp_disconnect_target(struct srp_target_port *target)
 979 {
 980         struct srp_rdma_ch *ch;
 981         int i, ret;
 982
 983         /* XXX should send SRP_I_LOGOUT request */
 984
 985         for (i = 0; i < target->ch_count; i++) {
 986                 ch = &target->ch[i];
 987                 ch->connected = false;
 988                 ret = 0;
 989                 if (target->using_rdma_cm) {
 990                         if (ch->rdma_cm.cm_id)
 991                                 rdma_disconnect(ch->rdma_cm.cm_id);
 992                 } else {
 993                         if (ch->ib_cm.cm_id)
 994                                 ret = ib_send_cm_dreq(ch->ib_cm.cm_id,
 995                                                       NULL, 0);
 996                 }
 997                 if (ret < 0) {
 998                         shost_printk(KERN_DEBUG, target->scsi_host,
 999                                      PFX "Sending CM DREQ failed\n");
1000                 }
1001         }
1002 }
1003
1004 static void srp_free_req_data(struct srp_target_port *target,
1005                               struct srp_rdma_ch *ch)
1006 {
1007         struct srp_device *dev = target->srp_host->srp_dev;
1008         struct ib_device *ibdev = dev->dev;
1009         struct srp_request *req;
1010         int i;
1011
1012         if (!ch->req_ring)
1013                 return;
1014
1015         for (i = 0; i < target->req_ring_size; ++i) {
1016                 req = &ch->req_ring[i];
1017                 if (dev->use_fast_reg) {
1018                         kfree(req->fr_list);
1019                 } else {
1020                         kfree(req->fmr_list);
1021                         kfree(req->map_page);
1022                 }
1023                 if (req->indirect_dma_addr) {
1024                         ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
1025                                             target->indirect_size,
1026                                             DMA_TO_DEVICE);
1027                 }
1028                 kfree(req->indirect_desc);
1029         }
1030
1031         kfree(ch->req_ring);
1032         ch->req_ring = NULL;
1033 }
1034
1035 static int srp_alloc_req_data(struct srp_rdma_ch *ch)
1036 {
1037         struct srp_target_port *target = ch->target;
1038         struct srp_device *srp_dev = target->srp_host->srp_dev;
1039         struct ib_device *ibdev = srp_dev->dev;
1040         struct srp_request *req;
1041         void *mr_list;
1042         dma_addr_t dma_addr;
1043         int i, ret = -ENOMEM;
1044
1045         ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
1046                                GFP_KERNEL);
1047         if (!ch->req_ring)
1048                 goto out;
1049
1050         for (i = 0; i < target->req_ring_size; ++i) {
1051                 req = &ch->req_ring[i];
1052                 mr_list = kmalloc_array(target->mr_per_cmd, sizeof(void *),
1053                                         GFP_KERNEL);
1054                 if (!mr_list)
1055                         goto out;
1056                 if (srp_dev->use_fast_reg) {
1057                         req->fr_list = mr_list;
1058                 } else {
1059                         req->fmr_list = mr_list;
1060                         req->map_page = kmalloc_array(srp_dev->max_pages_per_mr,
1061                                                       sizeof(void *),
1062                                                       GFP_KERNEL);
1063                         if (!req->map_page)
1064                                 goto out;
1065                 }
1066                 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
1067                 if (!req->indirect_desc)
1068                         goto out;
1069
1070                 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
1071                                              target->indirect_size,
1072                                              DMA_TO_DEVICE);
1073                 if (ib_dma_mapping_error(ibdev, dma_addr))
1074                         goto out;
1075
1076                 req->indirect_dma_addr = dma_addr;
1077         }
1078         ret = 0;
1079
1080 out:
1081         return ret;
1082 }
1083
1084 /**
1085  * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
1086  * @shost: SCSI host whose attributes to remove from sysfs.
1087  *
1088  * Note: Any attributes defined in the host template and that did not exist
1089  * before invocation of this function will be ignored.
1090  */
1091 static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
1092 {
1093         struct device_attribute **attr;
1094
1095         for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
1096                 device_remove_file(&shost->shost_dev, *attr);
1097 }
1098
1099 static void srp_remove_target(struct srp_target_port *target)
1100 {
1101         struct srp_rdma_ch *ch;
1102         int i;
1103
1104         WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
1105
1106         srp_del_scsi_host_attr(target->scsi_host);
1107         srp_rport_get(target->rport);
1108         srp_remove_host(target->scsi_host);
1109         scsi_remove_host(target->scsi_host);
1110         srp_stop_rport_timers(target->rport);
1111         srp_disconnect_target(target);
1112         kobj_ns_drop(KOBJ_NS_TYPE_NET, target->net);
1113         for (i = 0; i < target->ch_count; i++) {
1114                 ch = &target->ch[i];
1115                 srp_free_ch_ib(target, ch);
1116         }
1117         cancel_work_sync(&target->tl_err_work);
1118         srp_rport_put(target->rport);
1119         for (i = 0; i < target->ch_count; i++) {
1120                 ch = &target->ch[i];
1121                 srp_free_req_data(target, ch);
1122         }
1123         kfree(target->ch);
1124         target->ch = NULL;
1125
1126         spin_lock(&target->srp_host->target_lock);
1127         list_del(&target->list);
1128         spin_unlock(&target->srp_host->target_lock);
1129
1130         scsi_host_put(target->scsi_host);
1131 }
1132
1133 static void srp_remove_work(struct work_struct *work)
1134 {
1135         struct srp_target_port *target =
1136                 container_of(work, struct srp_target_port, remove_work);
1137
1138         WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
1139
1140         srp_remove_target(target);
1141 }
1142
1143 static void srp_rport_delete(struct srp_rport *rport)
1144 {
1145         struct srp_target_port *target = rport->lld_data;
1146
1147         srp_queue_remove_work(target);
1148 }
1149
1150 /**
1151  * srp_connected_ch() - number of connected channels
1152  * @target: SRP target port.
1153  */
1154 static int srp_connected_ch(struct srp_target_port *target)
1155 {
1156         int i, c = 0;
1157
1158         for (i = 0; i < target->ch_count; i++)
1159                 c += target->ch[i].connected;
1160
1161         return c;
1162 }
1163
1164 static int srp_connect_ch(struct srp_rdma_ch *ch, uint32_t max_iu_len,
1165                           bool multich)
1166 {
1167         struct srp_target_port *target = ch->target;
1168         int ret;
1169
1170         WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0);
1171
1172         ret = srp_lookup_path(ch);
1173         if (ret)
1174                 goto out;
1175
1176         while (1) {
1177                 init_completion(&ch->done);
1178                 ret = srp_send_req(ch, max_iu_len, multich);
1179                 if (ret)
1180                         goto out;
1181                 ret = wait_for_completion_interruptible(&ch->done);
1182                 if (ret < 0)
1183                         goto out;
1184
1185                 /*
1186                  * The CM event handling code will set status to
1187                  * SRP_PORT_REDIRECT if we get a port redirect REJ
1188                  * back, or SRP_DLID_REDIRECT if we get a lid/qp
1189                  * redirect REJ back.
1190                  */
1191                 ret = ch->status;
1192                 switch (ret) {
1193                 case 0:
1194                         ch->connected = true;
1195                         goto out;
1196
1197                 case SRP_PORT_REDIRECT:
1198                         ret = srp_lookup_path(ch);
1199                         if (ret)
1200                                 goto out;
1201                         break;
1202
1203                 case SRP_DLID_REDIRECT:
1204                         break;
1205
1206                 case SRP_STALE_CONN:
1207                         shost_printk(KERN_ERR, target->scsi_host, PFX
1208                                      "giving up on stale connection\n");
1209                         ret = -ECONNRESET;
1210                         goto out;
1211
1212                 default:
1213                         goto out;
1214                 }
1215         }
1216
1217 out:
1218         return ret <= 0 ? ret : -ENODEV;
1219 }
1220
1221 static void srp_inv_rkey_err_done(struct ib_cq *cq, struct ib_wc *wc)
1222 {
1223         srp_handle_qp_err(cq, wc, "INV RKEY");
1224 }
1225
1226 static int srp_inv_rkey(struct srp_request *req, struct srp_rdma_ch *ch,
1227                 u32 rkey)
1228 {
1229         struct ib_send_wr wr = {
1230                 .opcode             = IB_WR_LOCAL_INV,
1231                 .next               = NULL,
1232                 .num_sge            = 0,
1233                 .send_flags         = 0,
1234                 .ex.invalidate_rkey = rkey,
1235         };
1236
1237         wr.wr_cqe = &req->reg_cqe;
1238         req->reg_cqe.done = srp_inv_rkey_err_done;
1239         return ib_post_send(ch->qp, &wr, NULL);
1240 }
1241
1242 static void srp_unmap_data(struct scsi_cmnd *scmnd,
1243                            struct srp_rdma_ch *ch,
1244                            struct srp_request *req)
1245 {
1246         struct srp_target_port *target = ch->target;
1247         struct srp_device *dev = target->srp_host->srp_dev;
1248         struct ib_device *ibdev = dev->dev;
1249         int i, res;
1250
1251         if (!scsi_sglist(scmnd) ||
1252             (scmnd->sc_data_direction != DMA_TO_DEVICE &&
1253              scmnd->sc_data_direction != DMA_FROM_DEVICE))
1254                 return;
1255
1256         if (dev->use_fast_reg) {
1257                 struct srp_fr_desc **pfr;
1258
1259                 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
1260                         res = srp_inv_rkey(req, ch, (*pfr)->mr->rkey);
1261                         if (res < 0) {
1262                                 shost_printk(KERN_ERR, target->scsi_host, PFX
1263                                   "Queueing INV WR for rkey %#x failed (%d)\n",
1264                                   (*pfr)->mr->rkey, res);
1265                                 queue_work(system_long_wq,
1266                                            &target->tl_err_work);
1267                         }
1268                 }
1269                 if (req->nmdesc)
1270                         srp_fr_pool_put(ch->fr_pool, req->fr_list,
1271                                         req->nmdesc);
1272         } else if (dev->use_fmr) {
1273                 struct ib_pool_fmr **pfmr;
1274
1275                 for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
1276                         ib_fmr_pool_unmap(*pfmr);
1277         }
1278
1279         ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
1280                         scmnd->sc_data_direction);
1281 }
1282
1283 /**
1284  * srp_claim_req - Take ownership of the scmnd associated with a request.
1285  * @ch: SRP RDMA channel.
1286  * @req: SRP request.
1287  * @sdev: If not NULL, only take ownership for this SCSI device.
1288  * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
1289  *         ownership of @req->scmnd if it equals @scmnd.
1290  *
1291  * Return value:
1292  * Either NULL or a pointer to the SCSI command the caller became owner of.
1293  */
1294 static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
1295                                        struct srp_request *req,
1296                                        struct scsi_device *sdev,
1297                                        struct scsi_cmnd *scmnd)
1298 {
1299         unsigned long flags;
1300
1301         spin_lock_irqsave(&ch->lock, flags);
1302         if (req->scmnd &&
1303             (!sdev || req->scmnd->device == sdev) &&
1304             (!scmnd || req->scmnd == scmnd)) {
1305                 scmnd = req->scmnd;
1306                 req->scmnd = NULL;
1307         } else {
1308                 scmnd = NULL;
1309         }
1310         spin_unlock_irqrestore(&ch->lock, flags);
1311
1312         return scmnd;
1313 }
1314
1315 /**
1316  * srp_free_req() - Unmap data and adjust ch->req_lim.
1317  * @ch:     SRP RDMA channel.
1318  * @req:    Request to be freed.
1319  * @scmnd:  SCSI command associated with @req.
1320  * @req_lim_delta: Amount to be added to @target->req_lim.
1321  */
1322 static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
1323                          struct scsi_cmnd *scmnd, s32 req_lim_delta)
1324 {
1325         unsigned long flags;
1326
1327         srp_unmap_data(scmnd, ch, req);
1328
1329         spin_lock_irqsave(&ch->lock, flags);
1330         ch->req_lim += req_lim_delta;
1331         spin_unlock_irqrestore(&ch->lock, flags);
1332 }
1333
1334 static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
1335                            struct scsi_device *sdev, int result)
1336 {
1337         struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
1338
1339         if (scmnd) {
1340                 srp_free_req(ch, req, scmnd, 0);
1341                 scmnd->result = result;
1342                 scmnd->scsi_done(scmnd);
1343         }
1344 }
1345
1346 static void srp_terminate_io(struct srp_rport *rport)
1347 {
1348         struct srp_target_port *target = rport->lld_data;
1349         struct srp_rdma_ch *ch;
1350         int i, j;
1351
1352         for (i = 0; i < target->ch_count; i++) {
1353                 ch = &target->ch[i];
1354
1355                 for (j = 0; j < target->req_ring_size; ++j) {
1356                         struct srp_request *req = &ch->req_ring[j];
1357
1358                         srp_finish_req(ch, req, NULL,
1359                                        DID_TRANSPORT_FAILFAST << 16);
1360                 }
1361         }
1362 }
1363
1364 /* Calculate maximum initiator to target information unit length. */
1365 static uint32_t srp_max_it_iu_len(int cmd_sg_cnt, bool use_imm_data)
1366 {
1367         uint32_t max_iu_len = sizeof(struct srp_cmd) + SRP_MAX_ADD_CDB_LEN +
1368                 sizeof(struct srp_indirect_buf) +
1369                 cmd_sg_cnt * sizeof(struct srp_direct_buf);
1370
1371         if (use_imm_data)
1372                 max_iu_len = max(max_iu_len, SRP_IMM_DATA_OFFSET +
1373                                  srp_max_imm_data);
1374
1375         return max_iu_len;
1376 }
1377
1378 /*
1379  * It is up to the caller to ensure that srp_rport_reconnect() calls are
1380  * serialized and that no concurrent srp_queuecommand(), srp_abort(),
1381  * srp_reset_device() or srp_reset_host() calls will occur while this function
1382  * is in progress. One way to realize that is not to call this function
1383  * directly but to call srp_reconnect_rport() instead since that last function
1384  * serializes calls of this function via rport->mutex and also blocks
1385  * srp_queuecommand() calls before invoking this function.
1386  */
1387 static int srp_rport_reconnect(struct srp_rport *rport)
1388 {
1389         struct srp_target_port *target = rport->lld_data;
1390         struct srp_rdma_ch *ch;
1391         uint32_t max_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt,
1392                                                 srp_use_imm_data);
1393         int i, j, ret = 0;
1394         bool multich = false;
1395
1396         srp_disconnect_target(target);
1397
1398         if (target->state == SRP_TARGET_SCANNING)
1399                 return -ENODEV;
1400
1401         /*
1402          * Now get a new local CM ID so that we avoid confusing the target in
1403          * case things are really fouled up. Doing so also ensures that all CM
1404          * callbacks will have finished before a new QP is allocated.
1405          */
1406         for (i = 0; i < target->ch_count; i++) {
1407                 ch = &target->ch[i];
1408                 ret += srp_new_cm_id(ch);
1409         }
1410         for (i = 0; i < target->ch_count; i++) {
1411                 ch = &target->ch[i];
1412                 for (j = 0; j < target->req_ring_size; ++j) {
1413                         struct srp_request *req = &ch->req_ring[j];
1414
1415                         srp_finish_req(ch, req, NULL, DID_RESET << 16);
1416                 }
1417         }
1418         for (i = 0; i < target->ch_count; i++) {
1419                 ch = &target->ch[i];
1420                 /*
1421                  * Whether or not creating a new CM ID succeeded, create a new
1422                  * QP. This guarantees that all completion callback function
1423                  * invocations have finished before request resetting starts.
1424                  */
1425                 ret += srp_create_ch_ib(ch);
1426
1427                 INIT_LIST_HEAD(&ch->free_tx);
1428                 for (j = 0; j < target->queue_size; ++j)
1429                         list_add(&ch->tx_ring[j]->list, &ch->free_tx);
1430         }
1431
1432         target->qp_in_error = false;
1433
1434         for (i = 0; i < target->ch_count; i++) {
1435                 ch = &target->ch[i];
1436                 if (ret)
1437                         break;
1438                 ret = srp_connect_ch(ch, max_iu_len, multich);
1439                 multich = true;
1440         }
1441
1442         if (ret == 0)
1443                 shost_printk(KERN_INFO, target->scsi_host,
1444                              PFX "reconnect succeeded\n");
1445
1446         return ret;
1447 }
1448
1449 static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
1450                          unsigned int dma_len, u32 rkey)
1451 {
1452         struct srp_direct_buf *desc = state->desc;
1453
1454         WARN_ON_ONCE(!dma_len);
1455
1456         desc->va = cpu_to_be64(dma_addr);
1457         desc->key = cpu_to_be32(rkey);
1458         desc->len = cpu_to_be32(dma_len);
1459
1460         state->total_len += dma_len;
1461         state->desc++;
1462         state->ndesc++;
1463 }
1464
1465 static int srp_map_finish_fmr(struct srp_map_state *state,
1466                               struct srp_rdma_ch *ch)
1467 {
1468         struct srp_target_port *target = ch->target;
1469         struct srp_device *dev = target->srp_host->srp_dev;
1470         struct ib_pool_fmr *fmr;
1471         u64 io_addr = 0;
1472
1473         if (state->fmr.next >= state->fmr.end) {
1474                 shost_printk(KERN_ERR, ch->target->scsi_host,
1475                              PFX "Out of MRs (mr_per_cmd = %d)\n",
1476                              ch->target->mr_per_cmd);
1477                 return -ENOMEM;
1478         }
1479
1480         WARN_ON_ONCE(!dev->use_fmr);
1481
1482         if (state->npages == 0)
1483                 return 0;
1484
1485         if (state->npages == 1 && target->global_rkey) {
1486                 srp_map_desc(state, state->base_dma_addr, state->dma_len,
1487                              target->global_rkey);
1488                 goto reset_state;
1489         }
1490
1491         fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
1492                                    state->npages, io_addr);
1493         if (IS_ERR(fmr))
1494                 return PTR_ERR(fmr);
1495
1496         *state->fmr.next++ = fmr;
1497         state->nmdesc++;
1498
1499         srp_map_desc(state, state->base_dma_addr & ~dev->mr_page_mask,
1500                      state->dma_len, fmr->fmr->rkey);
1501
1502 reset_state:
1503         state->npages = 0;
1504         state->dma_len = 0;
1505
1506         return 0;
1507 }
1508
1509 static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc)
1510 {
1511         srp_handle_qp_err(cq, wc, "FAST REG");
1512 }
1513
1514 /*
1515  * Map up to sg_nents elements of state->sg where *sg_offset_p is the offset
1516  * where to start in the first element. If sg_offset_p != NULL then
1517  * *sg_offset_p is updated to the offset in state->sg[retval] of the first
1518  * byte that has not yet been mapped.
1519  */
1520 static int srp_map_finish_fr(struct srp_map_state *state,
1521                              struct srp_request *req,
1522                              struct srp_rdma_ch *ch, int sg_nents,
1523                              unsigned int *sg_offset_p)
1524 {
1525         struct srp_target_port *target = ch->target;
1526         struct srp_device *dev = target->srp_host->srp_dev;
1527         struct ib_reg_wr wr;
1528         struct srp_fr_desc *desc;
1529         u32 rkey;
1530         int n, err;
1531
1532         if (state->fr.next >= state->fr.end) {
1533                 shost_printk(KERN_ERR, ch->target->scsi_host,
1534                              PFX "Out of MRs (mr_per_cmd = %d)\n",
1535                              ch->target->mr_per_cmd);
1536                 return -ENOMEM;
1537         }
1538
1539         WARN_ON_ONCE(!dev->use_fast_reg);
1540
1541         if (sg_nents == 1 && target->global_rkey) {
1542                 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
1543
1544                 srp_map_desc(state, sg_dma_address(state->sg) + sg_offset,
1545                              sg_dma_len(state->sg) - sg_offset,
1546                              target->global_rkey);
1547                 if (sg_offset_p)
1548                         *sg_offset_p = 0;
1549                 return 1;
1550         }
1551
1552         desc = srp_fr_pool_get(ch->fr_pool);
1553         if (!desc)
1554                 return -ENOMEM;
1555
1556         rkey = ib_inc_rkey(desc->mr->rkey);
1557         ib_update_fast_reg_key(desc->mr, rkey);
1558
1559         n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, sg_offset_p,
1560                          dev->mr_page_size);
1561         if (unlikely(n < 0)) {
1562                 srp_fr_pool_put(ch->fr_pool, &desc, 1);
1563                 pr_debug("%s: ib_map_mr_sg(%d, %d) returned %d.\n",
1564                          dev_name(&req->scmnd->device->sdev_gendev), sg_nents,
1565                          sg_offset_p ? *sg_offset_p : -1, n);
1566                 return n;
1567         }
1568
1569         WARN_ON_ONCE(desc->mr->length == 0);
1570
1571         req->reg_cqe.done = srp_reg_mr_err_done;
1572
1573         wr.wr.next = NULL;
1574         wr.wr.opcode = IB_WR_REG_MR;
1575         wr.wr.wr_cqe = &req->reg_cqe;
1576         wr.wr.num_sge = 0;
1577         wr.wr.send_flags = 0;
1578         wr.mr = desc->mr;
1579         wr.key = desc->mr->rkey;
1580         wr.access = (IB_ACCESS_LOCAL_WRITE |
1581                      IB_ACCESS_REMOTE_READ |
1582                      IB_ACCESS_REMOTE_WRITE);
1583
1584         *state->fr.next++ = desc;
1585         state->nmdesc++;
1586
1587         srp_map_desc(state, desc->mr->iova,
1588                      desc->mr->length, desc->mr->rkey);
1589
1590         err = ib_post_send(ch->qp, &wr.wr, NULL);
1591         if (unlikely(err)) {
1592                 WARN_ON_ONCE(err == -ENOMEM);
1593                 return err;
1594         }
1595
1596         return n;
1597 }
1598
1599 static int srp_map_sg_entry(struct srp_map_state *state,
1600                             struct srp_rdma_ch *ch,
1601                             struct scatterlist *sg)
1602 {
1603         struct srp_target_port *target = ch->target;
1604         struct srp_device *dev = target->srp_host->srp_dev;
1605         dma_addr_t dma_addr = sg_dma_address(sg);
1606         unsigned int dma_len = sg_dma_len(sg);
1607         unsigned int len = 0;
1608         int ret;
1609
1610         WARN_ON_ONCE(!dma_len);
1611
1612         while (dma_len) {
1613                 unsigned offset = dma_addr & ~dev->mr_page_mask;
1614
1615                 if (state->npages == dev->max_pages_per_mr ||
1616                     (state->npages > 0 && offset != 0)) {
1617                         ret = srp_map_finish_fmr(state, ch);
1618                         if (ret)
1619                                 return ret;
1620                 }
1621
1622                 len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
1623
1624                 if (!state->npages)
1625                         state->base_dma_addr = dma_addr;
1626                 state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
1627                 state->dma_len += len;
1628                 dma_addr += len;
1629                 dma_len -= len;
1630         }
1631
1632         /*
1633          * If the end of the MR is not on a page boundary then we need to
1634          * close it out and start a new one -- we can only merge at page
1635          * boundaries.
1636          */
1637         ret = 0;
1638         if ((dma_addr & ~dev->mr_page_mask) != 0)
1639                 ret = srp_map_finish_fmr(state, ch);
1640         return ret;
1641 }
1642
1643 static int srp_map_sg_fmr(struct srp_map_state *state, struct srp_rdma_ch *ch,
1644                           struct srp_request *req, struct scatterlist *scat,
1645                           int count)
1646 {
1647         struct scatterlist *sg;
1648         int i, ret;
1649
1650         state->pages = req->map_page;
1651         state->fmr.next = req->fmr_list;
1652         state->fmr.end = req->fmr_list + ch->target->mr_per_cmd;
1653
1654         for_each_sg(scat, sg, count, i) {
1655                 ret = srp_map_sg_entry(state, ch, sg);
1656                 if (ret)
1657                         return ret;
1658         }
1659
1660         ret = srp_map_finish_fmr(state, ch);
1661         if (ret)
1662                 return ret;
1663
1664         return 0;
1665 }
1666
1667 static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch,
1668                          struct srp_request *req, struct scatterlist *scat,
1669                          int count)
1670 {
1671         unsigned int sg_offset = 0;
1672
1673         state->fr.next = req->fr_list;
1674         state->fr.end = req->fr_list + ch->target->mr_per_cmd;
1675         state->sg = scat;
1676
1677         if (count == 0)
1678                 return 0;
1679
1680         while (count) {
1681                 int i, n;
1682
1683                 n = srp_map_finish_fr(state, req, ch, count, &sg_offset);
1684                 if (unlikely(n < 0))
1685                         return n;
1686
1687                 count -= n;
1688                 for (i = 0; i < n; i++)
1689                         state->sg = sg_next(state->sg);
1690         }
1691
1692         return 0;
1693 }
1694
1695 static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch,
1696                           struct srp_request *req, struct scatterlist *scat,
1697                           int count)
1698 {
1699         struct srp_target_port *target = ch->target;
1700         struct scatterlist *sg;
1701         int i;
1702
1703         for_each_sg(scat, sg, count, i) {
1704                 srp_map_desc(state, sg_dma_address(sg), sg_dma_len(sg),
1705                              target->global_rkey);
1706         }
1707
1708         return 0;
1709 }
1710
1711 /*
1712  * Register the indirect data buffer descriptor with the HCA.
1713  *
1714  * Note: since the indirect data buffer descriptor has been allocated with
1715  * kmalloc() it is guaranteed that this buffer is a physically contiguous
1716  * memory buffer.
1717  */
1718 static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
1719                        void **next_mr, void **end_mr, u32 idb_len,
1720                        __be32 *idb_rkey)
1721 {
1722         struct srp_target_port *target = ch->target;
1723         struct srp_device *dev = target->srp_host->srp_dev;
1724         struct srp_map_state state;
1725         struct srp_direct_buf idb_desc;
1726         u64 idb_pages[1];
1727         struct scatterlist idb_sg[1];
1728         int ret;
1729
1730         memset(&state, 0, sizeof(state));
1731         memset(&idb_desc, 0, sizeof(idb_desc));
1732         state.gen.next = next_mr;
1733         state.gen.end = end_mr;
1734         state.desc = &idb_desc;
1735         state.base_dma_addr = req->indirect_dma_addr;
1736         state.dma_len = idb_len;
1737
1738         if (dev->use_fast_reg) {
1739                 state.sg = idb_sg;
1740                 sg_init_one(idb_sg, req->indirect_desc, idb_len);
1741                 idb_sg->dma_address = req->indirect_dma_addr; /* hack! */
1742 #ifdef CONFIG_NEED_SG_DMA_LENGTH
1743                 idb_sg->dma_length = idb_sg->length;          /* hack^2 */
1744 #endif
1745                 ret = srp_map_finish_fr(&state, req, ch, 1, NULL);
1746                 if (ret < 0)
1747                         return ret;
1748                 WARN_ON_ONCE(ret < 1);
1749         } else if (dev->use_fmr) {
1750                 state.pages = idb_pages;
1751                 state.pages[0] = (req->indirect_dma_addr &
1752                                   dev->mr_page_mask);
1753                 state.npages = 1;
1754                 ret = srp_map_finish_fmr(&state, ch);
1755                 if (ret < 0)
1756                         return ret;
1757         } else {
1758                 return -EINVAL;
1759         }
1760
1761         *idb_rkey = idb_desc.key;
1762
1763         return 0;
1764 }
1765
1766 static void srp_check_mapping(struct srp_map_state *state,
1767                               struct srp_rdma_ch *ch, struct srp_request *req,
1768                               struct scatterlist *scat, int count)
1769 {
1770         struct srp_device *dev = ch->target->srp_host->srp_dev;
1771         struct srp_fr_desc **pfr;
1772         u64 desc_len = 0, mr_len = 0;
1773         int i;
1774
1775         for (i = 0; i < state->ndesc; i++)
1776                 desc_len += be32_to_cpu(req->indirect_desc[i].len);
1777         if (dev->use_fast_reg)
1778                 for (i = 0, pfr = req->fr_list; i < state->nmdesc; i++, pfr++)
1779                         mr_len += (*pfr)->mr->length;
1780         else if (dev->use_fmr)
1781                 for (i = 0; i < state->nmdesc; i++)
1782                         mr_len += be32_to_cpu(req->indirect_desc[i].len);
1783         if (desc_len != scsi_bufflen(req->scmnd) ||
1784             mr_len > scsi_bufflen(req->scmnd))
1785                 pr_err("Inconsistent: scsi len %d <> desc len %lld <> mr len %lld; ndesc %d; nmdesc = %d\n",
1786                        scsi_bufflen(req->scmnd), desc_len, mr_len,
1787                        state->ndesc, state->nmdesc);
1788 }
1789
1790 /**
1791  * srp_map_data() - map SCSI data buffer onto an SRP request
1792  * @scmnd: SCSI command to map
1793  * @ch: SRP RDMA channel
1794  * @req: SRP request
1795  *
1796  * Returns the length in bytes of the SRP_CMD IU or a negative value if
1797  * mapping failed. The size of any immediate data is not included in the
1798  * return value.
1799  */
1800 static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
1801                         struct srp_request *req)
1802 {
1803         struct srp_target_port *target = ch->target;
1804         struct scatterlist *scat, *sg;
1805         struct srp_cmd *cmd = req->cmd->buf;
1806         int i, len, nents, count, ret;
1807         struct srp_device *dev;
1808         struct ib_device *ibdev;
1809         struct srp_map_state state;
1810         struct srp_indirect_buf *indirect_hdr;
1811         u64 data_len;
1812         u32 idb_len, table_len;
1813         __be32 idb_rkey;
1814         u8 fmt;
1815
1816         req->cmd->num_sge = 1;
1817
1818         if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
1819                 return sizeof(struct srp_cmd) + cmd->add_cdb_len;
1820
1821         if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
1822             scmnd->sc_data_direction != DMA_TO_DEVICE) {
1823                 shost_printk(KERN_WARNING, target->scsi_host,
1824                              PFX "Unhandled data direction %d\n",
1825                              scmnd->sc_data_direction);
1826                 return -EINVAL;
1827         }
1828
1829         nents = scsi_sg_count(scmnd);
1830         scat  = scsi_sglist(scmnd);
1831         data_len = scsi_bufflen(scmnd);
1832
1833         dev = target->srp_host->srp_dev;
1834         ibdev = dev->dev;
1835
1836         count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
1837         if (unlikely(count == 0))
1838                 return -EIO;
1839
1840         if (ch->use_imm_data &&
1841             count <= SRP_MAX_IMM_SGE &&
1842             SRP_IMM_DATA_OFFSET + data_len <= ch->max_it_iu_len &&
1843             scmnd->sc_data_direction == DMA_TO_DEVICE) {
1844                 struct srp_imm_buf *buf;
1845                 struct ib_sge *sge = &req->cmd->sge[1];
1846
1847                 fmt = SRP_DATA_DESC_IMM;
1848                 len = SRP_IMM_DATA_OFFSET;
1849                 req->nmdesc = 0;
1850                 buf = (void *)cmd->add_data + cmd->add_cdb_len;
1851                 buf->len = cpu_to_be32(data_len);
1852                 WARN_ON_ONCE((void *)(buf + 1) > (void *)cmd + len);
1853                 for_each_sg(scat, sg, count, i) {
1854                         sge[i].addr   = sg_dma_address(sg);
1855                         sge[i].length = sg_dma_len(sg);
1856                         sge[i].lkey   = target->lkey;
1857                 }
1858                 req->cmd->num_sge += count;
1859                 goto map_complete;
1860         }
1861
1862         fmt = SRP_DATA_DESC_DIRECT;
1863         len = sizeof(struct srp_cmd) + cmd->add_cdb_len +
1864                 sizeof(struct srp_direct_buf);
1865
1866         if (count == 1 && target->global_rkey) {
1867                 /*
1868                  * The midlayer only generated a single gather/scatter
1869                  * entry, or DMA mapping coalesced everything to a
1870                  * single entry.  So a direct descriptor along with
1871                  * the DMA MR suffices.
1872                  */
1873                 struct srp_direct_buf *buf;
1874
1875                 buf = (void *)cmd->add_data + cmd->add_cdb_len;
1876                 buf->va  = cpu_to_be64(sg_dma_address(scat));
1877                 buf->key = cpu_to_be32(target->global_rkey);
1878                 buf->len = cpu_to_be32(sg_dma_len(scat));
1879
1880                 req->nmdesc = 0;
1881                 goto map_complete;
1882         }
1883
1884         /*
1885          * We have more than one scatter/gather entry, so build our indirect
1886          * descriptor table, trying to merge as many entries as we can.
1887          */
1888         indirect_hdr = (void *)cmd->add_data + cmd->add_cdb_len;
1889
1890         ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
1891                                    target->indirect_size, DMA_TO_DEVICE);
1892
1893         memset(&state, 0, sizeof(state));
1894         state.desc = req->indirect_desc;
1895         if (dev->use_fast_reg)
1896                 ret = srp_map_sg_fr(&state, ch, req, scat, count);
1897         else if (dev->use_fmr)
1898                 ret = srp_map_sg_fmr(&state, ch, req, scat, count);
1899         else
1900                 ret = srp_map_sg_dma(&state, ch, req, scat, count);
1901         req->nmdesc = state.nmdesc;
1902         if (ret < 0)
1903                 goto unmap;
1904
1905         {
1906                 DEFINE_DYNAMIC_DEBUG_METADATA(ddm,
1907                         "Memory mapping consistency check");
1908                 if (DYNAMIC_DEBUG_BRANCH(ddm))
1909                         srp_check_mapping(&state, ch, req, scat, count);
1910         }
1911
1912         /* We've mapped the request, now pull as much of the indirect
1913          * descriptor table as we can into the command buffer. If this
1914          * target is not using an external indirect table, we are
1915          * guaranteed to fit into the command, as the SCSI layer won't
1916          * give us more S/G entries than we allow.
1917          */
1918         if (state.ndesc == 1) {
1919                 /*
1920                  * Memory registration collapsed the sg-list into one entry,
1921                  * so use a direct descriptor.
1922                  */
1923                 struct srp_direct_buf *buf;
1924
1925                 buf = (void *)cmd->add_data + cmd->add_cdb_len;
1926                 *buf = req->indirect_desc[0];
1927                 goto map_complete;
1928         }
1929
1930         if (unlikely(target->cmd_sg_cnt < state.ndesc &&
1931                                                 !target->allow_ext_sg)) {
1932                 shost_printk(KERN_ERR, target->scsi_host,
1933                              "Could not fit S/G list into SRP_CMD\n");
1934                 ret = -EIO;
1935                 goto unmap;
1936         }
1937
1938         count = min(state.ndesc, target->cmd_sg_cnt);
1939         table_len = state.ndesc * sizeof (struct srp_direct_buf);
1940         idb_len = sizeof(struct srp_indirect_buf) + table_len;
1941
1942         fmt = SRP_DATA_DESC_INDIRECT;
1943         len = sizeof(struct srp_cmd) + cmd->add_cdb_len +
1944                 sizeof(struct srp_indirect_buf);
1945         len += count * sizeof (struct srp_direct_buf);
1946
1947         memcpy(indirect_hdr->desc_list, req->indirect_desc,
1948                count * sizeof (struct srp_direct_buf));
1949
1950         if (!target->global_rkey) {
1951                 ret = srp_map_idb(ch, req, state.gen.next, state.gen.end,
1952                                   idb_len, &idb_rkey);
1953                 if (ret < 0)
1954                         goto unmap;
1955                 req->nmdesc++;
1956         } else {
1957                 idb_rkey = cpu_to_be32(target->global_rkey);
1958         }
1959
1960         indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
1961         indirect_hdr->table_desc.key = idb_rkey;
1962         indirect_hdr->table_desc.len = cpu_to_be32(table_len);
1963         indirect_hdr->len = cpu_to_be32(state.total_len);
1964
1965         if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1966                 cmd->data_out_desc_cnt = count;
1967         else
1968                 cmd->data_in_desc_cnt = count;
1969
1970         ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
1971                                       DMA_TO_DEVICE);
1972
1973 map_complete:
1974         if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1975                 cmd->buf_fmt = fmt << 4;
1976         else
1977                 cmd->buf_fmt = fmt;
1978
1979         return len;
1980
1981 unmap:
1982         srp_unmap_data(scmnd, ch, req);
1983         if (ret == -ENOMEM && req->nmdesc >= target->mr_pool_size)
1984                 ret = -E2BIG;
1985         return ret;
1986 }
1987
1988 /*
1989  * Return an IU and possible credit to the free pool
1990  */
1991 static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
1992                           enum srp_iu_type iu_type)
1993 {
1994         unsigned long flags;
1995
1996         spin_lock_irqsave(&ch->lock, flags);
1997         list_add(&iu->list, &ch->free_tx);
1998         if (iu_type != SRP_IU_RSP)
1999                 ++ch->req_lim;
2000         spin_unlock_irqrestore(&ch->lock, flags);
2001 }
2002
2003 /*
2004  * Must be called with ch->lock held to protect req_lim and free_tx.
2005  * If IU is not sent, it must be returned using srp_put_tx_iu().
2006  *
2007  * Note:
2008  * An upper limit for the number of allocated information units for each
2009  * request type is:
2010  * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
2011  *   more than Scsi_Host.can_queue requests.
2012  * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
2013  * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
2014  *   one unanswered SRP request to an initiator.
2015  */
2016 static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
2017                                       enum srp_iu_type iu_type)
2018 {
2019         struct srp_target_port *target = ch->target;
2020         s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
2021         struct srp_iu *iu;
2022
2023         lockdep_assert_held(&ch->lock);
2024
2025         ib_process_cq_direct(ch->send_cq, -1);
2026
2027         if (list_empty(&ch->free_tx))
2028                 return NULL;
2029
2030         /* Initiator responses to target requests do not consume credits */
2031         if (iu_type != SRP_IU_RSP) {
2032                 if (ch->req_lim <= rsv) {
2033                         ++target->zero_req_lim;
2034                         return NULL;
2035                 }
2036
2037                 --ch->req_lim;
2038         }
2039
2040         iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
2041         list_del(&iu->list);
2042         return iu;
2043 }
2044
2045 /*
2046  * Note: if this function is called from inside ib_drain_sq() then it will
2047  * be called without ch->lock being held. If ib_drain_sq() dequeues a WQE
2048  * with status IB_WC_SUCCESS then that's a bug.
2049  */
2050 static void srp_send_done(struct ib_cq *cq, struct ib_wc *wc)
2051 {
2052         struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
2053         struct srp_rdma_ch *ch = cq->cq_context;
2054
2055         if (unlikely(wc->status != IB_WC_SUCCESS)) {
2056                 srp_handle_qp_err(cq, wc, "SEND");
2057                 return;
2058         }
2059
2060         lockdep_assert_held(&ch->lock);
2061
2062         list_add(&iu->list, &ch->free_tx);
2063 }
2064
2065 /**
2066  * srp_post_send() - send an SRP information unit
2067  * @ch: RDMA channel over which to send the information unit.
2068  * @iu: Information unit to send.
2069  * @len: Length of the information unit excluding immediate data.
2070  */
2071 static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
2072 {
2073         struct srp_target_port *target = ch->target;
2074         struct ib_send_wr wr;
2075
2076         if (WARN_ON_ONCE(iu->num_sge > SRP_MAX_SGE))
2077                 return -EINVAL;
2078
2079         iu->sge[0].addr   = iu->dma;
2080         iu->sge[0].length = len;
2081         iu->sge[0].lkey   = target->lkey;
2082
2083         iu->cqe.done = srp_send_done;
2084
2085         wr.next       = NULL;
2086         wr.wr_cqe     = &iu->cqe;
2087         wr.sg_list    = &iu->sge[0];
2088         wr.num_sge    = iu->num_sge;
2089         wr.opcode     = IB_WR_SEND;
2090         wr.send_flags = IB_SEND_SIGNALED;
2091
2092         return ib_post_send(ch->qp, &wr, NULL);
2093 }
2094
2095 static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
2096 {
2097         struct srp_target_port *target = ch->target;
2098         struct ib_recv_wr wr;
2099         struct ib_sge list;
2100
2101         list.addr   = iu->dma;
2102         list.length = iu->size;
2103         list.lkey   = target->lkey;
2104
2105         iu->cqe.done = srp_recv_done;
2106
2107         wr.next     = NULL;
2108         wr.wr_cqe   = &iu->cqe;
2109         wr.sg_list  = &list;
2110         wr.num_sge  = 1;
2111
2112         return ib_post_recv(ch->qp, &wr, NULL);
2113 }
2114
2115 static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
2116 {
2117         struct srp_target_port *target = ch->target;
2118         struct srp_request *req;
2119         struct scsi_cmnd *scmnd;
2120         unsigned long flags;
2121
2122         if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
2123                 spin_lock_irqsave(&ch->lock, flags);
2124                 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
2125                 if (rsp->tag == ch->tsk_mgmt_tag) {
2126                         ch->tsk_mgmt_status = -1;
2127                         if (be32_to_cpu(rsp->resp_data_len) >= 4)
2128                                 ch->tsk_mgmt_status = rsp->data[3];
2129                         complete(&ch->tsk_mgmt_done);
2130                 } else {
2131                         shost_printk(KERN_ERR, target->scsi_host,
2132                                      "Received tsk mgmt response too late for tag %#llx\n",
2133                                      rsp->tag);
2134                 }
2135                 spin_unlock_irqrestore(&ch->lock, flags);
2136         } else {
2137                 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
2138                 if (scmnd && scmnd->host_scribble) {
2139                         req = (void *)scmnd->host_scribble;
2140                         scmnd = srp_claim_req(ch, req, NULL, scmnd);
2141                 } else {
2142                         scmnd = NULL;
2143                 }
2144                 if (!scmnd) {
2145                         shost_printk(KERN_ERR, target->scsi_host,
2146                                      "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
2147                                      rsp->tag, ch - target->ch, ch->qp->qp_num);
2148
2149                         spin_lock_irqsave(&ch->lock, flags);
2150                         ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
2151                         spin_unlock_irqrestore(&ch->lock, flags);
2152
2153                         return;
2154                 }
2155                 scmnd->result = rsp->status;
2156
2157                 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
2158                         memcpy(scmnd->sense_buffer, rsp->data +
2159                                be32_to_cpu(rsp->resp_data_len),
2160                                min_t(int, be32_to_cpu(rsp->sense_data_len),
2161                                      SCSI_SENSE_BUFFERSIZE));
2162                 }
2163
2164                 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
2165                         scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
2166                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER))
2167                         scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt));
2168                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
2169                         scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
2170                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
2171                         scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
2172
2173                 srp_free_req(ch, req, scmnd,
2174                              be32_to_cpu(rsp->req_lim_delta));
2175
2176                 scmnd->host_scribble = NULL;
2177                 scmnd->scsi_done(scmnd);
2178         }
2179 }
2180
2181 static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
2182                                void *rsp, int len)
2183 {
2184         struct srp_target_port *target = ch->target;
2185         struct ib_device *dev = target->srp_host->srp_dev->dev;
2186         unsigned long flags;
2187         struct srp_iu *iu;
2188         int err;
2189
2190         spin_lock_irqsave(&ch->lock, flags);
2191         ch->req_lim += req_delta;
2192         iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
2193         spin_unlock_irqrestore(&ch->lock, flags);
2194
2195         if (!iu) {
2196                 shost_printk(KERN_ERR, target->scsi_host, PFX
2197                              "no IU available to send response\n");
2198                 return 1;
2199         }
2200
2201         iu->num_sge = 1;
2202         ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
2203         memcpy(iu->buf, rsp, len);
2204         ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
2205
2206         err = srp_post_send(ch, iu, len);
2207         if (err) {
2208                 shost_printk(KERN_ERR, target->scsi_host, PFX
2209                              "unable to post response: %d\n", err);
2210                 srp_put_tx_iu(ch, iu, SRP_IU_RSP);
2211         }
2212
2213         return err;
2214 }
2215
2216 static void srp_process_cred_req(struct srp_rdma_ch *ch,
2217                                  struct srp_cred_req *req)
2218 {
2219         struct srp_cred_rsp rsp = {
2220                 .opcode = SRP_CRED_RSP,
2221                 .tag = req->tag,
2222         };
2223         s32 delta = be32_to_cpu(req->req_lim_delta);
2224
2225         if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
2226                 shost_printk(KERN_ERR, ch->target->scsi_host, PFX
2227                              "problems processing SRP_CRED_REQ\n");
2228 }
2229
2230 static void srp_process_aer_req(struct srp_rdma_ch *ch,
2231                                 struct srp_aer_req *req)
2232 {
2233         struct srp_target_port *target = ch->target;
2234         struct srp_aer_rsp rsp = {
2235                 .opcode = SRP_AER_RSP,
2236                 .tag = req->tag,
2237         };
2238         s32 delta = be32_to_cpu(req->req_lim_delta);
2239
2240         shost_printk(KERN_ERR, target->scsi_host, PFX
2241                      "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun));
2242
2243         if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
2244                 shost_printk(KERN_ERR, target->scsi_host, PFX
2245                              "problems processing SRP_AER_REQ\n");
2246 }
2247
2248 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc)
2249 {
2250         struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
2251         struct srp_rdma_ch *ch = cq->cq_context;
2252         struct srp_target_port *target = ch->target;
2253         struct ib_device *dev = target->srp_host->srp_dev->dev;
2254         int res;
2255         u8 opcode;
2256
2257         if (unlikely(wc->status != IB_WC_SUCCESS)) {
2258                 srp_handle_qp_err(cq, wc, "RECV");
2259                 return;
2260         }
2261
2262         ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
2263                                    DMA_FROM_DEVICE);
2264
2265         opcode = *(u8 *) iu->buf;
2266
2267         if (0) {
2268                 shost_printk(KERN_ERR, target->scsi_host,
2269                              PFX "recv completion, opcode 0x%02x\n", opcode);
2270                 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
2271                                iu->buf, wc->byte_len, true);
2272         }
2273
2274         switch (opcode) {
2275         case SRP_RSP:
2276                 srp_process_rsp(ch, iu->buf);
2277                 break;
2278
2279         case SRP_CRED_REQ:
2280                 srp_process_cred_req(ch, iu->buf);
2281                 break;
2282
2283         case SRP_AER_REQ:
2284                 srp_process_aer_req(ch, iu->buf);
2285                 break;
2286
2287         case SRP_T_LOGOUT:
2288                 /* XXX Handle target logout */
2289                 shost_printk(KERN_WARNING, target->scsi_host,
2290                              PFX "Got target logout request\n");
2291                 break;
2292
2293         default:
2294                 shost_printk(KERN_WARNING, target->scsi_host,
2295                              PFX "Unhandled SRP opcode 0x%02x\n", opcode);
2296                 break;
2297         }
2298
2299         ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
2300                                       DMA_FROM_DEVICE);
2301
2302         res = srp_post_recv(ch, iu);
2303         if (res != 0)
2304                 shost_printk(KERN_ERR, target->scsi_host,
2305                              PFX "Recv failed with error code %d\n", res);
2306 }
2307
2308 /**
2309  * srp_tl_err_work() - handle a transport layer error
2310  * @work: Work structure embedded in an SRP target port.
2311  *
2312  * Note: This function may get invoked before the rport has been created,
2313  * hence the target->rport test.
2314  */
2315 static void srp_tl_err_work(struct work_struct *work)
2316 {
2317         struct srp_target_port *target;
2318
2319         target = container_of(work, struct srp_target_port, tl_err_work);
2320         if (target->rport)
2321                 srp_start_tl_fail_timers(target->rport);
2322 }
2323
2324 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
2325                 const char *opname)
2326 {
2327         struct srp_rdma_ch *ch = cq->cq_context;
2328         struct srp_target_port *target = ch->target;
2329
2330         if (ch->connected && !target->qp_in_error) {
2331                 shost_printk(KERN_ERR, target->scsi_host,
2332                              PFX "failed %s status %s (%d) for CQE %p\n",
2333                              opname, ib_wc_status_msg(wc->status), wc->status,
2334                              wc->wr_cqe);
2335                 queue_work(system_long_wq, &target->tl_err_work);
2336         }
2337         target->qp_in_error = true;
2338 }
2339
2340 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
2341 {
2342         struct srp_target_port *target = host_to_target(shost);
2343         struct srp_rport *rport = target->rport;
2344         struct srp_rdma_ch *ch;
2345         struct srp_request *req;
2346         struct srp_iu *iu;
2347         struct srp_cmd *cmd;
2348         struct ib_device *dev;
2349         unsigned long flags;
2350         u32 tag;
2351         u16 idx;
2352         int len, ret;
2353         const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;
2354
2355         /*
2356          * The SCSI EH thread is the only context from which srp_queuecommand()
2357          * can get invoked for blocked devices (SDEV_BLOCK /
2358          * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
2359          * locking the rport mutex if invoked from inside the SCSI EH.
2360          */
2361         if (in_scsi_eh)
2362                 mutex_lock(&rport->mutex);
2363
2364         scmnd->result = srp_chkready(target->rport);
2365         if (unlikely(scmnd->result))
2366                 goto err;
2367
2368         WARN_ON_ONCE(scmnd->request->tag < 0);
2369         tag = blk_mq_unique_tag(scmnd->request);
2370         ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
2371         idx = blk_mq_unique_tag_to_tag(tag);
2372         WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n",
2373                   dev_name(&shost->shost_gendev), tag, idx,
2374                   target->req_ring_size);
2375
2376         spin_lock_irqsave(&ch->lock, flags);
2377         iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
2378         spin_unlock_irqrestore(&ch->lock, flags);
2379
2380         if (!iu)
2381                 goto err;
2382
2383         req = &ch->req_ring[idx];
2384         dev = target->srp_host->srp_dev->dev;
2385         ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_it_iu_len,
2386                                    DMA_TO_DEVICE);
2387
2388         scmnd->host_scribble = (void *) req;
2389
2390         cmd = iu->buf;
2391         memset(cmd, 0, sizeof *cmd);
2392
2393         cmd->opcode = SRP_CMD;
2394         int_to_scsilun(scmnd->device->lun, &cmd->lun);
2395         cmd->tag    = tag;
2396         memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);
2397         if (unlikely(scmnd->cmd_len > sizeof(cmd->cdb))) {
2398                 cmd->add_cdb_len = round_up(scmnd->cmd_len - sizeof(cmd->cdb),
2399                                             4);
2400                 if (WARN_ON_ONCE(cmd->add_cdb_len > SRP_MAX_ADD_CDB_LEN))
2401                         goto err_iu;
2402         }
2403
2404         req->scmnd    = scmnd;
2405         req->cmd      = iu;
2406
2407         len = srp_map_data(scmnd, ch, req);
2408         if (len < 0) {
2409                 shost_printk(KERN_ERR, target->scsi_host,
2410                              PFX "Failed to map data (%d)\n", len);
2411                 /*
2412                  * If we ran out of memory descriptors (-ENOMEM) because an
2413                  * application is queuing many requests with more than
2414                  * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
2415                  * to reduce queue depth temporarily.
2416                  */
2417                 scmnd->result = len == -ENOMEM ?
2418                         DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
2419                 goto err_iu;
2420         }
2421
2422         ib_dma_sync_single_for_device(dev, iu->dma, ch->max_it_iu_len,
2423                                       DMA_TO_DEVICE);
2424
2425         if (srp_post_send(ch, iu, len)) {
2426                 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
2427                 scmnd->result = DID_ERROR << 16;
2428                 goto err_unmap;
2429         }
2430
2431         ret = 0;
2432
2433 unlock_rport:
2434         if (in_scsi_eh)
2435                 mutex_unlock(&rport->mutex);
2436
2437         return ret;
2438
2439 err_unmap:
2440         srp_unmap_data(scmnd, ch, req);
2441
2442 err_iu:
2443         srp_put_tx_iu(ch, iu, SRP_IU_CMD);
2444
2445         /*
2446          * Avoid that the loops that iterate over the request ring can
2447          * encounter a dangling SCSI command pointer.
2448          */
2449         req->scmnd = NULL;
2450
2451 err:
2452         if (scmnd->result) {
2453                 scmnd->scsi_done(scmnd);
2454                 ret = 0;
2455         } else {
2456                 ret = SCSI_MLQUEUE_HOST_BUSY;
2457         }
2458
2459         goto unlock_rport;
2460 }
2461
2462 /*
2463  * Note: the resources allocated in this function are freed in
2464  * srp_free_ch_ib().
2465  */
2466 static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
2467 {
2468         struct srp_target_port *target = ch->target;
2469         int i;
2470
2471         ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
2472                               GFP_KERNEL);
2473         if (!ch->rx_ring)
2474                 goto err_no_ring;
2475         ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
2476                               GFP_KERNEL);
2477         if (!ch->tx_ring)
2478                 goto err_no_ring;
2479
2480         for (i = 0; i < target->queue_size; ++i) {
2481                 ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
2482                                               ch->max_ti_iu_len,
2483                                               GFP_KERNEL, DMA_FROM_DEVICE);
2484                 if (!ch->rx_ring[i])
2485                         goto err;
2486         }
2487
2488         for (i = 0; i < target->queue_size; ++i) {
2489                 ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
2490                                               ch->max_it_iu_len,
2491                                               GFP_KERNEL, DMA_TO_DEVICE);
2492                 if (!ch->tx_ring[i])
2493                         goto err;
2494
2495                 list_add(&ch->tx_ring[i]->list, &ch->free_tx);
2496         }
2497
2498         return 0;
2499
2500 err:
2501         for (i = 0; i < target->queue_size; ++i) {
2502                 srp_free_iu(target->srp_host, ch->rx_ring[i]);
2503                 srp_free_iu(target->srp_host, ch->tx_ring[i]);
2504         }
2505
2506
2507 err_no_ring:
2508         kfree(ch->tx_ring);
2509         ch->tx_ring = NULL;
2510         kfree(ch->rx_ring);
2511         ch->rx_ring = NULL;
2512
2513         return -ENOMEM;
2514 }
2515
2516 static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
2517 {
2518         uint64_t T_tr_ns, max_compl_time_ms;
2519         uint32_t rq_tmo_jiffies;
2520
2521         /*
2522          * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
2523          * table 91), both the QP timeout and the retry count have to be set
2524          * for RC QP's during the RTR to RTS transition.
2525          */
2526         WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
2527                      (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));
2528
2529         /*
2530          * Set target->rq_tmo_jiffies to one second more than the largest time
2531          * it can take before an error completion is generated. See also
2532          * C9-140..142 in the IBTA spec for more information about how to
2533          * convert the QP Local ACK Timeout value to nanoseconds.
2534          */
2535         T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
2536         max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
2537         do_div(max_compl_time_ms, NSEC_PER_MSEC);
2538         rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);
2539
2540         return rq_tmo_jiffies;
2541 }
2542
2543 static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
2544                                const struct srp_login_rsp *lrsp,
2545                                struct srp_rdma_ch *ch)
2546 {
2547         struct srp_target_port *target = ch->target;
2548         struct ib_qp_attr *qp_attr = NULL;
2549         int attr_mask = 0;
2550         int ret = 0;
2551         int i;
2552
2553         if (lrsp->opcode == SRP_LOGIN_RSP) {
2554                 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
2555                 ch->req_lim       = be32_to_cpu(lrsp->req_lim_delta);
2556                 ch->use_imm_data  = lrsp->rsp_flags & SRP_LOGIN_RSP_IMMED_SUPP;
2557                 ch->max_it_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt,
2558                                                       ch->use_imm_data);
2559                 WARN_ON_ONCE(ch->max_it_iu_len >
2560                              be32_to_cpu(lrsp->max_it_iu_len));
2561
2562                 if (ch->use_imm_data)
2563                         shost_printk(KERN_DEBUG, target->scsi_host,
2564                                      PFX "using immediate data\n");
2565
2566                 /*
2567                  * Reserve credits for task management so we don't
2568                  * bounce requests back to the SCSI mid-layer.
2569                  */
2570                 target->scsi_host->can_queue
2571                         = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
2572                               target->scsi_host->can_queue);
2573                 target->scsi_host->cmd_per_lun
2574                         = min_t(int, target->scsi_host->can_queue,
2575                                 target->scsi_host->cmd_per_lun);
2576         } else {
2577                 shost_printk(KERN_WARNING, target->scsi_host,
2578                              PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
2579                 ret = -ECONNRESET;
2580                 goto error;
2581         }
2582
2583         if (!ch->rx_ring) {
2584                 ret = srp_alloc_iu_bufs(ch);
2585                 if (ret)
2586                         goto error;
2587         }
2588
2589         for (i = 0; i < target->queue_size; i++) {
2590                 struct srp_iu *iu = ch->rx_ring[i];
2591
2592                 ret = srp_post_recv(ch, iu);
2593                 if (ret)
2594                         goto error;
2595         }
2596
2597         if (!target->using_rdma_cm) {
2598                 ret = -ENOMEM;
2599                 qp_attr = kmalloc(sizeof(*qp_attr), GFP_KERNEL);
2600                 if (!qp_attr)
2601                         goto error;
2602
2603                 qp_attr->qp_state = IB_QPS_RTR;
2604                 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2605                 if (ret)
2606                         goto error_free;
2607
2608                 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2609                 if (ret)
2610                         goto error_free;
2611
2612                 qp_attr->qp_state = IB_QPS_RTS;
2613                 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2614                 if (ret)
2615                         goto error_free;
2616
2617                 target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
2618
2619                 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2620                 if (ret)
2621                         goto error_free;
2622
2623                 ret = ib_send_cm_rtu(cm_id, NULL, 0);
2624         }
2625
2626 error_free:
2627         kfree(qp_attr);
2628
2629 error:
2630         ch->status = ret;
2631 }
2632
2633 static void srp_ib_cm_rej_handler(struct ib_cm_id *cm_id,
2634                                   const struct ib_cm_event *event,
2635                                   struct srp_rdma_ch *ch)
2636 {
2637         struct srp_target_port *target = ch->target;
2638         struct Scsi_Host *shost = target->scsi_host;
2639         struct ib_class_port_info *cpi;
2640         int opcode;
2641         u16 dlid;
2642
2643         switch (event->param.rej_rcvd.reason) {
2644         case IB_CM_REJ_PORT_CM_REDIRECT:
2645                 cpi = event->param.rej_rcvd.ari;
2646                 dlid = be16_to_cpu(cpi->redirect_lid);
2647                 sa_path_set_dlid(&ch->ib_cm.path, dlid);
2648                 ch->ib_cm.path.pkey = cpi->redirect_pkey;
2649                 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
2650                 memcpy(ch->ib_cm.path.dgid.raw, cpi->redirect_gid, 16);
2651
2652                 ch->status = dlid ? SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
2653                 break;
2654
2655         case IB_CM_REJ_PORT_REDIRECT:
2656                 if (srp_target_is_topspin(target)) {
2657                         union ib_gid *dgid = &ch->ib_cm.path.dgid;
2658
2659                         /*
2660                          * Topspin/Cisco SRP gateways incorrectly send
2661                          * reject reason code 25 when they mean 24
2662                          * (port redirect).
2663                          */
2664                         memcpy(dgid->raw, event->param.rej_rcvd.ari, 16);
2665
2666                         shost_printk(KERN_DEBUG, shost,
2667                                      PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
2668                                      be64_to_cpu(dgid->global.subnet_prefix),
2669                                      be64_to_cpu(dgid->global.interface_id));
2670
2671                         ch->status = SRP_PORT_REDIRECT;
2672                 } else {
2673                         shost_printk(KERN_WARNING, shost,
2674                                      "  REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
2675                         ch->status = -ECONNRESET;
2676                 }
2677                 break;
2678
2679         case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
2680                 shost_printk(KERN_WARNING, shost,
2681                             "  REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
2682                 ch->status = -ECONNRESET;
2683                 break;
2684
2685         case IB_CM_REJ_CONSUMER_DEFINED:
2686                 opcode = *(u8 *) event->private_data;
2687                 if (opcode == SRP_LOGIN_REJ) {
2688                         struct srp_login_rej *rej = event->private_data;
2689                         u32 reason = be32_to_cpu(rej->reason);
2690
2691                         if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
2692                                 shost_printk(KERN_WARNING, shost,
2693                                              PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
2694                         else
2695                                 shost_printk(KERN_WARNING, shost, PFX
2696                                              "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
2697                                              target->sgid.raw,
2698                                              target->ib_cm.orig_dgid.raw,
2699                                              reason);
2700                 } else
2701                         shost_printk(KERN_WARNING, shost,
2702                                      "  REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
2703                                      " opcode 0x%02x\n", opcode);
2704                 ch->status = -ECONNRESET;
2705                 break;
2706
2707         case IB_CM_REJ_STALE_CONN:
2708                 shost_printk(KERN_WARNING, shost, "  REJ reason: stale connection\n");
2709                 ch->status = SRP_STALE_CONN;
2710                 break;
2711
2712         default:
2713                 shost_printk(KERN_WARNING, shost, "  REJ reason 0x%x\n",
2714                              event->param.rej_rcvd.reason);
2715                 ch->status = -ECONNRESET;
2716         }
2717 }
2718
2719 static int srp_ib_cm_handler(struct ib_cm_id *cm_id,
2720                              const struct ib_cm_event *event)
2721 {
2722         struct srp_rdma_ch *ch = cm_id->context;
2723         struct srp_target_port *target = ch->target;
2724         int comp = 0;
2725
2726         switch (event->event) {
2727         case IB_CM_REQ_ERROR:
2728                 shost_printk(KERN_DEBUG, target->scsi_host,
2729                              PFX "Sending CM REQ failed\n");
2730                 comp = 1;
2731                 ch->status = -ECONNRESET;
2732                 break;
2733
2734         case IB_CM_REP_RECEIVED:
2735                 comp = 1;
2736                 srp_cm_rep_handler(cm_id, event->private_data, ch);
2737                 break;
2738
2739         case IB_CM_REJ_RECEIVED:
2740                 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
2741                 comp = 1;
2742
2743                 srp_ib_cm_rej_handler(cm_id, event, ch);
2744                 break;
2745
2746         case IB_CM_DREQ_RECEIVED:
2747                 shost_printk(KERN_WARNING, target->scsi_host,
2748                              PFX "DREQ received - connection closed\n");
2749                 ch->connected = false;
2750                 if (ib_send_cm_drep(cm_id, NULL, 0))
2751                         shost_printk(KERN_ERR, target->scsi_host,
2752                                      PFX "Sending CM DREP failed\n");
2753                 queue_work(system_long_wq, &target->tl_err_work);
2754                 break;
2755
2756         case IB_CM_TIMEWAIT_EXIT:
2757                 shost_printk(KERN_ERR, target->scsi_host,
2758                              PFX "connection closed\n");
2759                 comp = 1;
2760
2761                 ch->status = 0;
2762                 break;
2763
2764         case IB_CM_MRA_RECEIVED:
2765         case IB_CM_DREQ_ERROR:
2766         case IB_CM_DREP_RECEIVED:
2767                 break;
2768
2769         default:
2770                 shost_printk(KERN_WARNING, target->scsi_host,
2771                              PFX "Unhandled CM event %d\n", event->event);
2772                 break;
2773         }
2774
2775         if (comp)
2776                 complete(&ch->done);
2777
2778         return 0;
2779 }
2780
2781 static void srp_rdma_cm_rej_handler(struct srp_rdma_ch *ch,
2782                                     struct rdma_cm_event *event)
2783 {
2784         struct srp_target_port *target = ch->target;
2785         struct Scsi_Host *shost = target->scsi_host;
2786         int opcode;
2787
2788         switch (event->status) {
2789         case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
2790                 shost_printk(KERN_WARNING, shost,
2791                             "  REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
2792                 ch->status = -ECONNRESET;
2793                 break;
2794
2795         case IB_CM_REJ_CONSUMER_DEFINED:
2796                 opcode = *(u8 *) event->param.conn.private_data;
2797                 if (opcode == SRP_LOGIN_REJ) {
2798                         struct srp_login_rej *rej =
2799                                 (struct srp_login_rej *)
2800                                 event->param.conn.private_data;
2801                         u32 reason = be32_to_cpu(rej->reason);
2802
2803                         if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
2804                                 shost_printk(KERN_WARNING, shost,
2805                                              PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
2806                         else
2807                                 shost_printk(KERN_WARNING, shost,
2808                                             PFX "SRP LOGIN REJECTED, reason 0x%08x\n", reason);
2809                 } else {
2810                         shost_printk(KERN_WARNING, shost,
2811                                      "  REJ reason: IB_CM_REJ_CONSUMER_DEFINED, opcode 0x%02x\n",
2812                                      opcode);
2813                 }
2814                 ch->status = -ECONNRESET;
2815                 break;
2816
2817         case IB_CM_REJ_STALE_CONN:
2818                 shost_printk(KERN_WARNING, shost,
2819                              "  REJ reason: stale connection\n");
2820                 ch->status = SRP_STALE_CONN;
2821                 break;
2822
2823         default:
2824                 shost_printk(KERN_WARNING, shost, "  REJ reason 0x%x\n",
2825                              event->status);
2826                 ch->status = -ECONNRESET;
2827                 break;
2828         }
2829 }
2830
2831 static int srp_rdma_cm_handler(struct rdma_cm_id *cm_id,
2832                                struct rdma_cm_event *event)
2833 {
2834         struct srp_rdma_ch *ch = cm_id->context;
2835         struct srp_target_port *target = ch->target;
2836         int comp = 0;
2837
2838         switch (event->event) {
2839         case RDMA_CM_EVENT_ADDR_RESOLVED:
2840                 ch->status = 0;
2841                 comp = 1;
2842                 break;
2843
2844         case RDMA_CM_EVENT_ADDR_ERROR:
2845                 ch->status = -ENXIO;
2846                 comp = 1;
2847                 break;
2848
2849         case RDMA_CM_EVENT_ROUTE_RESOLVED:
2850                 ch->status = 0;
2851                 comp = 1;
2852                 break;
2853
2854         case RDMA_CM_EVENT_ROUTE_ERROR:
2855         case RDMA_CM_EVENT_UNREACHABLE:
2856                 ch->status = -EHOSTUNREACH;
2857                 comp = 1;
2858                 break;
2859
2860         case RDMA_CM_EVENT_CONNECT_ERROR:
2861                 shost_printk(KERN_DEBUG, target->scsi_host,
2862                              PFX "Sending CM REQ failed\n");
2863                 comp = 1;
2864                 ch->status = -ECONNRESET;
2865                 break;
2866
2867         case RDMA_CM_EVENT_ESTABLISHED:
2868                 comp = 1;
2869                 srp_cm_rep_handler(NULL, event->param.conn.private_data, ch);
2870                 break;
2871
2872         case RDMA_CM_EVENT_REJECTED:
2873                 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
2874                 comp = 1;
2875
2876                 srp_rdma_cm_rej_handler(ch, event);
2877                 break;
2878
2879         case RDMA_CM_EVENT_DISCONNECTED:
2880                 if (ch->connected) {
2881                         shost_printk(KERN_WARNING, target->scsi_host,
2882                                      PFX "received DREQ\n");
2883                         rdma_disconnect(ch->rdma_cm.cm_id);
2884                         comp = 1;
2885                         ch->status = 0;
2886                         queue_work(system_long_wq, &target->tl_err_work);
2887                 }
2888                 break;
2889
2890         case RDMA_CM_EVENT_TIMEWAIT_EXIT:
2891                 shost_printk(KERN_ERR, target->scsi_host,
2892                              PFX "connection closed\n");
2893
2894                 comp = 1;
2895                 ch->status = 0;
2896                 break;
2897
2898         default:
2899                 shost_printk(KERN_WARNING, target->scsi_host,
2900                              PFX "Unhandled CM event %d\n", event->event);
2901                 break;
2902         }
2903
2904         if (comp)
2905                 complete(&ch->done);
2906
2907         return 0;
2908 }
2909
2910 /**
2911  * srp_change_queue_depth - setting device queue depth
2912  * @sdev: scsi device struct
2913  * @qdepth: requested queue depth
2914  *
2915  * Returns queue depth.
2916  */
2917 static int
2918 srp_change_queue_depth(struct scsi_device *sdev, int qdepth)
2919 {
2920         if (!sdev->tagged_supported)
2921                 qdepth = 1;
2922         return scsi_change_queue_depth(sdev, qdepth);
2923 }
2924
2925 static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun,
2926                              u8 func, u8 *status)
2927 {
2928         struct srp_target_port *target = ch->target;
2929         struct srp_rport *rport = target->rport;
2930         struct ib_device *dev = target->srp_host->srp_dev->dev;
2931         struct srp_iu *iu;
2932         struct srp_tsk_mgmt *tsk_mgmt;
2933         int res;
2934
2935         if (!ch->connected || target->qp_in_error)
2936                 return -1;
2937
2938         /*
2939          * Lock the rport mutex to avoid that srp_create_ch_ib() is
2940          * invoked while a task management function is being sent.
2941          */
2942         mutex_lock(&rport->mutex);
2943         spin_lock_irq(&ch->lock);
2944         iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
2945         spin_unlock_irq(&ch->lock);
2946
2947         if (!iu) {
2948                 mutex_unlock(&rport->mutex);
2949
2950                 return -1;
2951         }
2952
2953         iu->num_sge = 1;
2954
2955         ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
2956                                    DMA_TO_DEVICE);
2957         tsk_mgmt = iu->buf;
2958         memset(tsk_mgmt, 0, sizeof *tsk_mgmt);
2959
2960         tsk_mgmt->opcode        = SRP_TSK_MGMT;
2961         int_to_scsilun(lun, &tsk_mgmt->lun);
2962         tsk_mgmt->tsk_mgmt_func = func;
2963         tsk_mgmt->task_tag      = req_tag;
2964
2965         spin_lock_irq(&ch->lock);
2966         ch->tsk_mgmt_tag = (ch->tsk_mgmt_tag + 1) | SRP_TAG_TSK_MGMT;
2967         tsk_mgmt->tag = ch->tsk_mgmt_tag;
2968         spin_unlock_irq(&ch->lock);
2969
2970         init_completion(&ch->tsk_mgmt_done);
2971
2972         ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
2973                                       DMA_TO_DEVICE);
2974         if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
2975                 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
2976                 mutex_unlock(&rport->mutex);
2977
2978                 return -1;
2979         }
2980         res = wait_for_completion_timeout(&ch->tsk_mgmt_done,
2981                                         msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS));
2982         if (res > 0 && status)
2983                 *status = ch->tsk_mgmt_status;
2984         mutex_unlock(&rport->mutex);
2985
2986         WARN_ON_ONCE(res < 0);
2987
2988         return res > 0 ? 0 : -1;
2989 }
2990
2991 static int srp_abort(struct scsi_cmnd *scmnd)
2992 {
2993         struct srp_target_port *target = host_to_target(scmnd->device->host);
2994         struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
2995         u32 tag;
2996         u16 ch_idx;
2997         struct srp_rdma_ch *ch;
2998         int ret;
2999
3000         shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
3001
3002         if (!req)
3003                 return SUCCESS;
3004         tag = blk_mq_unique_tag(scmnd->request);
3005         ch_idx = blk_mq_unique_tag_to_hwq(tag);
3006         if (WARN_ON_ONCE(ch_idx >= target->ch_count))
3007                 return SUCCESS;
3008         ch = &target->ch[ch_idx];
3009         if (!srp_claim_req(ch, req, NULL, scmnd))
3010                 return SUCCESS;
3011         shost_printk(KERN_ERR, target->scsi_host,
3012                      "Sending SRP abort for tag %#x\n", tag);
3013         if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun,
3014                               SRP_TSK_ABORT_TASK, NULL) == 0)
3015                 ret = SUCCESS;
3016         else if (target->rport->state == SRP_RPORT_LOST)
3017                 ret = FAST_IO_FAIL;
3018         else
3019                 ret = FAILED;
3020         if (ret == SUCCESS) {
3021                 srp_free_req(ch, req, scmnd, 0);
3022                 scmnd->result = DID_ABORT << 16;
3023                 scmnd->scsi_done(scmnd);
3024         }
3025
3026         return ret;
3027 }
3028
3029 static int srp_reset_device(struct scsi_cmnd *scmnd)
3030 {
3031         struct srp_target_port *target = host_to_target(scmnd->device->host);
3032         struct srp_rdma_ch *ch;
3033         u8 status;
3034
3035         shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
3036
3037         ch = &target->ch[0];
3038         if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
3039                               SRP_TSK_LUN_RESET, &status))
3040                 return FAILED;
3041         if (status)
3042                 return FAILED;
3043
3044         return SUCCESS;
3045 }
3046
3047 static int srp_reset_host(struct scsi_cmnd *scmnd)
3048 {
3049         struct srp_target_port *target = host_to_target(scmnd->device->host);
3050
3051         shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
3052
3053         return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
3054 }
3055
3056 static int srp_target_alloc(struct scsi_target *starget)
3057 {
3058         struct Scsi_Host *shost = dev_to_shost(starget->dev.parent);
3059         struct srp_target_port *target = host_to_target(shost);
3060
3061         if (target->target_can_queue)
3062                 starget->can_queue = target->target_can_queue;
3063         return 0;
3064 }
3065
3066 static int srp_slave_alloc(struct scsi_device *sdev)
3067 {
3068         struct Scsi_Host *shost = sdev->host;
3069         struct srp_target_port *target = host_to_target(shost);
3070         struct srp_device *srp_dev = target->srp_host->srp_dev;
3071         struct ib_device *ibdev = srp_dev->dev;
3072
3073         if (!(ibdev->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG))
3074                 blk_queue_virt_boundary(sdev->request_queue,
3075                                         ~srp_dev->mr_page_mask);
3076
3077         return 0;
3078 }
3079
3080 static int srp_slave_configure(struct scsi_device *sdev)
3081 {
3082         struct Scsi_Host *shost = sdev->host;
3083         struct srp_target_port *target = host_to_target(shost);
3084         struct request_queue *q = sdev->request_queue;
3085         unsigned long timeout;
3086
3087         if (sdev->type == TYPE_DISK) {
3088                 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
3089                 blk_queue_rq_timeout(q, timeout);
3090         }
3091
3092         return 0;
3093 }
3094
3095 static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
3096                            char *buf)
3097 {
3098         struct srp_target_port *target = host_to_target(class_to_shost(dev));
3099
3100         return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->id_ext));
3101 }
3102
3103 static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
3104                              char *buf)
3105 {
3106         struct srp_target_port *target = host_to_target(class_to_shost(dev));
3107
3108         return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid));
3109 }
3110
3111 static ssize_t show_service_id(struct device *dev,
3112                                struct device_attribute *attr, char *buf)
3113 {
3114         struct srp_target_port *target = host_to_target(class_to_shost(dev));
3115
3116         if (target->using_rdma_cm)
3117                 return -ENOENT;
3118         return sprintf(buf, "0x%016llx\n",
3119                        be64_to_cpu(target->ib_cm.service_id));
3120 }
3121
3122 static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
3123                          char *buf)
3124 {
3125         struct srp_target_port *target = host_to_target(class_to_shost(dev));
3126
3127         if (target->using_rdma_cm)
3128                 return -ENOENT;
3129         return sprintf(buf, "0x%04x\n", be16_to_cpu(target->ib_cm.pkey));
3130 }
3131
3132 static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
3133                          char *buf)
3134 {
3135         struct srp_target_port *target = host_to_target(class_to_shost(dev));
3136
3137         return sprintf(buf, "%pI6\n", target->sgid.raw);
3138 }
3139
3140 static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
3141                          char *buf)
3142 {
3143         struct srp_target_port *target = host_to_target(class_to_shost(dev));
3144         struct srp_rdma_ch *ch = &target->ch[0];
3145
3146         if (target->using_rdma_cm)
3147                 return -ENOENT;
3148         return sprintf(buf, "%pI6\n", ch->ib_cm.path.dgid.raw);
3149 }
3150
3151 static ssize_t show_orig_dgid(struct device *dev,
3152                               struct device_attribute *attr, char *buf)
3153 {
3154         struct srp_target_port *target = host_to_target(class_to_shost(dev));
3155
3156         if (target->using_rdma_cm)
3157                 return -ENOENT;
3158         return sprintf(buf, "%pI6\n", target->ib_cm.orig_dgid.raw);
3159 }
3160
3161 static ssize_t show_req_lim(struct device *dev,
3162                             struct device_attribute *attr, char *buf)
3163 {
3164         struct srp_target_port *target = host_to_target(class_to_shost(dev));
3165         struct srp_rdma_ch *ch;
3166         int i, req_lim = INT_MAX;
3167
3168         for (i = 0; i < target->ch_count; i++) {
3169                 ch = &target->ch[i];
3170                 req_lim = min(req_lim, ch->req_lim);
3171         }
3172         return sprintf(buf, "%d\n", req_lim);
3173 }
3174
3175 static ssize_t show_zero_req_lim(struct device *dev,
3176                                  struct device_attribute *attr, char *buf)
3177 {
3178         struct srp_target_port *target = host_to_target(class_to_shost(dev));
3179
3180         return sprintf(buf, "%d\n", target->zero_req_lim);
3181 }
3182
3183 static ssize_t show_local_ib_port(struct device *dev,
3184                                   struct device_attribute *attr, char *buf)
3185 {
3186         struct srp_target_port *target = host_to_target(class_to_shost(dev));
3187
3188         return sprintf(buf, "%d\n", target->srp_host->port);
3189 }
3190
3191 static ssize_t show_local_ib_device(struct device *dev,
3192                                     struct device_attribute *attr, char *buf)
3193 {
3194         struct srp_target_port *target = host_to_target(class_to_shost(dev));
3195
3196         return sprintf(buf, "%s\n",
3197                        dev_name(&target->srp_host->srp_dev->dev->dev));
3198 }
3199
3200 static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr,
3201                              char *buf)
3202 {
3203         struct srp_target_port *target = host_to_target(class_to_shost(dev));
3204
3205         return sprintf(buf, "%d\n", target->ch_count);
3206 }
3207
3208 static ssize_t show_comp_vector(struct device *dev,
3209                                 struct device_attribute *attr, char *buf)
3210 {
3211         struct srp_target_port *target = host_to_target(class_to_shost(dev));
3212
3213         return sprintf(buf, "%d\n", target->comp_vector);
3214 }
3215
3216 static ssize_t show_tl_retry_count(struct device *dev,
3217                                    struct device_attribute *attr, char *buf)
3218 {
3219         struct srp_target_port *target = host_to_target(class_to_shost(dev));
3220
3221         return sprintf(buf, "%d\n", target->tl_retry_count);
3222 }
3223
3224 static ssize_t show_cmd_sg_entries(struct device *dev,
3225                                    struct device_attribute *attr, char *buf)
3226 {
3227         struct srp_target_port *target = host_to_target(class_to_shost(dev));
3228
3229         return sprintf(buf, "%u\n", target->cmd_sg_cnt);
3230 }
3231
3232 static ssize_t show_allow_ext_sg(struct device *dev,
3233                                  struct device_attribute *attr, char *buf)
3234 {
3235         struct srp_target_port *target = host_to_target(class_to_shost(dev));
3236
3237         return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
3238 }
3239
3240 static DEVICE_ATTR(id_ext,          S_IRUGO, show_id_ext,          NULL);
3241 static DEVICE_ATTR(ioc_guid,        S_IRUGO, show_ioc_guid,        NULL);
3242 static DEVICE_ATTR(service_id,      S_IRUGO, show_service_id,      NULL);
3243 static DEVICE_ATTR(pkey,            S_IRUGO, show_pkey,            NULL);
3244 static DEVICE_ATTR(sgid,            S_IRUGO, show_sgid,            NULL);
3245 static DEVICE_ATTR(dgid,            S_IRUGO, show_dgid,            NULL);
3246 static DEVICE_ATTR(orig_dgid,       S_IRUGO, show_orig_dgid,       NULL);
3247 static DEVICE_ATTR(req_lim,         S_IRUGO, show_req_lim,         NULL);
3248 static DEVICE_ATTR(zero_req_lim,    S_IRUGO, show_zero_req_lim,    NULL);
3249 static DEVICE_ATTR(local_ib_port,   S_IRUGO, show_local_ib_port,   NULL);
3250 static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
3251 static DEVICE_ATTR(ch_count,        S_IRUGO, show_ch_count,        NULL);
3252 static DEVICE_ATTR(comp_vector,     S_IRUGO, show_comp_vector,     NULL);
3253 static DEVICE_ATTR(tl_retry_count,  S_IRUGO, show_tl_retry_count,  NULL);
3254 static DEVICE_ATTR(cmd_sg_entries,  S_IRUGO, show_cmd_sg_entries,  NULL);
3255 static DEVICE_ATTR(allow_ext_sg,    S_IRUGO, show_allow_ext_sg,    NULL);
3256
3257 static struct device_attribute *srp_host_attrs[] = {
3258         &dev_attr_id_ext,
3259         &dev_attr_ioc_guid,
3260         &dev_attr_service_id,
3261         &dev_attr_pkey,
3262         &dev_attr_sgid,
3263         &dev_attr_dgid,
3264         &dev_attr_orig_dgid,
3265         &dev_attr_req_lim,
3266         &dev_attr_zero_req_lim,
3267         &dev_attr_local_ib_port,
3268         &dev_attr_local_ib_device,
3269         &dev_attr_ch_count,
3270         &dev_attr_comp_vector,
3271         &dev_attr_tl_retry_count,
3272         &dev_attr_cmd_sg_entries,
3273         &dev_attr_allow_ext_sg,
3274         NULL
3275 };
3276
3277 static struct scsi_host_template srp_template = {
3278         .module                         = THIS_MODULE,
3279         .name                           = "InfiniBand SRP initiator",
3280         .proc_name                      = DRV_NAME,
3281         .target_alloc                   = srp_target_alloc,
3282         .slave_alloc                    = srp_slave_alloc,
3283         .slave_configure                = srp_slave_configure,
3284         .info                           = srp_target_info,
3285         .queuecommand                   = srp_queuecommand,
3286         .change_queue_depth             = srp_change_queue_depth,
3287         .eh_timed_out                   = srp_timed_out,
3288         .eh_abort_handler               = srp_abort,
3289         .eh_device_reset_handler        = srp_reset_device,
3290         .eh_host_reset_handler          = srp_reset_host,
3291         .skip_settle_delay              = true,
3292         .sg_tablesize                   = SRP_DEF_SG_TABLESIZE,
3293         .can_queue                      = SRP_DEFAULT_CMD_SQ_SIZE,
3294         .this_id                        = -1,
3295         .cmd_per_lun                    = SRP_DEFAULT_CMD_SQ_SIZE,
3296         .shost_attrs                    = srp_host_attrs,
3297         .track_queue_depth              = 1,
3298 };
3299
3300 static int srp_sdev_count(struct Scsi_Host *host)
3301 {
3302         struct scsi_device *sdev;
3303         int c = 0;
3304
3305         shost_for_each_device(sdev, host)
3306                 c++;
3307
3308         return c;
3309 }
3310
3311 /*
3312  * Return values:
3313  * < 0 upon failure. Caller is responsible for SRP target port cleanup.
3314  * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port
3315  *    removal has been scheduled.
3316  * 0 and target->state != SRP_TARGET_REMOVED upon success.
3317  */
3318 static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
3319 {
3320         struct srp_rport_identifiers ids;
3321         struct srp_rport *rport;
3322
3323         target->state = SRP_TARGET_SCANNING;
3324         sprintf(target->target_name, "SRP.T10:%016llX",
3325                 be64_to_cpu(target->id_ext));
3326
3327         if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dev.parent))
3328                 return -ENODEV;
3329
3330         memcpy(ids.port_id, &target->id_ext, 8);
3331         memcpy(ids.port_id + 8, &target->ioc_guid, 8);
3332         ids.roles = SRP_RPORT_ROLE_TARGET;
3333         rport = srp_rport_add(target->scsi_host, &ids);
3334         if (IS_ERR(rport)) {
3335                 scsi_remove_host(target->scsi_host);
3336                 return PTR_ERR(rport);
3337         }
3338
3339         rport->lld_data = target;
3340         target->rport = rport;
3341
3342         spin_lock(&host->target_lock);
3343         list_add_tail(&target->list, &host->target_list);
3344         spin_unlock(&host->target_lock);
3345
3346         scsi_scan_target(&target->scsi_host->shost_gendev,
3347                          0, target->scsi_id, SCAN_WILD_CARD, SCSI_SCAN_INITIAL);
3348
3349         if (srp_connected_ch(target) < target->ch_count ||
3350             target->qp_in_error) {
3351                 shost_printk(KERN_INFO, target->scsi_host,
3352                              PFX "SCSI scan failed - removing SCSI host\n");
3353                 srp_queue_remove_work(target);
3354                 goto out;
3355         }
3356
3357         pr_debug("%s: SCSI scan succeeded - detected %d LUNs\n",
3358                  dev_name(&target->scsi_host->shost_gendev),
3359                  srp_sdev_count(target->scsi_host));
3360
3361         spin_lock_irq(&target->lock);
3362         if (target->state == SRP_TARGET_SCANNING)
3363                 target->state = SRP_TARGET_LIVE;
3364         spin_unlock_irq(&target->lock);
3365
3366 out:
3367         return 0;
3368 }
3369
3370 static void srp_release_dev(struct device *dev)
3371 {
3372         struct srp_host *host =
3373                 container_of(dev, struct srp_host, dev);
3374
3375         complete(&host->released);
3376 }
3377
3378 static struct class srp_class = {
3379         .name    = "infiniband_srp",
3380         .dev_release = srp_release_dev
3381 };
3382
3383 /**
3384  * srp_conn_unique() - check whether the connection to a target is unique
3385  * @host:   SRP host.
3386  * @target: SRP target port.
3387  */
3388 static bool srp_conn_unique(struct srp_host *host,
3389                             struct srp_target_port *target)
3390 {
3391         struct srp_target_port *t;
3392         bool ret = false;
3393
3394         if (target->state == SRP_TARGET_REMOVED)
3395                 goto out;
3396
3397         ret = true;
3398
3399         spin_lock(&host->target_lock);
3400         list_for_each_entry(t, &host->target_list, list) {
3401                 if (t != target &&
3402                     target->id_ext == t->id_ext &&
3403                     target->ioc_guid == t->ioc_guid &&
3404                     target->initiator_ext == t->initiator_ext) {
3405                         ret = false;
3406                         break;
3407                 }
3408         }
3409         spin_unlock(&host->target_lock);
3410
3411 out:
3412         return ret;
3413 }
3414
3415 /*
3416  * Target ports are added by writing
3417  *
3418  *     id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
3419  *     pkey=<P_Key>,service_id=<service ID>
3420  * or
3421  *     id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,
3422  *     [src=<IPv4 address>,]dest=<IPv4 address>:<port number>
3423  *
3424  * to the add_target sysfs attribute.
3425  */
3426 enum {
3427         SRP_OPT_ERR             = 0,
3428         SRP_OPT_ID_EXT          = 1 << 0,
3429         SRP_OPT_IOC_GUID        = 1 << 1,
3430         SRP_OPT_DGID            = 1 << 2,
3431         SRP_OPT_PKEY            = 1 << 3,
3432         SRP_OPT_SERVICE_ID      = 1 << 4,
3433         SRP_OPT_MAX_SECT        = 1 << 5,
3434         SRP_OPT_MAX_CMD_PER_LUN = 1 << 6,
3435         SRP_OPT_IO_CLASS        = 1 << 7,
3436         SRP_OPT_INITIATOR_EXT   = 1 << 8,
3437         SRP_OPT_CMD_SG_ENTRIES  = 1 << 9,
3438         SRP_OPT_ALLOW_EXT_SG    = 1 << 10,
3439         SRP_OPT_SG_TABLESIZE    = 1 << 11,
3440         SRP_OPT_COMP_VECTOR     = 1 << 12,
3441         SRP_OPT_TL_RETRY_COUNT  = 1 << 13,
3442         SRP_OPT_QUEUE_SIZE      = 1 << 14,
3443         SRP_OPT_IP_SRC          = 1 << 15,
3444         SRP_OPT_IP_DEST         = 1 << 16,
3445         SRP_OPT_TARGET_CAN_QUEUE= 1 << 17,
3446 };
3447
3448 static unsigned int srp_opt_mandatory[] = {
3449         SRP_OPT_ID_EXT          |
3450         SRP_OPT_IOC_GUID        |
3451         SRP_OPT_DGID            |
3452         SRP_OPT_PKEY            |
3453         SRP_OPT_SERVICE_ID,
3454         SRP_OPT_ID_EXT          |
3455         SRP_OPT_IOC_GUID        |
3456         SRP_OPT_IP_DEST,
3457 };
3458
3459 static const match_table_t srp_opt_tokens = {
3460         { SRP_OPT_ID_EXT,               "id_ext=%s"             },
3461         { SRP_OPT_IOC_GUID,             "ioc_guid=%s"           },
3462         { SRP_OPT_DGID,                 "dgid=%s"               },
3463         { SRP_OPT_PKEY,                 "pkey=%x"               },
3464         { SRP_OPT_SERVICE_ID,           "service_id=%s"         },
3465         { SRP_OPT_MAX_SECT,             "max_sect=%d"           },
3466         { SRP_OPT_MAX_CMD_PER_LUN,      "max_cmd_per_lun=%d"    },
3467         { SRP_OPT_TARGET_CAN_QUEUE,     "target_can_queue=%d"   },
3468         { SRP_OPT_IO_CLASS,             "io_class=%x"           },
3469         { SRP_OPT_INITIATOR_EXT,        "initiator_ext=%s"      },
3470         { SRP_OPT_CMD_SG_ENTRIES,       "cmd_sg_entries=%u"     },
3471         { SRP_OPT_ALLOW_EXT_SG,         "allow_ext_sg=%u"       },
3472         { SRP_OPT_SG_TABLESIZE,         "sg_tablesize=%u"       },
3473         { SRP_OPT_COMP_VECTOR,          "comp_vector=%u"        },
3474         { SRP_OPT_TL_RETRY_COUNT,       "tl_retry_count=%u"     },
3475         { SRP_OPT_QUEUE_SIZE,           "queue_size=%d"         },
3476         { SRP_OPT_IP_SRC,               "src=%s"                },
3477         { SRP_OPT_IP_DEST,              "dest=%s"               },
3478         { SRP_OPT_ERR,                  NULL                    }
3479 };
3480
3481 /**
3482  * srp_parse_in - parse an IP address and port number combination
3483  * @net:           [in]  Network namespace.
3484  * @sa:            [out] Address family, IP address and port number.
3485  * @addr_port_str: [in]  IP address and port number.
3486  *
3487  * Parse the following address formats:
3488  * - IPv4: <ip_address>:<port>, e.g. 1.2.3.4:5.
3489  * - IPv6: \[<ipv6_address>\]:<port>, e.g. [1::2:3%4]:5.
3490  */
3491 static int srp_parse_in(struct net *net, struct sockaddr_storage *sa,
3492                         const char *addr_port_str)
3493 {
3494         char *addr_end, *addr = kstrdup(addr_port_str, GFP_KERNEL);
3495         char *port_str;
3496         int ret;
3497
3498         if (!addr)
3499                 return -ENOMEM;
3500         port_str = strrchr(addr, ':');
3501         if (!port_str)
3502                 return -EINVAL;
3503         *port_str++ = '\0';
3504         ret = inet_pton_with_scope(net, AF_INET, addr, port_str, sa);
3505         if (ret && addr[0]) {
3506                 addr_end = addr + strlen(addr) - 1;
3507                 if (addr[0] == '[' && *addr_end == ']') {
3508                         *addr_end = '\0';
3509                         ret = inet_pton_with_scope(net, AF_INET6, addr + 1,
3510                                                    port_str, sa);
3511                 }
3512         }
3513         kfree(addr);
3514         pr_debug("%s -> %pISpfsc\n", addr_port_str, sa);
3515         return ret;
3516 }
3517
3518 static int srp_parse_options(struct net *net, const char *buf,
3519                              struct srp_target_port *target)
3520 {
3521         char *options, *sep_opt;
3522         char *p;
3523         substring_t args[MAX_OPT_ARGS];
3524         unsigned long long ull;
3525         int opt_mask = 0;
3526         int token;
3527         int ret = -EINVAL;
3528         int i;
3529
3530         options = kstrdup(buf, GFP_KERNEL);
3531         if (!options)
3532                 return -ENOMEM;
3533
3534         sep_opt = options;
3535         while ((p = strsep(&sep_opt, ",\n")) != NULL) {
3536                 if (!*p)
3537                         continue;
3538
3539                 token = match_token(p, srp_opt_tokens, args);
3540                 opt_mask |= token;
3541
3542                 switch (token) {
3543                 case SRP_OPT_ID_EXT:
3544                         p = match_strdup(args);
3545                         if (!p) {
3546                                 ret = -ENOMEM;
3547                                 goto out;
3548                         }
3549                         ret = kstrtoull(p, 16, &ull);
3550                         if (ret) {
3551                                 pr_warn("invalid id_ext parameter '%s'\n", p);
3552                                 kfree(p);
3553                                 goto out;
3554                         }
3555                         target->id_ext = cpu_to_be64(ull);
3556                         kfree(p);
3557                         break;
3558
3559                 case SRP_OPT_IOC_GUID:
3560                         p = match_strdup(args);
3561                         if (!p) {
3562                                 ret = -ENOMEM;
3563                                 goto out;
3564                         }
3565                         ret = kstrtoull(p, 16, &ull);
3566                         if (ret) {
3567                                 pr_warn("invalid ioc_guid parameter '%s'\n", p);
3568                                 kfree(p);
3569                                 goto out;
3570                         }
3571                         target->ioc_guid = cpu_to_be64(ull);
3572                         kfree(p);
3573                         break;
3574
3575                 case SRP_OPT_DGID:
3576                         p = match_strdup(args);
3577                         if (!p) {
3578                                 ret = -ENOMEM;
3579                                 goto out;
3580                         }
3581                         if (strlen(p) != 32) {
3582                                 pr_warn("bad dest GID parameter '%s'\n", p);
3583                                 kfree(p);
3584                                 goto out;
3585                         }
3586
3587                         ret = hex2bin(target->ib_cm.orig_dgid.raw, p, 16);
3588                         kfree(p);
3589                         if (ret < 0)
3590                                 goto out;
3591                         break;
3592
3593                 case SRP_OPT_PKEY:
3594                         if (match_hex(args, &token)) {
3595                                 pr_warn("bad P_Key parameter '%s'\n", p);
3596                                 goto out;
3597                         }
3598                         target->ib_cm.pkey = cpu_to_be16(token);
3599                         break;
3600
3601                 case SRP_OPT_SERVICE_ID:
3602                         p = match_strdup(args);
3603                         if (!p) {
3604                                 ret = -ENOMEM;
3605                                 goto out;
3606                         }
3607                         ret = kstrtoull(p, 16, &ull);
3608                         if (ret) {
3609                                 pr_warn("bad service_id parameter '%s'\n", p);
3610                                 kfree(p);
3611                                 goto out;
3612                         }
3613                         target->ib_cm.service_id = cpu_to_be64(ull);
3614                         kfree(p);
3615                         break;
3616
3617                 case SRP_OPT_IP_SRC:
3618                         p = match_strdup(args);
3619                         if (!p) {
3620                                 ret = -ENOMEM;
3621                                 goto out;
3622                         }
3623                         ret = srp_parse_in(net, &target->rdma_cm.src.ss, p);
3624                         if (ret < 0) {
3625                                 pr_warn("bad source parameter '%s'\n", p);
3626                                 kfree(p);
3627                                 goto out;
3628                         }
3629                         target->rdma_cm.src_specified = true;
3630                         kfree(p);
3631                         break;
3632
3633                 case SRP_OPT_IP_DEST:
3634                         p = match_strdup(args);
3635                         if (!p) {
3636                                 ret = -ENOMEM;
3637                                 goto out;
3638                         }
3639                         ret = srp_parse_in(net, &target->rdma_cm.dst.ss, p);
3640                         if (ret < 0) {
3641                                 pr_warn("bad dest parameter '%s'\n", p);
3642                                 kfree(p);
3643                                 goto out;
3644                         }
3645                         target->using_rdma_cm = true;
3646                         kfree(p);
3647                         break;
3648
3649                 case SRP_OPT_MAX_SECT:
3650                         if (match_int(args, &token)) {
3651                                 pr_warn("bad max sect parameter '%s'\n", p);
3652                                 goto out;
3653                         }
3654                         target->scsi_host->max_sectors = token;
3655                         break;
3656
3657                 case SRP_OPT_QUEUE_SIZE:
3658                         if (match_int(args, &token) || token < 1) {
3659                                 pr_warn("bad queue_size parameter '%s'\n", p);
3660                                 goto out;
3661                         }
3662                         target->scsi_host->can_queue = token;
3663                         target->queue_size = token + SRP_RSP_SQ_SIZE +
3664                                              SRP_TSK_MGMT_SQ_SIZE;
3665                         if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3666                                 target->scsi_host->cmd_per_lun = token;
3667                         break;
3668
3669                 case SRP_OPT_MAX_CMD_PER_LUN:
3670                         if (match_int(args, &token) || token < 1) {
3671                                 pr_warn("bad max cmd_per_lun parameter '%s'\n",
3672                                         p);
3673                                 goto out;
3674                         }
3675                         target->scsi_host->cmd_per_lun = token;
3676                         break;
3677
3678                 case SRP_OPT_TARGET_CAN_QUEUE:
3679                         if (match_int(args, &token) || token < 1) {
3680                                 pr_warn("bad max target_can_queue parameter '%s'\n",
3681                                         p);
3682                                 goto out;
3683                         }
3684                         target->target_can_queue = token;
3685                         break;
3686
3687                 case SRP_OPT_IO_CLASS:
3688                         if (match_hex(args, &token)) {
3689                                 pr_warn("bad IO class parameter '%s'\n", p);
3690                                 goto out;
3691                         }
3692                         if (token != SRP_REV10_IB_IO_CLASS &&
3693                             token != SRP_REV16A_IB_IO_CLASS) {
3694                                 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
3695                                         token, SRP_REV10_IB_IO_CLASS,
3696                                         SRP_REV16A_IB_IO_CLASS);
3697                                 goto out;
3698                         }
3699                         target->io_class = token;
3700                         break;
3701
3702                 case SRP_OPT_INITIATOR_EXT:
3703                         p = match_strdup(args);
3704                         if (!p) {
3705                                 ret = -ENOMEM;
3706                                 goto out;
3707                         }
3708                         ret = kstrtoull(p, 16, &ull);
3709                         if (ret) {
3710                                 pr_warn("bad initiator_ext value '%s'\n", p);
3711                                 kfree(p);
3712                                 goto out;
3713                         }
3714                         target->initiator_ext = cpu_to_be64(ull);
3715                         kfree(p);
3716                         break;
3717
3718                 case SRP_OPT_CMD_SG_ENTRIES:
3719                         if (match_int(args, &token) || token < 1 || token > 255) {
3720                                 pr_warn("bad max cmd_sg_entries parameter '%s'\n",
3721                                         p);
3722                                 goto out;
3723                         }
3724                         target->cmd_sg_cnt = token;
3725                         break;
3726
3727                 case SRP_OPT_ALLOW_EXT_SG:
3728                         if (match_int(args, &token)) {
3729                                 pr_warn("bad allow_ext_sg parameter '%s'\n", p);
3730                                 goto out;
3731                         }
3732                         target->allow_ext_sg = !!token;
3733                         break;
3734
3735                 case SRP_OPT_SG_TABLESIZE:
3736                         if (match_int(args, &token) || token < 1 ||
3737                                         token > SG_MAX_SEGMENTS) {
3738                                 pr_warn("bad max sg_tablesize parameter '%s'\n",
3739                                         p);
3740                                 goto out;
3741                         }
3742                         target->sg_tablesize = token;
3743                         break;
3744
3745                 case SRP_OPT_COMP_VECTOR:
3746                         if (match_int(args, &token) || token < 0) {
3747                                 pr_warn("bad comp_vector parameter '%s'\n", p);
3748                                 goto out;
3749                         }
3750                         target->comp_vector = token;
3751                         break;
3752
3753                 case SRP_OPT_TL_RETRY_COUNT:
3754                         if (match_int(args, &token) || token < 2 || token > 7) {
3755                                 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
3756                                         p);
3757                                 goto out;
3758                         }
3759                         target->tl_retry_count = token;
3760                         break;
3761
3762                 default:
3763                         pr_warn("unknown parameter or missing value '%s' in target creation request\n",
3764                                 p);
3765                         goto out;
3766                 }
3767         }
3768
3769         for (i = 0; i < ARRAY_SIZE(srp_opt_mandatory); i++) {
3770                 if ((opt_mask & srp_opt_mandatory[i]) == srp_opt_mandatory[i]) {
3771                         ret = 0;
3772                         break;
3773                 }
3774         }
3775         if (ret)
3776                 pr_warn("target creation request is missing one or more parameters\n");
3777
3778         if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
3779             && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3780                 pr_warn("cmd_per_lun = %d > queue_size = %d\n",
3781                         target->scsi_host->cmd_per_lun,
3782                         target->scsi_host->can_queue);
3783
3784 out:
3785         kfree(options);
3786         return ret;
3787 }
3788
3789 static ssize_t srp_create_target(struct device *dev,
3790                                  struct device_attribute *attr,
3791                                  const char *buf, size_t count)
3792 {
3793         struct srp_host *host =
3794                 container_of(dev, struct srp_host, dev);
3795         struct Scsi_Host *target_host;
3796         struct srp_target_port *target;
3797         struct srp_rdma_ch *ch;
3798         struct srp_device *srp_dev = host->srp_dev;
3799         struct ib_device *ibdev = srp_dev->dev;
3800         int ret, node_idx, node, cpu, i;
3801         unsigned int max_sectors_per_mr, mr_per_cmd = 0;
3802         bool multich = false;
3803         uint32_t max_iu_len;
3804
3805         target_host = scsi_host_alloc(&srp_template,
3806                                       sizeof (struct srp_target_port));
3807         if (!target_host)
3808                 return -ENOMEM;
3809
3810         target_host->transportt  = ib_srp_transport_template;
3811         target_host->max_channel = 0;
3812         target_host->max_id      = 1;
3813         target_host->max_lun     = -1LL;
3814         target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
3815         target_host->max_segment_size = ib_dma_max_seg_size(ibdev);
3816
3817         target = host_to_target(target_host);
3818
3819         target->net             = kobj_ns_grab_current(KOBJ_NS_TYPE_NET);
3820         target->io_class        = SRP_REV16A_IB_IO_CLASS;
3821         target->scsi_host       = target_host;
3822         target->srp_host        = host;
3823         target->lkey            = host->srp_dev->pd->local_dma_lkey;
3824         target->global_rkey     = host->srp_dev->global_rkey;
3825         target->cmd_sg_cnt      = cmd_sg_entries;
3826         target->sg_tablesize    = indirect_sg_entries ? : cmd_sg_entries;
3827         target->allow_ext_sg    = allow_ext_sg;
3828         target->tl_retry_count  = 7;
3829         target->queue_size      = SRP_DEFAULT_QUEUE_SIZE;
3830
3831         /*
3832          * Avoid that the SCSI host can be removed by srp_remove_target()
3833          * before this function returns.
3834          */
3835         scsi_host_get(target->scsi_host);
3836
3837         ret = mutex_lock_interruptible(&host->add_target_mutex);
3838         if (ret < 0)
3839                 goto put;
3840
3841         ret = srp_parse_options(target->net, buf, target);
3842         if (ret)
3843                 goto out;
3844
3845         target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;
3846
3847         if (!srp_conn_unique(target->srp_host, target)) {
3848                 if (target->using_rdma_cm) {
3849                         shost_printk(KERN_INFO, target->scsi_host,
3850                                      PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;dest=%pIS\n",
3851                                      be64_to_cpu(target->id_ext),
3852                                      be64_to_cpu(target->ioc_guid),
3853                                      &target->rdma_cm.dst);
3854                 } else {
3855                         shost_printk(KERN_INFO, target->scsi_host,
3856                                      PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
3857                                      be64_to_cpu(target->id_ext),
3858                                      be64_to_cpu(target->ioc_guid),
3859                                      be64_to_cpu(target->initiator_ext));
3860                 }
3861                 ret = -EEXIST;
3862                 goto out;
3863         }
3864
3865         if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
3866             target->cmd_sg_cnt < target->sg_tablesize) {
3867                 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
3868                 target->sg_tablesize = target->cmd_sg_cnt;
3869         }
3870
3871         if (srp_dev->use_fast_reg || srp_dev->use_fmr) {
3872                 bool gaps_reg = (ibdev->attrs.device_cap_flags &
3873                                  IB_DEVICE_SG_GAPS_REG);
3874
3875                 max_sectors_per_mr = srp_dev->max_pages_per_mr <<
3876                                   (ilog2(srp_dev->mr_page_size) - 9);
3877                 if (!gaps_reg) {
3878                         /*
3879                          * FR and FMR can only map one HCA page per entry. If
3880                          * the start address is not aligned on a HCA page
3881                          * boundary two entries will be used for the head and
3882                          * the tail although these two entries combined
3883                          * contain at most one HCA page of data. Hence the "+
3884                          * 1" in the calculation below.
3885                          *
3886                          * The indirect data buffer descriptor is contiguous
3887                          * so the memory for that buffer will only be
3888                          * registered if register_always is true. Hence add
3889                          * one to mr_per_cmd if register_always has been set.
3890                          */
3891                         mr_per_cmd = register_always +
3892                                 (target->scsi_host->max_sectors + 1 +
3893                                  max_sectors_per_mr - 1) / max_sectors_per_mr;
3894                 } else {
3895                         mr_per_cmd = register_always +
3896                                 (target->sg_tablesize +
3897                                  srp_dev->max_pages_per_mr - 1) /
3898                                 srp_dev->max_pages_per_mr;
3899                 }
3900                 pr_debug("max_sectors = %u; max_pages_per_mr = %u; mr_page_size = %u; max_sectors_per_mr = %u; mr_per_cmd = %u\n",
3901                          target->scsi_host->max_sectors, srp_dev->max_pages_per_mr, srp_dev->mr_page_size,
3902                          max_sectors_per_mr, mr_per_cmd);
3903         }
3904
3905         target_host->sg_tablesize = target->sg_tablesize;
3906         target->mr_pool_size = target->scsi_host->can_queue * mr_per_cmd;
3907         target->mr_per_cmd = mr_per_cmd;
3908         target->indirect_size = target->sg_tablesize *
3909                                 sizeof (struct srp_direct_buf);
3910         max_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt, srp_use_imm_data);
3911
3912         INIT_WORK(&target->tl_err_work, srp_tl_err_work);
3913         INIT_WORK(&target->remove_work, srp_remove_work);
3914         spin_lock_init(&target->lock);
3915         ret = rdma_query_gid(ibdev, host->port, 0, &target->sgid);
3916         if (ret)
3917                 goto out;
3918
3919         ret = -ENOMEM;
3920         target->ch_count = max_t(unsigned, num_online_nodes(),
3921                                  min(ch_count ? :
3922                                      min(4 * num_online_nodes(),
3923                                          ibdev->num_comp_vectors),
3924                                      num_online_cpus()));
3925         target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
3926                              GFP_KERNEL);
3927         if (!target->ch)
3928                 goto out;
3929
3930         node_idx = 0;
3931         for_each_online_node(node) {
3932                 const int ch_start = (node_idx * target->ch_count /
3933                                       num_online_nodes());
3934                 const int ch_end = ((node_idx + 1) * target->ch_count /
3935                                     num_online_nodes());
3936                 const int cv_start = node_idx * ibdev->num_comp_vectors /
3937                                      num_online_nodes();
3938                 const int cv_end = (node_idx + 1) * ibdev->num_comp_vectors /
3939                                    num_online_nodes();
3940                 int cpu_idx = 0;
3941
3942                 for_each_online_cpu(cpu) {
3943                         if (cpu_to_node(cpu) != node)
3944                                 continue;
3945                         if (ch_start + cpu_idx >= ch_end)
3946                                 continue;
3947                         ch = &target->ch[ch_start + cpu_idx];
3948                         ch->target = target;
3949                         ch->comp_vector = cv_start == cv_end ? cv_start :
3950                                 cv_start + cpu_idx % (cv_end - cv_start);
3951                         spin_lock_init(&ch->lock);
3952                         INIT_LIST_HEAD(&ch->free_tx);
3953                         ret = srp_new_cm_id(ch);
3954                         if (ret)
3955                                 goto err_disconnect;
3956
3957                         ret = srp_create_ch_ib(ch);
3958                         if (ret)
3959                                 goto err_disconnect;
3960
3961                         ret = srp_alloc_req_data(ch);
3962                         if (ret)
3963                                 goto err_disconnect;
3964
3965                         ret = srp_connect_ch(ch, max_iu_len, multich);
3966                         if (ret) {
3967                                 char dst[64];
3968
3969                                 if (target->using_rdma_cm)
3970                                         snprintf(dst, sizeof(dst), "%pIS",
3971                                                  &target->rdma_cm.dst);
3972                                 else
3973                                         snprintf(dst, sizeof(dst), "%pI6",
3974                                                  target->ib_cm.orig_dgid.raw);
3975                                 shost_printk(KERN_ERR, target->scsi_host,
3976                                              PFX "Connection %d/%d to %s failed\n",
3977                                              ch_start + cpu_idx,
3978                                              target->ch_count, dst);
3979                                 if (node_idx == 0 && cpu_idx == 0) {
3980                                         goto free_ch;
3981                                 } else {
3982                                         srp_free_ch_ib(target, ch);
3983                                         srp_free_req_data(target, ch);
3984                                         target->ch_count = ch - target->ch;
3985                                         goto connected;
3986                                 }
3987                         }
3988
3989                         multich = true;
3990                         cpu_idx++;
3991                 }
3992                 node_idx++;
3993         }
3994
3995 connected:
3996         target->scsi_host->nr_hw_queues = target->ch_count;
3997
3998         ret = srp_add_target(host, target);
3999         if (ret)
4000                 goto err_disconnect;
4001
4002         if (target->state != SRP_TARGET_REMOVED) {
4003                 if (target->using_rdma_cm) {
4004                         shost_printk(KERN_DEBUG, target->scsi_host, PFX
4005                                      "new target: id_ext %016llx ioc_guid %016llx sgid %pI6 dest %pIS\n",
4006                                      be64_to_cpu(target->id_ext),
4007                                      be64_to_cpu(target->ioc_guid),
4008                                      target->sgid.raw, &target->rdma_cm.dst);
4009                 } else {
4010                         shost_printk(KERN_DEBUG, target->scsi_host, PFX
4011                                      "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
4012                                      be64_to_cpu(target->id_ext),
4013                                      be64_to_cpu(target->ioc_guid),
4014                                      be16_to_cpu(target->ib_cm.pkey),
4015                                      be64_to_cpu(target->ib_cm.service_id),
4016                                      target->sgid.raw,
4017                                      target->ib_cm.orig_dgid.raw);
4018                 }
4019         }
4020
4021         ret = count;
4022
4023 out:
4024         mutex_unlock(&host->add_target_mutex);
4025
4026 put:
4027         scsi_host_put(target->scsi_host);
4028         if (ret < 0) {
4029                 /*
4030                  * If a call to srp_remove_target() has not been scheduled,
4031                  * drop the network namespace reference now that was obtained
4032                  * earlier in this function.
4033                  */
4034                 if (target->state != SRP_TARGET_REMOVED)
4035                         kobj_ns_drop(KOBJ_NS_TYPE_NET, target->net);
4036                 scsi_host_put(target->scsi_host);
4037         }
4038
4039         return ret;
4040
4041 err_disconnect:
4042         srp_disconnect_target(target);
4043
4044 free_ch:
4045         for (i = 0; i < target->ch_count; i++) {
4046                 ch = &target->ch[i];
4047                 srp_free_ch_ib(target, ch);
4048                 srp_free_req_data(target, ch);
4049         }
4050
4051         kfree(target->ch);
4052         goto out;
4053 }
4054
4055 static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
4056
4057 static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
4058                           char *buf)
4059 {
4060         struct srp_host *host = container_of(dev, struct srp_host, dev);
4061
4062         return sprintf(buf, "%s\n", dev_name(&host->srp_dev->dev->dev));
4063 }
4064
4065 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
4066
4067 static ssize_t show_port(struct device *dev, struct device_attribute *attr,
4068                          char *buf)
4069 {
4070         struct srp_host *host = container_of(dev, struct srp_host, dev);
4071
4072         return sprintf(buf, "%d\n", host->port);
4073 }
4074
4075 static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
4076
4077 static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
4078 {
4079         struct srp_host *host;
4080
4081         host = kzalloc(sizeof *host, GFP_KERNEL);
4082         if (!host)
4083                 return NULL;
4084
4085         INIT_LIST_HEAD(&host->target_list);
4086         spin_lock_init(&host->target_lock);
4087         init_completion(&host->released);
4088         mutex_init(&host->add_target_mutex);
4089         host->srp_dev = device;
4090         host->port = port;
4091
4092         host->dev.class = &srp_class;
4093         host->dev.parent = device->dev->dev.parent;
4094         dev_set_name(&host->dev, "srp-%s-%d", dev_name(&device->dev->dev),
4095                      port);
4096
4097         if (device_register(&host->dev))
4098                 goto free_host;
4099         if (device_create_file(&host->dev, &dev_attr_add_target))
4100                 goto err_class;
4101         if (device_create_file(&host->dev, &dev_attr_ibdev))
4102                 goto err_class;
4103         if (device_create_file(&host->dev, &dev_attr_port))
4104                 goto err_class;
4105
4106         return host;
4107
4108 err_class:
4109         device_unregister(&host->dev);
4110
4111 free_host:
4112         kfree(host);
4113
4114         return NULL;
4115 }
4116
4117 static void srp_rename_dev(struct ib_device *device, void *client_data)
4118 {
4119         struct srp_device *srp_dev = client_data;
4120         struct srp_host *host, *tmp_host;
4121
4122         list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
4123                 char name[IB_DEVICE_NAME_MAX + 8];
4124
4125                 snprintf(name, sizeof(name), "srp-%s-%d",
4126                          dev_name(&device->dev), host->port);
4127                 device_rename(&host->dev, name);
4128         }
4129 }
4130
4131 static void srp_add_one(struct ib_device *device)
4132 {
4133         struct srp_device *srp_dev;
4134         struct ib_device_attr *attr = &device->attrs;
4135         struct srp_host *host;
4136         int mr_page_shift;
4137         unsigned int p;
4138         u64 max_pages_per_mr;
4139         unsigned int flags = 0;
4140
4141         srp_dev = kzalloc(sizeof(*srp_dev), GFP_KERNEL);
4142         if (!srp_dev)
4143                 return;
4144
4145         /*
4146          * Use the smallest page size supported by the HCA, down to a
4147          * minimum of 4096 bytes. We're unlikely to build large sglists
4148          * out of smaller entries.
4149          */
4150         mr_page_shift           = max(12, ffs(attr->page_size_cap) - 1);
4151         srp_dev->mr_page_size   = 1 << mr_page_shift;
4152         srp_dev->mr_page_mask   = ~((u64) srp_dev->mr_page_size - 1);
4153         max_pages_per_mr        = attr->max_mr_size;
4154         do_div(max_pages_per_mr, srp_dev->mr_page_size);
4155         pr_debug("%s: %llu / %u = %llu <> %u\n", __func__,
4156                  attr->max_mr_size, srp_dev->mr_page_size,
4157                  max_pages_per_mr, SRP_MAX_PAGES_PER_MR);
4158         srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
4159                                           max_pages_per_mr);
4160
4161         srp_dev->has_fmr = (device->ops.alloc_fmr &&
4162                             device->ops.dealloc_fmr &&
4163                             device->ops.map_phys_fmr &&
4164                             device->ops.unmap_fmr);
4165         srp_dev->has_fr = (attr->device_cap_flags &
4166                            IB_DEVICE_MEM_MGT_EXTENSIONS);
4167         if (!never_register && !srp_dev->has_fmr && !srp_dev->has_fr) {
4168                 dev_warn(&device->dev, "neither FMR nor FR is supported\n");
4169         } else if (!never_register &&
4170                    attr->max_mr_size >= 2 * srp_dev->mr_page_size) {
4171                 srp_dev->use_fast_reg = (srp_dev->has_fr &&
4172                                          (!srp_dev->has_fmr || prefer_fr));
4173                 srp_dev->use_fmr = !srp_dev->use_fast_reg && srp_dev->has_fmr;
4174         }
4175
4176         if (never_register || !register_always ||
4177             (!srp_dev->has_fmr && !srp_dev->has_fr))
4178                 flags |= IB_PD_UNSAFE_GLOBAL_RKEY;
4179
4180         if (srp_dev->use_fast_reg) {
4181                 srp_dev->max_pages_per_mr =
4182                         min_t(u32, srp_dev->max_pages_per_mr,
4183                               attr->max_fast_reg_page_list_len);
4184         }
4185         srp_dev->mr_max_size    = srp_dev->mr_page_size *
4186                                    srp_dev->max_pages_per_mr;
4187         pr_debug("%s: mr_page_shift = %d, device->max_mr_size = %#llx, device->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
4188                  dev_name(&device->dev), mr_page_shift, attr->max_mr_size,
4189                  attr->max_fast_reg_page_list_len,
4190                  srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
4191
4192         INIT_LIST_HEAD(&srp_dev->dev_list);
4193
4194         srp_dev->dev = device;
4195         srp_dev->pd  = ib_alloc_pd(device, flags);
4196         if (IS_ERR(srp_dev->pd))
4197                 goto free_dev;
4198
4199         if (flags & IB_PD_UNSAFE_GLOBAL_RKEY) {
4200                 srp_dev->global_rkey = srp_dev->pd->unsafe_global_rkey;
4201                 WARN_ON_ONCE(srp_dev->global_rkey == 0);
4202         }
4203
4204         rdma_for_each_port (device, p) {
4205                 host = srp_add_port(srp_dev, p);
4206                 if (host)
4207                         list_add_tail(&host->list, &srp_dev->dev_list);
4208         }
4209
4210         ib_set_client_data(device, &srp_client, srp_dev);
4211         return;
4212
4213 free_dev:
4214         kfree(srp_dev);
4215 }
4216
4217 static void srp_remove_one(struct ib_device *device, void *client_data)
4218 {
4219         struct srp_device *srp_dev;
4220         struct srp_host *host, *tmp_host;
4221         struct srp_target_port *target;
4222
4223         srp_dev = client_data;
4224         if (!srp_dev)
4225                 return;
4226
4227         list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
4228                 device_unregister(&host->dev);
4229                 /*
4230                  * Wait for the sysfs entry to go away, so that no new
4231                  * target ports can be created.
4232                  */
4233                 wait_for_completion(&host->released);
4234
4235                 /*
4236                  * Remove all target ports.
4237                  */
4238                 spin_lock(&host->target_lock);
4239                 list_for_each_entry(target, &host->target_list, list)
4240                         srp_queue_remove_work(target);
4241                 spin_unlock(&host->target_lock);
4242
4243                 /*
4244                  * Wait for tl_err and target port removal tasks.
4245                  */
4246                 flush_workqueue(system_long_wq);
4247                 flush_workqueue(srp_remove_wq);
4248
4249                 kfree(host);
4250         }
4251
4252         ib_dealloc_pd(srp_dev->pd);
4253
4254         kfree(srp_dev);
4255 }
4256
4257 static struct srp_function_template ib_srp_transport_functions = {
4258         .has_rport_state         = true,
4259         .reset_timer_if_blocked  = true,
4260         .reconnect_delay         = &srp_reconnect_delay,
4261         .fast_io_fail_tmo        = &srp_fast_io_fail_tmo,
4262         .dev_loss_tmo            = &srp_dev_loss_tmo,
4263         .reconnect               = srp_rport_reconnect,
4264         .rport_delete            = srp_rport_delete,
4265         .terminate_rport_io      = srp_terminate_io,
4266 };
4267
4268 static int __init srp_init_module(void)
4269 {
4270         int ret;
4271
4272         BUILD_BUG_ON(sizeof(struct srp_imm_buf) != 4);
4273         BUILD_BUG_ON(sizeof(struct srp_login_req) != 64);
4274         BUILD_BUG_ON(sizeof(struct srp_login_req_rdma) != 56);
4275         BUILD_BUG_ON(sizeof(struct srp_cmd) != 48);
4276
4277         if (srp_sg_tablesize) {
4278                 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
4279                 if (!cmd_sg_entries)
4280                         cmd_sg_entries = srp_sg_tablesize;
4281         }
4282
4283         if (!cmd_sg_entries)
4284                 cmd_sg_entries = SRP_DEF_SG_TABLESIZE;
4285
4286         if (cmd_sg_entries > 255) {
4287                 pr_warn("Clamping cmd_sg_entries to 255\n");
4288                 cmd_sg_entries = 255;
4289         }
4290
4291         if (!indirect_sg_entries)
4292                 indirect_sg_entries = cmd_sg_entries;
4293         else if (indirect_sg_entries < cmd_sg_entries) {
4294                 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
4295                         cmd_sg_entries);
4296                 indirect_sg_entries = cmd_sg_entries;
4297         }
4298
4299         if (indirect_sg_entries > SG_MAX_SEGMENTS) {
4300                 pr_warn("Clamping indirect_sg_entries to %u\n",
4301                         SG_MAX_SEGMENTS);
4302                 indirect_sg_entries = SG_MAX_SEGMENTS;
4303         }
4304
4305         srp_remove_wq = create_workqueue("srp_remove");
4306         if (!srp_remove_wq) {
4307                 ret = -ENOMEM;
4308                 goto out;
4309         }
4310
4311         ret = -ENOMEM;
4312         ib_srp_transport_template =
4313                 srp_attach_transport(&ib_srp_transport_functions);
4314         if (!ib_srp_transport_template)
4315                 goto destroy_wq;
4316
4317         ret = class_register(&srp_class);
4318         if (ret) {
4319                 pr_err("couldn't register class infiniband_srp\n");
4320                 goto release_tr;
4321         }
4322
4323         ib_sa_register_client(&srp_sa_client);
4324
4325         ret = ib_register_client(&srp_client);
4326         if (ret) {
4327                 pr_err("couldn't register IB client\n");
4328                 goto unreg_sa;
4329         }
4330
4331 out:
4332         return ret;
4333
4334 unreg_sa:
4335         ib_sa_unregister_client(&srp_sa_client);
4336         class_unregister(&srp_class);
4337
4338 release_tr:
4339         srp_release_transport(ib_srp_transport_template);
4340
4341 destroy_wq:
4342         destroy_workqueue(srp_remove_wq);
4343         goto out;
4344 }
4345
4346 static void __exit srp_cleanup_module(void)
4347 {
4348         ib_unregister_client(&srp_client);
4349         ib_sa_unregister_client(&srp_sa_client);
4350         class_unregister(&srp_class);
4351         srp_release_transport(ib_srp_transport_template);
4352         destroy_workqueue(srp_remove_wq);
4353 }
4354
4355 module_init(srp_init_module);
4356 module_exit(srp_cleanup_module);