drivers/vhost/scsi.c

   1 /*******************************************************************************
   2  * Vhost kernel TCM fabric driver for virtio SCSI initiators
   3  *
   4  * (C) Copyright 2010-2013 Datera, Inc.
   5  * (C) Copyright 2010-2012 IBM Corp.
   6  *
   7  * Licensed to the Linux Foundation under the General Public License (GPL) version 2.
   8  *
   9  * Authors: Nicholas A. Bellinger <nab@daterainc.com>
  10  *          Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License as published by
  14  * the Free Software Foundation; either version 2 of the License, or
  15  * (at your option) any later version.
  16  *
  17  * This program is distributed in the hope that it will be useful,
  18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20  * GNU General Public License for more details.
  21  *
  22  ****************************************************************************/
  23
  24 #include <linux/module.h>
  25 #include <linux/moduleparam.h>
  26 #include <generated/utsrelease.h>
  27 #include <linux/utsname.h>
  28 #include <linux/init.h>
  29 #include <linux/slab.h>
  30 #include <linux/kthread.h>
  31 #include <linux/types.h>
  32 #include <linux/string.h>
  33 #include <linux/configfs.h>
  34 #include <linux/ctype.h>
  35 #include <linux/compat.h>
  36 #include <linux/eventfd.h>
  37 #include <linux/fs.h>
  38 #include <linux/vmalloc.h>
  39 #include <linux/miscdevice.h>
  40 #include <asm/unaligned.h>
  41 #include <scsi/scsi_common.h>
  42 #include <scsi/scsi_proto.h>
  43 #include <target/target_core_base.h>
  44 #include <target/target_core_fabric.h>
  45 #include <linux/vhost.h>
  46 #include <linux/virtio_scsi.h>
  47 #include <linux/llist.h>
  48 #include <linux/bitmap.h>
  49
  50 #include "vhost.h"
  51
  52 #define VHOST_SCSI_VERSION  "v0.1"
  53 #define VHOST_SCSI_NAMELEN 256
  54 #define VHOST_SCSI_MAX_CDB_SIZE 32
  55 #define VHOST_SCSI_DEFAULT_TAGS 256
  56 #define VHOST_SCSI_PREALLOC_SGLS 2048
  57 #define VHOST_SCSI_PREALLOC_UPAGES 2048
  58 #define VHOST_SCSI_PREALLOC_PROT_SGLS 2048
  59
  60 struct vhost_scsi_inflight {
  61         /* Wait for the flush operation to finish */
  62         struct completion comp;
  63         /* Refcount for the inflight reqs */
  64         struct kref kref;
  65 };
  66
  67 struct vhost_scsi_cmd {
  68         /* Descriptor from vhost_get_vq_desc() for virt_queue segment */
  69         int tvc_vq_desc;
  70         /* virtio-scsi initiator task attribute */
  71         int tvc_task_attr;
  72         /* virtio-scsi response incoming iovecs */
  73         int tvc_in_iovs;
  74         /* virtio-scsi initiator data direction */
  75         enum dma_data_direction tvc_data_direction;
  76         /* Expected data transfer length from virtio-scsi header */
  77         u32 tvc_exp_data_len;
  78         /* The Tag from include/linux/virtio_scsi.h:struct virtio_scsi_cmd_req */
  79         u64 tvc_tag;
  80         /* The number of scatterlists associated with this cmd */
  81         u32 tvc_sgl_count;
  82         u32 tvc_prot_sgl_count;
  83         /* Saved unpacked SCSI LUN for vhost_scsi_submission_work() */
  84         u32 tvc_lun;
  85         /* Pointer to the SGL formatted memory from virtio-scsi */
  86         struct scatterlist *tvc_sgl;
  87         struct scatterlist *tvc_prot_sgl;
  88         struct page **tvc_upages;
  89         /* Pointer to response header iovec */
  90         struct iovec tvc_resp_iov;
  91         /* Pointer to vhost_scsi for our device */
  92         struct vhost_scsi *tvc_vhost;
  93         /* Pointer to vhost_virtqueue for the cmd */
  94         struct vhost_virtqueue *tvc_vq;
  95         /* Pointer to vhost nexus memory */
  96         struct vhost_scsi_nexus *tvc_nexus;
  97         /* The TCM I/O descriptor that is accessed via container_of() */
  98         struct se_cmd tvc_se_cmd;
  99         /* work item used for cmwq dispatch to vhost_scsi_submission_work() */
 100         struct work_struct work;
 101         /* Copy of the incoming SCSI command descriptor block (CDB) */
 102         unsigned char tvc_cdb[VHOST_SCSI_MAX_CDB_SIZE];
 103         /* Sense buffer that will be mapped into outgoing status */
 104         unsigned char tvc_sense_buf[TRANSPORT_SENSE_BUFFER];
 105         /* Completed commands list, serviced from vhost worker thread */
 106         struct llist_node tvc_completion_list;
 107         /* Used to track inflight cmd */
 108         struct vhost_scsi_inflight *inflight;
 109 };
 110
 111 struct vhost_scsi_nexus {
 112         /* Pointer to TCM session for I_T Nexus */
 113         struct se_session *tvn_se_sess;
 114 };
 115
 116 struct vhost_scsi_tpg {
 117         /* Vhost port target portal group tag for TCM */
 118         u16 tport_tpgt;
 119         /* Used to track number of TPG Port/Lun Links wrt to explict I_T Nexus shutdown */
 120         int tv_tpg_port_count;
 121         /* Used for vhost_scsi device reference to tpg_nexus, protected by tv_tpg_mutex */
 122         int tv_tpg_vhost_count;
 123         /* Used for enabling T10-PI with legacy devices */
 124         int tv_fabric_prot_type;
 125         /* list for vhost_scsi_list */
 126         struct list_head tv_tpg_list;
 127         /* Used to protect access for tpg_nexus */
 128         struct mutex tv_tpg_mutex;
 129         /* Pointer to the TCM VHost I_T Nexus for this TPG endpoint */
 130         struct vhost_scsi_nexus *tpg_nexus;
 131         /* Pointer back to vhost_scsi_tport */
 132         struct vhost_scsi_tport *tport;
 133         /* Returned by vhost_scsi_make_tpg() */
 134         struct se_portal_group se_tpg;
 135         /* Pointer back to vhost_scsi, protected by tv_tpg_mutex */
 136         struct vhost_scsi *vhost_scsi;
 137 };
 138
 139 struct vhost_scsi_tport {
 140         /* SCSI protocol the tport is providing */
 141         u8 tport_proto_id;
 142         /* Binary World Wide unique Port Name for Vhost Target port */
 143         u64 tport_wwpn;
 144         /* ASCII formatted WWPN for Vhost Target port */
 145         char tport_name[VHOST_SCSI_NAMELEN];
 146         /* Returned by vhost_scsi_make_tport() */
 147         struct se_wwn tport_wwn;
 148 };
 149
 150 struct vhost_scsi_evt {
 151         /* event to be sent to guest */
 152         struct virtio_scsi_event event;
 153         /* event list, serviced from vhost worker thread */
 154         struct llist_node list;
 155 };
 156
 157 enum {
 158         VHOST_SCSI_VQ_CTL = 0,
 159         VHOST_SCSI_VQ_EVT = 1,
 160         VHOST_SCSI_VQ_IO = 2,
 161 };
 162
 163 /* Note: can't set VIRTIO_F_VERSION_1 yet, since that implies ANY_LAYOUT. */
 164 enum {
 165         VHOST_SCSI_FEATURES = VHOST_FEATURES | (1ULL << VIRTIO_SCSI_F_HOTPLUG) |
 166                                                (1ULL << VIRTIO_SCSI_F_T10_PI)
 167 };
 168
 169 #define VHOST_SCSI_MAX_TARGET   256
 170 #define VHOST_SCSI_MAX_VQ       128
 171 #define VHOST_SCSI_MAX_EVENT    128
 172
 173 struct vhost_scsi_virtqueue {
 174         struct vhost_virtqueue vq;
 175         /*
 176          * Reference counting for inflight reqs, used for flush operation. At
 177          * each time, one reference tracks new commands submitted, while we
 178          * wait for another one to reach 0.
 179          */
 180         struct vhost_scsi_inflight inflights[2];
 181         /*
 182          * Indicate current inflight in use, protected by vq->mutex.
 183          * Writers must also take dev mutex and flush under it.
 184          */
 185         int inflight_idx;
 186 };
 187
 188 struct vhost_scsi {
 189         /* Protected by vhost_scsi->dev.mutex */
 190         struct vhost_scsi_tpg **vs_tpg;
 191         char vs_vhost_wwpn[TRANSPORT_IQN_LEN];
 192
 193         struct vhost_dev dev;
 194         struct vhost_scsi_virtqueue vqs[VHOST_SCSI_MAX_VQ];
 195
 196         struct vhost_work vs_completion_work; /* cmd completion work item */
 197         struct llist_head vs_completion_list; /* cmd completion queue */
 198
 199         struct vhost_work vs_event_work; /* evt injection work item */
 200         struct llist_head vs_event_list; /* evt injection queue */
 201
 202         bool vs_events_missed; /* any missed events, protected by vq->mutex */
 203         int vs_events_nr; /* num of pending events, protected by vq->mutex */
 204 };
 205
 206 /*
 207  * Context for processing request and control queue operations.
 208  */
 209 struct vhost_scsi_ctx {
 210         int head;
 211         unsigned int out, in;
 212         size_t req_size, rsp_size;
 213         size_t out_size, in_size;
 214         u8 *target, *lunp;
 215         void *req;
 216         struct iov_iter out_iter;
 217 };
 218
 219 static struct workqueue_struct *vhost_scsi_workqueue;
 220
 221 /* Global spinlock to protect vhost_scsi TPG list for vhost IOCTL access */
 222 static DEFINE_MUTEX(vhost_scsi_mutex);
 223 static LIST_HEAD(vhost_scsi_list);
 224
 225 static void vhost_scsi_done_inflight(struct kref *kref)
 226 {
 227         struct vhost_scsi_inflight *inflight;
 228
 229         inflight = container_of(kref, struct vhost_scsi_inflight, kref);
 230         complete(&inflight->comp);
 231 }
 232
 233 static void vhost_scsi_init_inflight(struct vhost_scsi *vs,
 234                                     struct vhost_scsi_inflight *old_inflight[])
 235 {
 236         struct vhost_scsi_inflight *new_inflight;
 237         struct vhost_virtqueue *vq;
 238         int idx, i;
 239
 240         for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
 241                 vq = &vs->vqs[i].vq;
 242
 243                 mutex_lock(&vq->mutex);
 244
 245                 /* store old infight */
 246                 idx = vs->vqs[i].inflight_idx;
 247                 if (old_inflight)
 248                         old_inflight[i] = &vs->vqs[i].inflights[idx];
 249
 250                 /* setup new infight */
 251                 vs->vqs[i].inflight_idx = idx ^ 1;
 252                 new_inflight = &vs->vqs[i].inflights[idx ^ 1];
 253                 kref_init(&new_inflight->kref);
 254                 init_completion(&new_inflight->comp);
 255
 256                 mutex_unlock(&vq->mutex);
 257         }
 258 }
 259
 260 static struct vhost_scsi_inflight *
 261 vhost_scsi_get_inflight(struct vhost_virtqueue *vq)
 262 {
 263         struct vhost_scsi_inflight *inflight;
 264         struct vhost_scsi_virtqueue *svq;
 265
 266         svq = container_of(vq, struct vhost_scsi_virtqueue, vq);
 267         inflight = &svq->inflights[svq->inflight_idx];
 268         kref_get(&inflight->kref);
 269
 270         return inflight;
 271 }
 272
 273 static void vhost_scsi_put_inflight(struct vhost_scsi_inflight *inflight)
 274 {
 275         kref_put(&inflight->kref, vhost_scsi_done_inflight);
 276 }
 277
 278 static int vhost_scsi_check_true(struct se_portal_group *se_tpg)
 279 {
 280         return 1;
 281 }
 282
 283 static int vhost_scsi_check_false(struct se_portal_group *se_tpg)
 284 {
 285         return 0;
 286 }
 287
 288 static char *vhost_scsi_get_fabric_wwn(struct se_portal_group *se_tpg)
 289 {
 290         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
 291                                 struct vhost_scsi_tpg, se_tpg);
 292         struct vhost_scsi_tport *tport = tpg->tport;
 293
 294         return &tport->tport_name[0];
 295 }
 296
 297 static u16 vhost_scsi_get_tpgt(struct se_portal_group *se_tpg)
 298 {
 299         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
 300                                 struct vhost_scsi_tpg, se_tpg);
 301         return tpg->tport_tpgt;
 302 }
 303
 304 static int vhost_scsi_check_prot_fabric_only(struct se_portal_group *se_tpg)
 305 {
 306         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
 307                                 struct vhost_scsi_tpg, se_tpg);
 308
 309         return tpg->tv_fabric_prot_type;
 310 }
 311
 312 static u32 vhost_scsi_tpg_get_inst_index(struct se_portal_group *se_tpg)
 313 {
 314         return 1;
 315 }
 316
 317 static void vhost_scsi_release_cmd(struct se_cmd *se_cmd)
 318 {
 319         struct vhost_scsi_cmd *tv_cmd = container_of(se_cmd,
 320                                 struct vhost_scsi_cmd, tvc_se_cmd);
 321         struct se_session *se_sess = tv_cmd->tvc_nexus->tvn_se_sess;
 322         int i;
 323
 324         if (tv_cmd->tvc_sgl_count) {
 325                 for (i = 0; i < tv_cmd->tvc_sgl_count; i++)
 326                         put_page(sg_page(&tv_cmd->tvc_sgl[i]));
 327         }
 328         if (tv_cmd->tvc_prot_sgl_count) {
 329                 for (i = 0; i < tv_cmd->tvc_prot_sgl_count; i++)
 330                         put_page(sg_page(&tv_cmd->tvc_prot_sgl[i]));
 331         }
 332
 333         vhost_scsi_put_inflight(tv_cmd->inflight);
 334         target_free_tag(se_sess, se_cmd);
 335 }
 336
 337 static u32 vhost_scsi_sess_get_index(struct se_session *se_sess)
 338 {
 339         return 0;
 340 }
 341
 342 static int vhost_scsi_write_pending(struct se_cmd *se_cmd)
 343 {
 344         /* Go ahead and process the write immediately */
 345         target_execute_cmd(se_cmd);
 346         return 0;
 347 }
 348
 349 static void vhost_scsi_set_default_node_attrs(struct se_node_acl *nacl)
 350 {
 351         return;
 352 }
 353
 354 static int vhost_scsi_get_cmd_state(struct se_cmd *se_cmd)
 355 {
 356         return 0;
 357 }
 358
 359 static void vhost_scsi_complete_cmd(struct vhost_scsi_cmd *cmd)
 360 {
 361         struct vhost_scsi *vs = cmd->tvc_vhost;
 362
 363         llist_add(&cmd->tvc_completion_list, &vs->vs_completion_list);
 364
 365         vhost_work_queue(&vs->dev, &vs->vs_completion_work);
 366 }
 367
 368 static int vhost_scsi_queue_data_in(struct se_cmd *se_cmd)
 369 {
 370         struct vhost_scsi_cmd *cmd = container_of(se_cmd,
 371                                 struct vhost_scsi_cmd, tvc_se_cmd);
 372         vhost_scsi_complete_cmd(cmd);
 373         return 0;
 374 }
 375
 376 static int vhost_scsi_queue_status(struct se_cmd *se_cmd)
 377 {
 378         struct vhost_scsi_cmd *cmd = container_of(se_cmd,
 379                                 struct vhost_scsi_cmd, tvc_se_cmd);
 380         vhost_scsi_complete_cmd(cmd);
 381         return 0;
 382 }
 383
 384 static void vhost_scsi_queue_tm_rsp(struct se_cmd *se_cmd)
 385 {
 386         return;
 387 }
 388
 389 static void vhost_scsi_aborted_task(struct se_cmd *se_cmd)
 390 {
 391         return;
 392 }
 393
 394 static void vhost_scsi_free_evt(struct vhost_scsi *vs, struct vhost_scsi_evt *evt)
 395 {
 396         vs->vs_events_nr--;
 397         kfree(evt);
 398 }
 399
 400 static struct vhost_scsi_evt *
 401 vhost_scsi_allocate_evt(struct vhost_scsi *vs,
 402                        u32 event, u32 reason)
 403 {
 404         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
 405         struct vhost_scsi_evt *evt;
 406
 407         if (vs->vs_events_nr > VHOST_SCSI_MAX_EVENT) {
 408                 vs->vs_events_missed = true;
 409                 return NULL;
 410         }
 411
 412         evt = kzalloc(sizeof(*evt), GFP_KERNEL);
 413         if (!evt) {
 414                 vq_err(vq, "Failed to allocate vhost_scsi_evt\n");
 415                 vs->vs_events_missed = true;
 416                 return NULL;
 417         }
 418
 419         evt->event.event = cpu_to_vhost32(vq, event);
 420         evt->event.reason = cpu_to_vhost32(vq, reason);
 421         vs->vs_events_nr++;
 422
 423         return evt;
 424 }
 425
 426 static void vhost_scsi_free_cmd(struct vhost_scsi_cmd *cmd)
 427 {
 428         struct se_cmd *se_cmd = &cmd->tvc_se_cmd;
 429
 430         /* TODO locking against target/backend threads? */
 431         transport_generic_free_cmd(se_cmd, 0);
 432
 433 }
 434
 435 static int vhost_scsi_check_stop_free(struct se_cmd *se_cmd)
 436 {
 437         return target_put_sess_cmd(se_cmd);
 438 }
 439
 440 static void
 441 vhost_scsi_do_evt_work(struct vhost_scsi *vs, struct vhost_scsi_evt *evt)
 442 {
 443         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
 444         struct virtio_scsi_event *event = &evt->event;
 445         struct virtio_scsi_event __user *eventp;
 446         unsigned out, in;
 447         int head, ret;
 448
 449         if (!vq->private_data) {
 450                 vs->vs_events_missed = true;
 451                 return;
 452         }
 453
 454 again:
 455         vhost_disable_notify(&vs->dev, vq);
 456         head = vhost_get_vq_desc(vq, vq->iov,
 457                         ARRAY_SIZE(vq->iov), &out, &in,
 458                         NULL, NULL);
 459         if (head < 0) {
 460                 vs->vs_events_missed = true;
 461                 return;
 462         }
 463         if (head == vq->num) {
 464                 if (vhost_enable_notify(&vs->dev, vq))
 465                         goto again;
 466                 vs->vs_events_missed = true;
 467                 return;
 468         }
 469
 470         if ((vq->iov[out].iov_len != sizeof(struct virtio_scsi_event))) {
 471                 vq_err(vq, "Expecting virtio_scsi_event, got %zu bytes\n",
 472                                 vq->iov[out].iov_len);
 473                 vs->vs_events_missed = true;
 474                 return;
 475         }
 476
 477         if (vs->vs_events_missed) {
 478                 event->event |= cpu_to_vhost32(vq, VIRTIO_SCSI_T_EVENTS_MISSED);
 479                 vs->vs_events_missed = false;
 480         }
 481
 482         eventp = vq->iov[out].iov_base;
 483         ret = __copy_to_user(eventp, event, sizeof(*event));
 484         if (!ret)
 485                 vhost_add_used_and_signal(&vs->dev, vq, head, 0);
 486         else
 487                 vq_err(vq, "Faulted on vhost_scsi_send_event\n");
 488 }
 489
 490 static void vhost_scsi_evt_work(struct vhost_work *work)
 491 {
 492         struct vhost_scsi *vs = container_of(work, struct vhost_scsi,
 493                                         vs_event_work);
 494         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
 495         struct vhost_scsi_evt *evt, *t;
 496         struct llist_node *llnode;
 497
 498         mutex_lock(&vq->mutex);
 499         llnode = llist_del_all(&vs->vs_event_list);
 500         llist_for_each_entry_safe(evt, t, llnode, list) {
 501                 vhost_scsi_do_evt_work(vs, evt);
 502                 vhost_scsi_free_evt(vs, evt);
 503         }
 504         mutex_unlock(&vq->mutex);
 505 }
 506
 507 /* Fill in status and signal that we are done processing this command
 508  *
 509  * This is scheduled in the vhost work queue so we are called with the owner
 510  * process mm and can access the vring.
 511  */
 512 static void vhost_scsi_complete_cmd_work(struct vhost_work *work)
 513 {
 514         struct vhost_scsi *vs = container_of(work, struct vhost_scsi,
 515                                         vs_completion_work);
 516         DECLARE_BITMAP(signal, VHOST_SCSI_MAX_VQ);
 517         struct virtio_scsi_cmd_resp v_rsp;
 518         struct vhost_scsi_cmd *cmd, *t;
 519         struct llist_node *llnode;
 520         struct se_cmd *se_cmd;
 521         struct iov_iter iov_iter;
 522         int ret, vq;
 523
 524         bitmap_zero(signal, VHOST_SCSI_MAX_VQ);
 525         llnode = llist_del_all(&vs->vs_completion_list);
 526         llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list) {
 527                 se_cmd = &cmd->tvc_se_cmd;
 528
 529                 pr_debug("%s tv_cmd %p resid %u status %#02x\n", __func__,
 530                         cmd, se_cmd->residual_count, se_cmd->scsi_status);
 531
 532                 memset(&v_rsp, 0, sizeof(v_rsp));
 533                 v_rsp.resid = cpu_to_vhost32(cmd->tvc_vq, se_cmd->residual_count);
 534                 /* TODO is status_qualifier field needed? */
 535                 v_rsp.status = se_cmd->scsi_status;
 536                 v_rsp.sense_len = cpu_to_vhost32(cmd->tvc_vq,
 537                                                  se_cmd->scsi_sense_length);
 538                 memcpy(v_rsp.sense, cmd->tvc_sense_buf,
 539                        se_cmd->scsi_sense_length);
 540
 541                 iov_iter_init(&iov_iter, READ, &cmd->tvc_resp_iov,
 542                               cmd->tvc_in_iovs, sizeof(v_rsp));
 543                 ret = copy_to_iter(&v_rsp, sizeof(v_rsp), &iov_iter);
 544                 if (likely(ret == sizeof(v_rsp))) {
 545                         struct vhost_scsi_virtqueue *q;
 546                         vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0);
 547                         q = container_of(cmd->tvc_vq, struct vhost_scsi_virtqueue, vq);
 548                         vq = q - vs->vqs;
 549                         __set_bit(vq, signal);
 550                 } else
 551                         pr_err("Faulted on virtio_scsi_cmd_resp\n");
 552
 553                 vhost_scsi_free_cmd(cmd);
 554         }
 555
 556         vq = -1;
 557         while ((vq = find_next_bit(signal, VHOST_SCSI_MAX_VQ, vq + 1))
 558                 < VHOST_SCSI_MAX_VQ)
 559                 vhost_signal(&vs->dev, &vs->vqs[vq].vq);
 560 }
 561
 562 static struct vhost_scsi_cmd *
 563 vhost_scsi_get_tag(struct vhost_virtqueue *vq, struct vhost_scsi_tpg *tpg,
 564                    unsigned char *cdb, u64 scsi_tag, u16 lun, u8 task_attr,
 565                    u32 exp_data_len, int data_direction)
 566 {
 567         struct vhost_scsi_cmd *cmd;
 568         struct vhost_scsi_nexus *tv_nexus;
 569         struct se_session *se_sess;
 570         struct scatterlist *sg, *prot_sg;
 571         struct page **pages;
 572         int tag, cpu;
 573
 574         tv_nexus = tpg->tpg_nexus;
 575         if (!tv_nexus) {
 576                 pr_err("Unable to locate active struct vhost_scsi_nexus\n");
 577                 return ERR_PTR(-EIO);
 578         }
 579         se_sess = tv_nexus->tvn_se_sess;
 580
 581         tag = sbitmap_queue_get(&se_sess->sess_tag_pool, &cpu);
 582         if (tag < 0) {
 583                 pr_err("Unable to obtain tag for vhost_scsi_cmd\n");
 584                 return ERR_PTR(-ENOMEM);
 585         }
 586
 587         cmd = &((struct vhost_scsi_cmd *)se_sess->sess_cmd_map)[tag];
 588         sg = cmd->tvc_sgl;
 589         prot_sg = cmd->tvc_prot_sgl;
 590         pages = cmd->tvc_upages;
 591         memset(cmd, 0, sizeof(*cmd));
 592         cmd->tvc_sgl = sg;
 593         cmd->tvc_prot_sgl = prot_sg;
 594         cmd->tvc_upages = pages;
 595         cmd->tvc_se_cmd.map_tag = tag;
 596         cmd->tvc_se_cmd.map_cpu = cpu;
 597         cmd->tvc_tag = scsi_tag;
 598         cmd->tvc_lun = lun;
 599         cmd->tvc_task_attr = task_attr;
 600         cmd->tvc_exp_data_len = exp_data_len;
 601         cmd->tvc_data_direction = data_direction;
 602         cmd->tvc_nexus = tv_nexus;
 603         cmd->inflight = vhost_scsi_get_inflight(vq);
 604
 605         memcpy(cmd->tvc_cdb, cdb, VHOST_SCSI_MAX_CDB_SIZE);
 606
 607         return cmd;
 608 }
 609
 610 /*
 611  * Map a user memory range into a scatterlist
 612  *
 613  * Returns the number of scatterlist entries used or -errno on error.
 614  */
 615 static int
 616 vhost_scsi_map_to_sgl(struct vhost_scsi_cmd *cmd,
 617                       struct iov_iter *iter,
 618                       struct scatterlist *sgl,
 619                       bool write)
 620 {
 621         struct page **pages = cmd->tvc_upages;
 622         struct scatterlist *sg = sgl;
 623         ssize_t bytes;
 624         size_t offset;
 625         unsigned int npages = 0;
 626
 627         bytes = iov_iter_get_pages(iter, pages, LONG_MAX,
 628                                 VHOST_SCSI_PREALLOC_UPAGES, &offset);
 629         /* No pages were pinned */
 630         if (bytes <= 0)
 631                 return bytes < 0 ? bytes : -EFAULT;
 632
 633         iov_iter_advance(iter, bytes);
 634
 635         while (bytes) {
 636                 unsigned n = min_t(unsigned, PAGE_SIZE - offset, bytes);
 637                 sg_set_page(sg++, pages[npages++], n, offset);
 638                 bytes -= n;
 639                 offset = 0;
 640         }
 641         return npages;
 642 }
 643
 644 static int
 645 vhost_scsi_calc_sgls(struct iov_iter *iter, size_t bytes, int max_sgls)
 646 {
 647         int sgl_count = 0;
 648
 649         if (!iter || !iter->iov) {
 650                 pr_err("%s: iter->iov is NULL, but expected bytes: %zu"
 651                        " present\n", __func__, bytes);
 652                 return -EINVAL;
 653         }
 654
 655         sgl_count = iov_iter_npages(iter, 0xffff);
 656         if (sgl_count > max_sgls) {
 657                 pr_err("%s: requested sgl_count: %d exceeds pre-allocated"
 658                        " max_sgls: %d\n", __func__, sgl_count, max_sgls);
 659                 return -EINVAL;
 660         }
 661         return sgl_count;
 662 }
 663
 664 static int
 665 vhost_scsi_iov_to_sgl(struct vhost_scsi_cmd *cmd, bool write,
 666                       struct iov_iter *iter,
 667                       struct scatterlist *sg, int sg_count)
 668 {
 669         struct scatterlist *p = sg;
 670         int ret;
 671
 672         while (iov_iter_count(iter)) {
 673                 ret = vhost_scsi_map_to_sgl(cmd, iter, sg, write);
 674                 if (ret < 0) {
 675                         while (p < sg) {
 676                                 struct page *page = sg_page(p++);
 677                                 if (page)
 678                                         put_page(page);
 679                         }
 680                         return ret;
 681                 }
 682                 sg += ret;
 683         }
 684         return 0;
 685 }
 686
 687 static int
 688 vhost_scsi_mapal(struct vhost_scsi_cmd *cmd,
 689                  size_t prot_bytes, struct iov_iter *prot_iter,
 690                  size_t data_bytes, struct iov_iter *data_iter)
 691 {
 692         int sgl_count, ret;
 693         bool write = (cmd->tvc_data_direction == DMA_FROM_DEVICE);
 694
 695         if (prot_bytes) {
 696                 sgl_count = vhost_scsi_calc_sgls(prot_iter, prot_bytes,
 697                                                  VHOST_SCSI_PREALLOC_PROT_SGLS);
 698                 if (sgl_count < 0)
 699                         return sgl_count;
 700
 701                 sg_init_table(cmd->tvc_prot_sgl, sgl_count);
 702                 cmd->tvc_prot_sgl_count = sgl_count;
 703                 pr_debug("%s prot_sg %p prot_sgl_count %u\n", __func__,
 704                          cmd->tvc_prot_sgl, cmd->tvc_prot_sgl_count);
 705
 706                 ret = vhost_scsi_iov_to_sgl(cmd, write, prot_iter,
 707                                             cmd->tvc_prot_sgl,
 708                                             cmd->tvc_prot_sgl_count);
 709                 if (ret < 0) {
 710                         cmd->tvc_prot_sgl_count = 0;
 711                         return ret;
 712                 }
 713         }
 714         sgl_count = vhost_scsi_calc_sgls(data_iter, data_bytes,
 715                                          VHOST_SCSI_PREALLOC_SGLS);
 716         if (sgl_count < 0)
 717                 return sgl_count;
 718
 719         sg_init_table(cmd->tvc_sgl, sgl_count);
 720         cmd->tvc_sgl_count = sgl_count;
 721         pr_debug("%s data_sg %p data_sgl_count %u\n", __func__,
 722                   cmd->tvc_sgl, cmd->tvc_sgl_count);
 723
 724         ret = vhost_scsi_iov_to_sgl(cmd, write, data_iter,
 725                                     cmd->tvc_sgl, cmd->tvc_sgl_count);
 726         if (ret < 0) {
 727                 cmd->tvc_sgl_count = 0;
 728                 return ret;
 729         }
 730         return 0;
 731 }
 732
 733 static int vhost_scsi_to_tcm_attr(int attr)
 734 {
 735         switch (attr) {
 736         case VIRTIO_SCSI_S_SIMPLE:
 737                 return TCM_SIMPLE_TAG;
 738         case VIRTIO_SCSI_S_ORDERED:
 739                 return TCM_ORDERED_TAG;
 740         case VIRTIO_SCSI_S_HEAD:
 741                 return TCM_HEAD_TAG;
 742         case VIRTIO_SCSI_S_ACA:
 743                 return TCM_ACA_TAG;
 744         default:
 745                 break;
 746         }
 747         return TCM_SIMPLE_TAG;
 748 }
 749
 750 static void vhost_scsi_submission_work(struct work_struct *work)
 751 {
 752         struct vhost_scsi_cmd *cmd =
 753                 container_of(work, struct vhost_scsi_cmd, work);
 754         struct vhost_scsi_nexus *tv_nexus;
 755         struct se_cmd *se_cmd = &cmd->tvc_se_cmd;
 756         struct scatterlist *sg_ptr, *sg_prot_ptr = NULL;
 757         int rc;
 758
 759         /* FIXME: BIDI operation */
 760         if (cmd->tvc_sgl_count) {
 761                 sg_ptr = cmd->tvc_sgl;
 762
 763                 if (cmd->tvc_prot_sgl_count)
 764                         sg_prot_ptr = cmd->tvc_prot_sgl;
 765                 else
 766                         se_cmd->prot_pto = true;
 767         } else {
 768                 sg_ptr = NULL;
 769         }
 770         tv_nexus = cmd->tvc_nexus;
 771
 772         se_cmd->tag = 0;
 773         rc = target_submit_cmd_map_sgls(se_cmd, tv_nexus->tvn_se_sess,
 774                         cmd->tvc_cdb, &cmd->tvc_sense_buf[0],
 775                         cmd->tvc_lun, cmd->tvc_exp_data_len,
 776                         vhost_scsi_to_tcm_attr(cmd->tvc_task_attr),
 777                         cmd->tvc_data_direction, TARGET_SCF_ACK_KREF,
 778                         sg_ptr, cmd->tvc_sgl_count, NULL, 0, sg_prot_ptr,
 779                         cmd->tvc_prot_sgl_count);
 780         if (rc < 0) {
 781                 transport_send_check_condition_and_sense(se_cmd,
 782                                 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE, 0);
 783                 transport_generic_free_cmd(se_cmd, 0);
 784         }
 785 }
 786
 787 static void
 788 vhost_scsi_send_bad_target(struct vhost_scsi *vs,
 789                            struct vhost_virtqueue *vq,
 790                            int head, unsigned out)
 791 {
 792         struct virtio_scsi_cmd_resp __user *resp;
 793         struct virtio_scsi_cmd_resp rsp;
 794         int ret;
 795
 796         memset(&rsp, 0, sizeof(rsp));
 797         rsp.response = VIRTIO_SCSI_S_BAD_TARGET;
 798         resp = vq->iov[out].iov_base;
 799         ret = __copy_to_user(resp, &rsp, sizeof(rsp));
 800         if (!ret)
 801                 vhost_add_used_and_signal(&vs->dev, vq, head, 0);
 802         else
 803                 pr_err("Faulted on virtio_scsi_cmd_resp\n");
 804 }
 805
 806 static int
 807 vhost_scsi_get_desc(struct vhost_scsi *vs, struct vhost_virtqueue *vq,
 808                     struct vhost_scsi_ctx *vc)
 809 {
 810         int ret = -ENXIO;
 811
 812         vc->head = vhost_get_vq_desc(vq, vq->iov,
 813                                      ARRAY_SIZE(vq->iov), &vc->out, &vc->in,
 814                                      NULL, NULL);
 815
 816         pr_debug("vhost_get_vq_desc: head: %d, out: %u in: %u\n",
 817                  vc->head, vc->out, vc->in);
 818
 819         /* On error, stop handling until the next kick. */
 820         if (unlikely(vc->head < 0))
 821                 goto done;
 822
 823         /* Nothing new?  Wait for eventfd to tell us they refilled. */
 824         if (vc->head == vq->num) {
 825                 if (unlikely(vhost_enable_notify(&vs->dev, vq))) {
 826                         vhost_disable_notify(&vs->dev, vq);
 827                         ret = -EAGAIN;
 828                 }
 829                 goto done;
 830         }
 831
 832         /*
 833          * Get the size of request and response buffers.
 834          * FIXME: Not correct for BIDI operation
 835          */
 836         vc->out_size = iov_length(vq->iov, vc->out);
 837         vc->in_size = iov_length(&vq->iov[vc->out], vc->in);
 838
 839         /*
 840          * Copy over the virtio-scsi request header, which for a
 841          * ANY_LAYOUT enabled guest may span multiple iovecs, or a
 842          * single iovec may contain both the header + outgoing
 843          * WRITE payloads.
 844          *
 845          * copy_from_iter() will advance out_iter, so that it will
 846          * point at the start of the outgoing WRITE payload, if
 847          * DMA_TO_DEVICE is set.
 848          */
 849         iov_iter_init(&vc->out_iter, WRITE, vq->iov, vc->out, vc->out_size);
 850         ret = 0;
 851
 852 done:
 853         return ret;
 854 }
 855
 856 static int
 857 vhost_scsi_chk_size(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc)
 858 {
 859         if (unlikely(vc->in_size < vc->rsp_size)) {
 860                 vq_err(vq,
 861                        "Response buf too small, need min %zu bytes got %zu",
 862                        vc->rsp_size, vc->in_size);
 863                 return -EINVAL;
 864         } else if (unlikely(vc->out_size < vc->req_size)) {
 865                 vq_err(vq,
 866                        "Request buf too small, need min %zu bytes got %zu",
 867                        vc->req_size, vc->out_size);
 868                 return -EIO;
 869         }
 870
 871         return 0;
 872 }
 873
 874 static int
 875 vhost_scsi_get_req(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc,
 876                    struct vhost_scsi_tpg **tpgp)
 877 {
 878         int ret = -EIO;
 879
 880         if (unlikely(!copy_from_iter_full(vc->req, vc->req_size,
 881                                           &vc->out_iter))) {
 882                 vq_err(vq, "Faulted on copy_from_iter_full\n");
 883         } else if (unlikely(*vc->lunp != 1)) {
 884                 /* virtio-scsi spec requires byte 0 of the lun to be 1 */
 885                 vq_err(vq, "Illegal virtio-scsi lun: %u\n", *vc->lunp);
 886         } else {
 887                 struct vhost_scsi_tpg **vs_tpg, *tpg;
 888
 889                 vs_tpg = vq->private_data;      /* validated at handler entry */
 890
 891                 tpg = READ_ONCE(vs_tpg[*vc->target]);
 892                 if (unlikely(!tpg)) {
 893                         vq_err(vq, "Target 0x%x does not exist\n", *vc->target);
 894                 } else {
 895                         if (tpgp)
 896                                 *tpgp = tpg;
 897                         ret = 0;
 898                 }
 899         }
 900
 901         return ret;
 902 }
 903
 904 static void
 905 vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
 906 {
 907         struct vhost_scsi_tpg **vs_tpg, *tpg;
 908         struct virtio_scsi_cmd_req v_req;
 909         struct virtio_scsi_cmd_req_pi v_req_pi;
 910         struct vhost_scsi_ctx vc;
 911         struct vhost_scsi_cmd *cmd;
 912         struct iov_iter in_iter, prot_iter, data_iter;
 913         u64 tag;
 914         u32 exp_data_len, data_direction;
 915         int ret, prot_bytes;
 916         u16 lun;
 917         u8 task_attr;
 918         bool t10_pi = vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI);
 919         void *cdb;
 920
 921         mutex_lock(&vq->mutex);
 922         /*
 923          * We can handle the vq only after the endpoint is setup by calling the
 924          * VHOST_SCSI_SET_ENDPOINT ioctl.
 925          */
 926         vs_tpg = vq->private_data;
 927         if (!vs_tpg)
 928                 goto out;
 929
 930         memset(&vc, 0, sizeof(vc));
 931         vc.rsp_size = sizeof(struct virtio_scsi_cmd_resp);
 932
 933         vhost_disable_notify(&vs->dev, vq);
 934
 935         for (;;) {
 936                 ret = vhost_scsi_get_desc(vs, vq, &vc);
 937                 if (ret)
 938                         goto err;
 939
 940                 /*
 941                  * Setup pointers and values based upon different virtio-scsi
 942                  * request header if T10_PI is enabled in KVM guest.
 943                  */
 944                 if (t10_pi) {
 945                         vc.req = &v_req_pi;
 946                         vc.req_size = sizeof(v_req_pi);
 947                         vc.lunp = &v_req_pi.lun[0];
 948                         vc.target = &v_req_pi.lun[1];
 949                 } else {
 950                         vc.req = &v_req;
 951                         vc.req_size = sizeof(v_req);
 952                         vc.lunp = &v_req.lun[0];
 953                         vc.target = &v_req.lun[1];
 954                 }
 955
 956                 /*
 957                  * Validate the size of request and response buffers.
 958                  * Check for a sane response buffer so we can report
 959                  * early errors back to the guest.
 960                  */
 961                 ret = vhost_scsi_chk_size(vq, &vc);
 962                 if (ret)
 963                         goto err;
 964
 965                 ret = vhost_scsi_get_req(vq, &vc, &tpg);
 966                 if (ret)
 967                         goto err;
 968
 969                 ret = -EIO;     /* bad target on any error from here on */
 970
 971                 /*
 972                  * Determine data_direction by calculating the total outgoing
 973                  * iovec sizes + incoming iovec sizes vs. virtio-scsi request +
 974                  * response headers respectively.
 975                  *
 976                  * For DMA_TO_DEVICE this is out_iter, which is already pointing
 977                  * to the right place.
 978                  *
 979                  * For DMA_FROM_DEVICE, the iovec will be just past the end
 980                  * of the virtio-scsi response header in either the same
 981                  * or immediately following iovec.
 982                  *
 983                  * Any associated T10_PI bytes for the outgoing / incoming
 984                  * payloads are included in calculation of exp_data_len here.
 985                  */
 986                 prot_bytes = 0;
 987
 988                 if (vc.out_size > vc.req_size) {
 989                         data_direction = DMA_TO_DEVICE;
 990                         exp_data_len = vc.out_size - vc.req_size;
 991                         data_iter = vc.out_iter;
 992                 } else if (vc.in_size > vc.rsp_size) {
 993                         data_direction = DMA_FROM_DEVICE;
 994                         exp_data_len = vc.in_size - vc.rsp_size;
 995
 996                         iov_iter_init(&in_iter, READ, &vq->iov[vc.out], vc.in,
 997                                       vc.rsp_size + exp_data_len);
 998                         iov_iter_advance(&in_iter, vc.rsp_size);
 999                         data_iter = in_iter;
1000                 } else {
1001                         data_direction = DMA_NONE;
1002                         exp_data_len = 0;
1003                 }
1004                 /*
1005                  * If T10_PI header + payload is present, setup prot_iter values
1006                  * and recalculate data_iter for vhost_scsi_mapal() mapping to
1007                  * host scatterlists via get_user_pages_fast().
1008                  */
1009                 if (t10_pi) {
1010                         if (v_req_pi.pi_bytesout) {
1011                                 if (data_direction != DMA_TO_DEVICE) {
1012                                         vq_err(vq, "Received non zero pi_bytesout,"
1013                                                 " but wrong data_direction\n");
1014                                         goto err;
1015                                 }
1016                                 prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesout);
1017                         } else if (v_req_pi.pi_bytesin) {
1018                                 if (data_direction != DMA_FROM_DEVICE) {
1019                                         vq_err(vq, "Received non zero pi_bytesin,"
1020                                                 " but wrong data_direction\n");
1021                                         goto err;
1022                                 }
1023                                 prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesin);
1024                         }
1025                         /*
1026                          * Set prot_iter to data_iter and truncate it to
1027                          * prot_bytes, and advance data_iter past any
1028                          * preceeding prot_bytes that may be present.
1029                          *
1030                          * Also fix up the exp_data_len to reflect only the
1031                          * actual data payload length.
1032                          */
1033                         if (prot_bytes) {
1034                                 exp_data_len -= prot_bytes;
1035                                 prot_iter = data_iter;
1036                                 iov_iter_truncate(&prot_iter, prot_bytes);
1037                                 iov_iter_advance(&data_iter, prot_bytes);
1038                         }
1039                         tag = vhost64_to_cpu(vq, v_req_pi.tag);
1040                         task_attr = v_req_pi.task_attr;
1041                         cdb = &v_req_pi.cdb[0];
1042                         lun = ((v_req_pi.lun[2] << 8) | v_req_pi.lun[3]) & 0x3FFF;
1043                 } else {
1044                         tag = vhost64_to_cpu(vq, v_req.tag);
1045                         task_attr = v_req.task_attr;
1046                         cdb = &v_req.cdb[0];
1047                         lun = ((v_req.lun[2] << 8) | v_req.lun[3]) & 0x3FFF;
1048                 }
1049                 /*
1050                  * Check that the received CDB size does not exceeded our
1051                  * hardcoded max for vhost-scsi, then get a pre-allocated
1052                  * cmd descriptor for the new virtio-scsi tag.
1053                  *
1054                  * TODO what if cdb was too small for varlen cdb header?
1055                  */
1056                 if (unlikely(scsi_command_size(cdb) > VHOST_SCSI_MAX_CDB_SIZE)) {
1057                         vq_err(vq, "Received SCSI CDB with command_size: %d that"
1058                                 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
1059                                 scsi_command_size(cdb), VHOST_SCSI_MAX_CDB_SIZE);
1060                                 goto err;
1061                 }
1062                 cmd = vhost_scsi_get_tag(vq, tpg, cdb, tag, lun, task_attr,
1063                                          exp_data_len + prot_bytes,
1064                                          data_direction);
1065                 if (IS_ERR(cmd)) {
1066                         vq_err(vq, "vhost_scsi_get_tag failed %ld\n",
1067                                PTR_ERR(cmd));
1068                         goto err;
1069                 }
1070                 cmd->tvc_vhost = vs;
1071                 cmd->tvc_vq = vq;
1072                 cmd->tvc_resp_iov = vq->iov[vc.out];
1073                 cmd->tvc_in_iovs = vc.in;
1074
1075                 pr_debug("vhost_scsi got command opcode: %#02x, lun: %d\n",
1076                          cmd->tvc_cdb[0], cmd->tvc_lun);
1077                 pr_debug("cmd: %p exp_data_len: %d, prot_bytes: %d data_direction:"
1078                          " %d\n", cmd, exp_data_len, prot_bytes, data_direction);
1079
1080                 if (data_direction != DMA_NONE) {
1081                         if (unlikely(vhost_scsi_mapal(cmd, prot_bytes,
1082                                                       &prot_iter, exp_data_len,
1083                                                       &data_iter))) {
1084                                 vq_err(vq, "Failed to map iov to sgl\n");
1085                                 vhost_scsi_release_cmd(&cmd->tvc_se_cmd);
1086                                 goto err;
1087                         }
1088                 }
1089                 /*
1090                  * Save the descriptor from vhost_get_vq_desc() to be used to
1091                  * complete the virtio-scsi request in TCM callback context via
1092                  * vhost_scsi_queue_data_in() and vhost_scsi_queue_status()
1093                  */
1094                 cmd->tvc_vq_desc = vc.head;
1095                 /*
1096                  * Dispatch cmd descriptor for cmwq execution in process
1097                  * context provided by vhost_scsi_workqueue.  This also ensures
1098                  * cmd is executed on the same kworker CPU as this vhost
1099                  * thread to gain positive L2 cache locality effects.
1100                  */
1101                 INIT_WORK(&cmd->work, vhost_scsi_submission_work);
1102                 queue_work(vhost_scsi_workqueue, &cmd->work);
1103                 ret = 0;
1104 err:
1105                 /*
1106                  * ENXIO:  No more requests, or read error, wait for next kick
1107                  * EINVAL: Invalid response buffer, drop the request
1108                  * EIO:    Respond with bad target
1109                  * EAGAIN: Pending request
1110                  */
1111                 if (ret == -ENXIO)
1112                         break;
1113                 else if (ret == -EIO)
1114                         vhost_scsi_send_bad_target(vs, vq, vc.head, vc.out);
1115         }
1116 out:
1117         mutex_unlock(&vq->mutex);
1118 }
1119
1120 static void
1121 vhost_scsi_send_tmf_reject(struct vhost_scsi *vs,
1122                            struct vhost_virtqueue *vq,
1123                            struct vhost_scsi_ctx *vc)
1124 {
1125         struct virtio_scsi_ctrl_tmf_resp rsp;
1126         struct iov_iter iov_iter;
1127         int ret;
1128
1129         pr_debug("%s\n", __func__);
1130         memset(&rsp, 0, sizeof(rsp));
1131         rsp.response = VIRTIO_SCSI_S_FUNCTION_REJECTED;
1132
1133         iov_iter_init(&iov_iter, READ, &vq->iov[vc->out], vc->in, sizeof(rsp));
1134
1135         ret = copy_to_iter(&rsp, sizeof(rsp), &iov_iter);
1136         if (likely(ret == sizeof(rsp)))
1137                 vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0);
1138         else
1139                 pr_err("Faulted on virtio_scsi_ctrl_tmf_resp\n");
1140 }
1141
1142 static void
1143 vhost_scsi_send_an_resp(struct vhost_scsi *vs,
1144                         struct vhost_virtqueue *vq,
1145                         struct vhost_scsi_ctx *vc)
1146 {
1147         struct virtio_scsi_ctrl_an_resp rsp;
1148         struct iov_iter iov_iter;
1149         int ret;
1150
1151         pr_debug("%s\n", __func__);
1152         memset(&rsp, 0, sizeof(rsp));   /* event_actual = 0 */
1153         rsp.response = VIRTIO_SCSI_S_OK;
1154
1155         iov_iter_init(&iov_iter, READ, &vq->iov[vc->out], vc->in, sizeof(rsp));
1156
1157         ret = copy_to_iter(&rsp, sizeof(rsp), &iov_iter);
1158         if (likely(ret == sizeof(rsp)))
1159                 vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0);
1160         else
1161                 pr_err("Faulted on virtio_scsi_ctrl_an_resp\n");
1162 }
1163
1164 static void
1165 vhost_scsi_ctl_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
1166 {
1167         union {
1168                 __virtio32 type;
1169                 struct virtio_scsi_ctrl_an_req an;
1170                 struct virtio_scsi_ctrl_tmf_req tmf;
1171         } v_req;
1172         struct vhost_scsi_ctx vc;
1173         size_t typ_size;
1174         int ret;
1175
1176         mutex_lock(&vq->mutex);
1177         /*
1178          * We can handle the vq only after the endpoint is setup by calling the
1179          * VHOST_SCSI_SET_ENDPOINT ioctl.
1180          */
1181         if (!vq->private_data)
1182                 goto out;
1183
1184         memset(&vc, 0, sizeof(vc));
1185
1186         vhost_disable_notify(&vs->dev, vq);
1187
1188         for (;;) {
1189                 ret = vhost_scsi_get_desc(vs, vq, &vc);
1190                 if (ret)
1191                         goto err;
1192
1193                 /*
1194                  * Get the request type first in order to setup
1195                  * other parameters dependent on the type.
1196                  */
1197                 vc.req = &v_req.type;
1198                 typ_size = sizeof(v_req.type);
1199
1200                 if (unlikely(!copy_from_iter_full(vc.req, typ_size,
1201                                                   &vc.out_iter))) {
1202                         vq_err(vq, "Faulted on copy_from_iter tmf type\n");
1203                         /*
1204                          * The size of the response buffer depends on the
1205                          * request type and must be validated against it.
1206                          * Since the request type is not known, don't send
1207                          * a response.
1208                          */
1209                         continue;
1210                 }
1211
1212                 switch (v_req.type) {
1213                 case VIRTIO_SCSI_T_TMF:
1214                         vc.req = &v_req.tmf;
1215                         vc.req_size = sizeof(struct virtio_scsi_ctrl_tmf_req);
1216                         vc.rsp_size = sizeof(struct virtio_scsi_ctrl_tmf_resp);
1217                         vc.lunp = &v_req.tmf.lun[0];
1218                         vc.target = &v_req.tmf.lun[1];
1219                         break;
1220                 case VIRTIO_SCSI_T_AN_QUERY:
1221                 case VIRTIO_SCSI_T_AN_SUBSCRIBE:
1222                         vc.req = &v_req.an;
1223                         vc.req_size = sizeof(struct virtio_scsi_ctrl_an_req);
1224                         vc.rsp_size = sizeof(struct virtio_scsi_ctrl_an_resp);
1225                         vc.lunp = &v_req.an.lun[0];
1226                         vc.target = NULL;
1227                         break;
1228                 default:
1229                         vq_err(vq, "Unknown control request %d", v_req.type);
1230                         continue;
1231                 }
1232
1233                 /*
1234                  * Validate the size of request and response buffers.
1235                  * Check for a sane response buffer so we can report
1236                  * early errors back to the guest.
1237                  */
1238                 ret = vhost_scsi_chk_size(vq, &vc);
1239                 if (ret)
1240                         goto err;
1241
1242                 /*
1243                  * Get the rest of the request now that its size is known.
1244                  */
1245                 vc.req += typ_size;
1246                 vc.req_size -= typ_size;
1247
1248                 ret = vhost_scsi_get_req(vq, &vc, NULL);
1249                 if (ret)
1250                         goto err;
1251
1252                 if (v_req.type == VIRTIO_SCSI_T_TMF)
1253                         vhost_scsi_send_tmf_reject(vs, vq, &vc);
1254                 else
1255                         vhost_scsi_send_an_resp(vs, vq, &vc);
1256 err:
1257                 /*
1258                  * ENXIO:  No more requests, or read error, wait for next kick
1259                  * EINVAL: Invalid response buffer, drop the request
1260                  * EIO:    Respond with bad target
1261                  * EAGAIN: Pending request
1262                  */
1263                 if (ret == -ENXIO)
1264                         break;
1265                 else if (ret == -EIO)
1266                         vhost_scsi_send_bad_target(vs, vq, vc.head, vc.out);
1267         }
1268 out:
1269         mutex_unlock(&vq->mutex);
1270 }
1271
1272 static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)
1273 {
1274         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1275                                                 poll.work);
1276         struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1277
1278         pr_debug("%s: The handling func for control queue.\n", __func__);
1279         vhost_scsi_ctl_handle_vq(vs, vq);
1280 }
1281
1282 static void
1283 vhost_scsi_send_evt(struct vhost_scsi *vs,
1284                    struct vhost_scsi_tpg *tpg,
1285                    struct se_lun *lun,
1286                    u32 event,
1287                    u32 reason)
1288 {
1289         struct vhost_scsi_evt *evt;
1290
1291         evt = vhost_scsi_allocate_evt(vs, event, reason);
1292         if (!evt)
1293                 return;
1294
1295         if (tpg && lun) {
1296                 /* TODO: share lun setup code with virtio-scsi.ko */
1297                 /*
1298                  * Note: evt->event is zeroed when we allocate it and
1299                  * lun[4-7] need to be zero according to virtio-scsi spec.
1300                  */
1301                 evt->event.lun[0] = 0x01;
1302                 evt->event.lun[1] = tpg->tport_tpgt;
1303                 if (lun->unpacked_lun >= 256)
1304                         evt->event.lun[2] = lun->unpacked_lun >> 8 | 0x40 ;
1305                 evt->event.lun[3] = lun->unpacked_lun & 0xFF;
1306         }
1307
1308         llist_add(&evt->list, &vs->vs_event_list);
1309         vhost_work_queue(&vs->dev, &vs->vs_event_work);
1310 }
1311
1312 static void vhost_scsi_evt_handle_kick(struct vhost_work *work)
1313 {
1314         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1315                                                 poll.work);
1316         struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1317
1318         mutex_lock(&vq->mutex);
1319         if (!vq->private_data)
1320                 goto out;
1321
1322         if (vs->vs_events_missed)
1323                 vhost_scsi_send_evt(vs, NULL, NULL, VIRTIO_SCSI_T_NO_EVENT, 0);
1324 out:
1325         mutex_unlock(&vq->mutex);
1326 }
1327
1328 static void vhost_scsi_handle_kick(struct vhost_work *work)
1329 {
1330         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1331                                                 poll.work);
1332         struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1333
1334         vhost_scsi_handle_vq(vs, vq);
1335 }
1336
1337 static void vhost_scsi_flush_vq(struct vhost_scsi *vs, int index)
1338 {
1339         vhost_poll_flush(&vs->vqs[index].vq.poll);
1340 }
1341
1342 /* Callers must hold dev mutex */
1343 static void vhost_scsi_flush(struct vhost_scsi *vs)
1344 {
1345         struct vhost_scsi_inflight *old_inflight[VHOST_SCSI_MAX_VQ];
1346         int i;
1347
1348         /* Init new inflight and remember the old inflight */
1349         vhost_scsi_init_inflight(vs, old_inflight);
1350
1351         /*
1352          * The inflight->kref was initialized to 1. We decrement it here to
1353          * indicate the start of the flush operation so that it will reach 0
1354          * when all the reqs are finished.
1355          */
1356         for (i = 0; i < VHOST_SCSI_MAX_VQ; i++)
1357                 kref_put(&old_inflight[i]->kref, vhost_scsi_done_inflight);
1358
1359         /* Flush both the vhost poll and vhost work */
1360         for (i = 0; i < VHOST_SCSI_MAX_VQ; i++)
1361                 vhost_scsi_flush_vq(vs, i);
1362         vhost_work_flush(&vs->dev, &vs->vs_completion_work);
1363         vhost_work_flush(&vs->dev, &vs->vs_event_work);
1364
1365         /* Wait for all reqs issued before the flush to be finished */
1366         for (i = 0; i < VHOST_SCSI_MAX_VQ; i++)
1367                 wait_for_completion(&old_inflight[i]->comp);
1368 }
1369
1370 /*
1371  * Called from vhost_scsi_ioctl() context to walk the list of available
1372  * vhost_scsi_tpg with an active struct vhost_scsi_nexus
1373  *
1374  *  The lock nesting rule is:
1375  *    vhost_scsi_mutex -> vs->dev.mutex -> tpg->tv_tpg_mutex -> vq->mutex
1376  */
1377 static int
1378 vhost_scsi_set_endpoint(struct vhost_scsi *vs,
1379                         struct vhost_scsi_target *t)
1380 {
1381         struct se_portal_group *se_tpg;
1382         struct vhost_scsi_tport *tv_tport;
1383         struct vhost_scsi_tpg *tpg;
1384         struct vhost_scsi_tpg **vs_tpg;
1385         struct vhost_virtqueue *vq;
1386         int index, ret, i, len;
1387         bool match = false;
1388
1389         mutex_lock(&vhost_scsi_mutex);
1390         mutex_lock(&vs->dev.mutex);
1391
1392         /* Verify that ring has been setup correctly. */
1393         for (index = 0; index < vs->dev.nvqs; ++index) {
1394                 /* Verify that ring has been setup correctly. */
1395                 if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {
1396                         ret = -EFAULT;
1397                         goto out;
1398                 }
1399         }
1400
1401         len = sizeof(vs_tpg[0]) * VHOST_SCSI_MAX_TARGET;
1402         vs_tpg = kzalloc(len, GFP_KERNEL);
1403         if (!vs_tpg) {
1404                 ret = -ENOMEM;
1405                 goto out;
1406         }
1407         if (vs->vs_tpg)
1408                 memcpy(vs_tpg, vs->vs_tpg, len);
1409
1410         list_for_each_entry(tpg, &vhost_scsi_list, tv_tpg_list) {
1411                 mutex_lock(&tpg->tv_tpg_mutex);
1412                 if (!tpg->tpg_nexus) {
1413                         mutex_unlock(&tpg->tv_tpg_mutex);
1414                         continue;
1415                 }
1416                 if (tpg->tv_tpg_vhost_count != 0) {
1417                         mutex_unlock(&tpg->tv_tpg_mutex);
1418                         continue;
1419                 }
1420                 tv_tport = tpg->tport;
1421
1422                 if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
1423                         if (vs->vs_tpg && vs->vs_tpg[tpg->tport_tpgt]) {
1424                                 kfree(vs_tpg);
1425                                 mutex_unlock(&tpg->tv_tpg_mutex);
1426                                 ret = -EEXIST;
1427                                 goto out;
1428                         }
1429                         /*
1430                          * In order to ensure individual vhost-scsi configfs
1431                          * groups cannot be removed while in use by vhost ioctl,
1432                          * go ahead and take an explicit se_tpg->tpg_group.cg_item
1433                          * dependency now.
1434                          */
1435                         se_tpg = &tpg->se_tpg;
1436                         ret = target_depend_item(&se_tpg->tpg_group.cg_item);
1437                         if (ret) {
1438                                 pr_warn("target_depend_item() failed: %d\n", ret);
1439                                 kfree(vs_tpg);
1440                                 mutex_unlock(&tpg->tv_tpg_mutex);
1441                                 goto out;
1442                         }
1443                         tpg->tv_tpg_vhost_count++;
1444                         tpg->vhost_scsi = vs;
1445                         vs_tpg[tpg->tport_tpgt] = tpg;
1446                         smp_mb__after_atomic();
1447                         match = true;
1448                 }
1449                 mutex_unlock(&tpg->tv_tpg_mutex);
1450         }
1451
1452         if (match) {
1453                 memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn,
1454                        sizeof(vs->vs_vhost_wwpn));
1455                 for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
1456                         vq = &vs->vqs[i].vq;
1457                         mutex_lock(&vq->mutex);
1458                         vq->private_data = vs_tpg;
1459                         vhost_vq_init_access(vq);
1460                         mutex_unlock(&vq->mutex);
1461                 }
1462                 ret = 0;
1463         } else {
1464                 ret = -EEXIST;
1465         }
1466
1467         /*
1468          * Act as synchronize_rcu to make sure access to
1469          * old vs->vs_tpg is finished.
1470          */
1471         vhost_scsi_flush(vs);
1472         kfree(vs->vs_tpg);
1473         vs->vs_tpg = vs_tpg;
1474
1475 out:
1476         mutex_unlock(&vs->dev.mutex);
1477         mutex_unlock(&vhost_scsi_mutex);
1478         return ret;
1479 }
1480
1481 static int
1482 vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
1483                           struct vhost_scsi_target *t)
1484 {
1485         struct se_portal_group *se_tpg;
1486         struct vhost_scsi_tport *tv_tport;
1487         struct vhost_scsi_tpg *tpg;
1488         struct vhost_virtqueue *vq;
1489         bool match = false;
1490         int index, ret, i;
1491         u8 target;
1492
1493         mutex_lock(&vhost_scsi_mutex);
1494         mutex_lock(&vs->dev.mutex);
1495         /* Verify that ring has been setup correctly. */
1496         for (index = 0; index < vs->dev.nvqs; ++index) {
1497                 if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {
1498                         ret = -EFAULT;
1499                         goto err_dev;
1500                 }
1501         }
1502
1503         if (!vs->vs_tpg) {
1504                 ret = 0;
1505                 goto err_dev;
1506         }
1507
1508         for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
1509                 target = i;
1510                 tpg = vs->vs_tpg[target];
1511                 if (!tpg)
1512                         continue;
1513
1514                 mutex_lock(&tpg->tv_tpg_mutex);
1515                 tv_tport = tpg->tport;
1516                 if (!tv_tport) {
1517                         ret = -ENODEV;
1518                         goto err_tpg;
1519                 }
1520
1521                 if (strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
1522                         pr_warn("tv_tport->tport_name: %s, tpg->tport_tpgt: %hu"
1523                                 " does not match t->vhost_wwpn: %s, t->vhost_tpgt: %hu\n",
1524                                 tv_tport->tport_name, tpg->tport_tpgt,
1525                                 t->vhost_wwpn, t->vhost_tpgt);
1526                         ret = -EINVAL;
1527                         goto err_tpg;
1528                 }
1529                 tpg->tv_tpg_vhost_count--;
1530                 tpg->vhost_scsi = NULL;
1531                 vs->vs_tpg[target] = NULL;
1532                 match = true;
1533                 mutex_unlock(&tpg->tv_tpg_mutex);
1534                 /*
1535                  * Release se_tpg->tpg_group.cg_item configfs dependency now
1536                  * to allow vhost-scsi WWPN se_tpg->tpg_group shutdown to occur.
1537                  */
1538                 se_tpg = &tpg->se_tpg;
1539                 target_undepend_item(&se_tpg->tpg_group.cg_item);
1540         }
1541         if (match) {
1542                 for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
1543                         vq = &vs->vqs[i].vq;
1544                         mutex_lock(&vq->mutex);
1545                         vq->private_data = NULL;
1546                         mutex_unlock(&vq->mutex);
1547                 }
1548         }
1549         /*
1550          * Act as synchronize_rcu to make sure access to
1551          * old vs->vs_tpg is finished.
1552          */
1553         vhost_scsi_flush(vs);
1554         kfree(vs->vs_tpg);
1555         vs->vs_tpg = NULL;
1556         WARN_ON(vs->vs_events_nr);
1557         mutex_unlock(&vs->dev.mutex);
1558         mutex_unlock(&vhost_scsi_mutex);
1559         return 0;
1560
1561 err_tpg:
1562         mutex_unlock(&tpg->tv_tpg_mutex);
1563 err_dev:
1564         mutex_unlock(&vs->dev.mutex);
1565         mutex_unlock(&vhost_scsi_mutex);
1566         return ret;
1567 }
1568
1569 static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features)
1570 {
1571         struct vhost_virtqueue *vq;
1572         int i;
1573
1574         if (features & ~VHOST_SCSI_FEATURES)
1575                 return -EOPNOTSUPP;
1576
1577         mutex_lock(&vs->dev.mutex);
1578         if ((features & (1 << VHOST_F_LOG_ALL)) &&
1579             !vhost_log_access_ok(&vs->dev)) {
1580                 mutex_unlock(&vs->dev.mutex);
1581                 return -EFAULT;
1582         }
1583
1584         for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
1585                 vq = &vs->vqs[i].vq;
1586                 mutex_lock(&vq->mutex);
1587                 vq->acked_features = features;
1588                 mutex_unlock(&vq->mutex);
1589         }
1590         mutex_unlock(&vs->dev.mutex);
1591         return 0;
1592 }
1593
1594 static int vhost_scsi_open(struct inode *inode, struct file *f)
1595 {
1596         struct vhost_scsi *vs;
1597         struct vhost_virtqueue **vqs;
1598         int r = -ENOMEM, i;
1599
1600         vs = kzalloc(sizeof(*vs), GFP_KERNEL | __GFP_NOWARN | __GFP_RETRY_MAYFAIL);
1601         if (!vs) {
1602                 vs = vzalloc(sizeof(*vs));
1603                 if (!vs)
1604                         goto err_vs;
1605         }
1606
1607         vqs = kmalloc_array(VHOST_SCSI_MAX_VQ, sizeof(*vqs), GFP_KERNEL);
1608         if (!vqs)
1609                 goto err_vqs;
1610
1611         vhost_work_init(&vs->vs_completion_work, vhost_scsi_complete_cmd_work);
1612         vhost_work_init(&vs->vs_event_work, vhost_scsi_evt_work);
1613
1614         vs->vs_events_nr = 0;
1615         vs->vs_events_missed = false;
1616
1617         vqs[VHOST_SCSI_VQ_CTL] = &vs->vqs[VHOST_SCSI_VQ_CTL].vq;
1618         vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
1619         vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick;
1620         vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick;
1621         for (i = VHOST_SCSI_VQ_IO; i < VHOST_SCSI_MAX_VQ; i++) {
1622                 vqs[i] = &vs->vqs[i].vq;
1623                 vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick;
1624         }
1625         vhost_dev_init(&vs->dev, vqs, VHOST_SCSI_MAX_VQ, UIO_MAXIOV);
1626
1627         vhost_scsi_init_inflight(vs, NULL);
1628
1629         f->private_data = vs;
1630         return 0;
1631
1632 err_vqs:
1633         kvfree(vs);
1634 err_vs:
1635         return r;
1636 }
1637
1638 static int vhost_scsi_release(struct inode *inode, struct file *f)
1639 {
1640         struct vhost_scsi *vs = f->private_data;
1641         struct vhost_scsi_target t;
1642
1643         mutex_lock(&vs->dev.mutex);
1644         memcpy(t.vhost_wwpn, vs->vs_vhost_wwpn, sizeof(t.vhost_wwpn));
1645         mutex_unlock(&vs->dev.mutex);
1646         vhost_scsi_clear_endpoint(vs, &t);
1647         vhost_dev_stop(&vs->dev);
1648         vhost_dev_cleanup(&vs->dev);
1649         /* Jobs can re-queue themselves in evt kick handler. Do extra flush. */
1650         vhost_scsi_flush(vs);
1651         kfree(vs->dev.vqs);
1652         kvfree(vs);
1653         return 0;
1654 }
1655
1656 static long
1657 vhost_scsi_ioctl(struct file *f,
1658                  unsigned int ioctl,
1659                  unsigned long arg)
1660 {
1661         struct vhost_scsi *vs = f->private_data;
1662         struct vhost_scsi_target backend;
1663         void __user *argp = (void __user *)arg;
1664         u64 __user *featurep = argp;
1665         u32 __user *eventsp = argp;
1666         u32 events_missed;
1667         u64 features;
1668         int r, abi_version = VHOST_SCSI_ABI_VERSION;
1669         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
1670
1671         switch (ioctl) {
1672         case VHOST_SCSI_SET_ENDPOINT:
1673                 if (copy_from_user(&backend, argp, sizeof backend))
1674                         return -EFAULT;
1675                 if (backend.reserved != 0)
1676                         return -EOPNOTSUPP;
1677
1678                 return vhost_scsi_set_endpoint(vs, &backend);
1679         case VHOST_SCSI_CLEAR_ENDPOINT:
1680                 if (copy_from_user(&backend, argp, sizeof backend))
1681                         return -EFAULT;
1682                 if (backend.reserved != 0)
1683                         return -EOPNOTSUPP;
1684
1685                 return vhost_scsi_clear_endpoint(vs, &backend);
1686         case VHOST_SCSI_GET_ABI_VERSION:
1687                 if (copy_to_user(argp, &abi_version, sizeof abi_version))
1688                         return -EFAULT;
1689                 return 0;
1690         case VHOST_SCSI_SET_EVENTS_MISSED:
1691                 if (get_user(events_missed, eventsp))
1692                         return -EFAULT;
1693                 mutex_lock(&vq->mutex);
1694                 vs->vs_events_missed = events_missed;
1695                 mutex_unlock(&vq->mutex);
1696                 return 0;
1697         case VHOST_SCSI_GET_EVENTS_MISSED:
1698                 mutex_lock(&vq->mutex);
1699                 events_missed = vs->vs_events_missed;
1700                 mutex_unlock(&vq->mutex);
1701                 if (put_user(events_missed, eventsp))
1702                         return -EFAULT;
1703                 return 0;
1704         case VHOST_GET_FEATURES:
1705                 features = VHOST_SCSI_FEATURES;
1706                 if (copy_to_user(featurep, &features, sizeof features))
1707                         return -EFAULT;
1708                 return 0;
1709         case VHOST_SET_FEATURES:
1710                 if (copy_from_user(&features, featurep, sizeof features))
1711                         return -EFAULT;
1712                 return vhost_scsi_set_features(vs, features);
1713         default:
1714                 mutex_lock(&vs->dev.mutex);
1715                 r = vhost_dev_ioctl(&vs->dev, ioctl, argp);
1716                 /* TODO: flush backend after dev ioctl. */
1717                 if (r == -ENOIOCTLCMD)
1718                         r = vhost_vring_ioctl(&vs->dev, ioctl, argp);
1719                 mutex_unlock(&vs->dev.mutex);
1720                 return r;
1721         }
1722 }
1723
1724 #ifdef CONFIG_COMPAT
1725 static long vhost_scsi_compat_ioctl(struct file *f, unsigned int ioctl,
1726                                 unsigned long arg)
1727 {
1728         return vhost_scsi_ioctl(f, ioctl, (unsigned long)compat_ptr(arg));
1729 }
1730 #endif
1731
1732 static const struct file_operations vhost_scsi_fops = {
1733         .owner          = THIS_MODULE,
1734         .release        = vhost_scsi_release,
1735         .unlocked_ioctl = vhost_scsi_ioctl,
1736 #ifdef CONFIG_COMPAT
1737         .compat_ioctl   = vhost_scsi_compat_ioctl,
1738 #endif
1739         .open           = vhost_scsi_open,
1740         .llseek         = noop_llseek,
1741 };
1742
1743 static struct miscdevice vhost_scsi_misc = {
1744         MISC_DYNAMIC_MINOR,
1745         "vhost-scsi",
1746         &vhost_scsi_fops,
1747 };
1748
1749 static int __init vhost_scsi_register(void)
1750 {
1751         return misc_register(&vhost_scsi_misc);
1752 }
1753
1754 static void vhost_scsi_deregister(void)
1755 {
1756         misc_deregister(&vhost_scsi_misc);
1757 }
1758
1759 static char *vhost_scsi_dump_proto_id(struct vhost_scsi_tport *tport)
1760 {
1761         switch (tport->tport_proto_id) {
1762         case SCSI_PROTOCOL_SAS:
1763                 return "SAS";
1764         case SCSI_PROTOCOL_FCP:
1765                 return "FCP";
1766         case SCSI_PROTOCOL_ISCSI:
1767                 return "iSCSI";
1768         default:
1769                 break;
1770         }
1771
1772         return "Unknown";
1773 }
1774
1775 static void
1776 vhost_scsi_do_plug(struct vhost_scsi_tpg *tpg,
1777                   struct se_lun *lun, bool plug)
1778 {
1779
1780         struct vhost_scsi *vs = tpg->vhost_scsi;
1781         struct vhost_virtqueue *vq;
1782         u32 reason;
1783
1784         if (!vs)
1785                 return;
1786
1787         mutex_lock(&vs->dev.mutex);
1788
1789         if (plug)
1790                 reason = VIRTIO_SCSI_EVT_RESET_RESCAN;
1791         else
1792                 reason = VIRTIO_SCSI_EVT_RESET_REMOVED;
1793
1794         vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
1795         mutex_lock(&vq->mutex);
1796         if (vhost_has_feature(vq, VIRTIO_SCSI_F_HOTPLUG))
1797                 vhost_scsi_send_evt(vs, tpg, lun,
1798                                    VIRTIO_SCSI_T_TRANSPORT_RESET, reason);
1799         mutex_unlock(&vq->mutex);
1800         mutex_unlock(&vs->dev.mutex);
1801 }
1802
1803 static void vhost_scsi_hotplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun)
1804 {
1805         vhost_scsi_do_plug(tpg, lun, true);
1806 }
1807
1808 static void vhost_scsi_hotunplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun)
1809 {
1810         vhost_scsi_do_plug(tpg, lun, false);
1811 }
1812
1813 static int vhost_scsi_port_link(struct se_portal_group *se_tpg,
1814                                struct se_lun *lun)
1815 {
1816         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
1817                                 struct vhost_scsi_tpg, se_tpg);
1818
1819         mutex_lock(&vhost_scsi_mutex);
1820
1821         mutex_lock(&tpg->tv_tpg_mutex);
1822         tpg->tv_tpg_port_count++;
1823         mutex_unlock(&tpg->tv_tpg_mutex);
1824
1825         vhost_scsi_hotplug(tpg, lun);
1826
1827         mutex_unlock(&vhost_scsi_mutex);
1828
1829         return 0;
1830 }
1831
1832 static void vhost_scsi_port_unlink(struct se_portal_group *se_tpg,
1833                                   struct se_lun *lun)
1834 {
1835         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
1836                                 struct vhost_scsi_tpg, se_tpg);
1837
1838         mutex_lock(&vhost_scsi_mutex);
1839
1840         mutex_lock(&tpg->tv_tpg_mutex);
1841         tpg->tv_tpg_port_count--;
1842         mutex_unlock(&tpg->tv_tpg_mutex);
1843
1844         vhost_scsi_hotunplug(tpg, lun);
1845
1846         mutex_unlock(&vhost_scsi_mutex);
1847 }
1848
1849 static void vhost_scsi_free_cmd_map_res(struct se_session *se_sess)
1850 {
1851         struct vhost_scsi_cmd *tv_cmd;
1852         unsigned int i;
1853
1854         if (!se_sess->sess_cmd_map)
1855                 return;
1856
1857         for (i = 0; i < VHOST_SCSI_DEFAULT_TAGS; i++) {
1858                 tv_cmd = &((struct vhost_scsi_cmd *)se_sess->sess_cmd_map)[i];
1859
1860                 kfree(tv_cmd->tvc_sgl);
1861                 kfree(tv_cmd->tvc_prot_sgl);
1862                 kfree(tv_cmd->tvc_upages);
1863         }
1864 }
1865
1866 static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_store(
1867                 struct config_item *item, const char *page, size_t count)
1868 {
1869         struct se_portal_group *se_tpg = attrib_to_tpg(item);
1870         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
1871                                 struct vhost_scsi_tpg, se_tpg);
1872         unsigned long val;
1873         int ret = kstrtoul(page, 0, &val);
1874
1875         if (ret) {
1876                 pr_err("kstrtoul() returned %d for fabric_prot_type\n", ret);
1877                 return ret;
1878         }
1879         if (val != 0 && val != 1 && val != 3) {
1880                 pr_err("Invalid vhost_scsi fabric_prot_type: %lu\n", val);
1881                 return -EINVAL;
1882         }
1883         tpg->tv_fabric_prot_type = val;
1884
1885         return count;
1886 }
1887
1888 static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_show(
1889                 struct config_item *item, char *page)
1890 {
1891         struct se_portal_group *se_tpg = attrib_to_tpg(item);
1892         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
1893                                 struct vhost_scsi_tpg, se_tpg);
1894
1895         return sprintf(page, "%d\n", tpg->tv_fabric_prot_type);
1896 }
1897
1898 CONFIGFS_ATTR(vhost_scsi_tpg_attrib_, fabric_prot_type);
1899
1900 static struct configfs_attribute *vhost_scsi_tpg_attrib_attrs[] = {
1901         &vhost_scsi_tpg_attrib_attr_fabric_prot_type,
1902         NULL,
1903 };
1904
1905 static int vhost_scsi_nexus_cb(struct se_portal_group *se_tpg,
1906                                struct se_session *se_sess, void *p)
1907 {
1908         struct vhost_scsi_cmd *tv_cmd;
1909         unsigned int i;
1910
1911         for (i = 0; i < VHOST_SCSI_DEFAULT_TAGS; i++) {
1912                 tv_cmd = &((struct vhost_scsi_cmd *)se_sess->sess_cmd_map)[i];
1913
1914                 tv_cmd->tvc_sgl = kcalloc(VHOST_SCSI_PREALLOC_SGLS,
1915                                           sizeof(struct scatterlist),
1916                                           GFP_KERNEL);
1917                 if (!tv_cmd->tvc_sgl) {
1918                         pr_err("Unable to allocate tv_cmd->tvc_sgl\n");
1919                         goto out;
1920                 }
1921
1922                 tv_cmd->tvc_upages = kcalloc(VHOST_SCSI_PREALLOC_UPAGES,
1923                                              sizeof(struct page *),
1924                                              GFP_KERNEL);
1925                 if (!tv_cmd->tvc_upages) {
1926                         pr_err("Unable to allocate tv_cmd->tvc_upages\n");
1927                         goto out;
1928                 }
1929
1930                 tv_cmd->tvc_prot_sgl = kcalloc(VHOST_SCSI_PREALLOC_PROT_SGLS,
1931                                                sizeof(struct scatterlist),
1932                                                GFP_KERNEL);
1933                 if (!tv_cmd->tvc_prot_sgl) {
1934                         pr_err("Unable to allocate tv_cmd->tvc_prot_sgl\n");
1935                         goto out;
1936                 }
1937         }
1938         return 0;
1939 out:
1940         vhost_scsi_free_cmd_map_res(se_sess);
1941         return -ENOMEM;
1942 }
1943
1944 static int vhost_scsi_make_nexus(struct vhost_scsi_tpg *tpg,
1945                                 const char *name)
1946 {
1947         struct vhost_scsi_nexus *tv_nexus;
1948
1949         mutex_lock(&tpg->tv_tpg_mutex);
1950         if (tpg->tpg_nexus) {
1951                 mutex_unlock(&tpg->tv_tpg_mutex);
1952                 pr_debug("tpg->tpg_nexus already exists\n");
1953                 return -EEXIST;
1954         }
1955
1956         tv_nexus = kzalloc(sizeof(*tv_nexus), GFP_KERNEL);
1957         if (!tv_nexus) {
1958                 mutex_unlock(&tpg->tv_tpg_mutex);
1959                 pr_err("Unable to allocate struct vhost_scsi_nexus\n");
1960                 return -ENOMEM;
1961         }
1962         /*
1963          * Since we are running in 'demo mode' this call with generate a
1964          * struct se_node_acl for the vhost_scsi struct se_portal_group with
1965          * the SCSI Initiator port name of the passed configfs group 'name'.
1966          */
1967         tv_nexus->tvn_se_sess = target_setup_session(&tpg->se_tpg,
1968                                         VHOST_SCSI_DEFAULT_TAGS,
1969                                         sizeof(struct vhost_scsi_cmd),
1970                                         TARGET_PROT_DIN_PASS | TARGET_PROT_DOUT_PASS,
1971                                         (unsigned char *)name, tv_nexus,
1972                                         vhost_scsi_nexus_cb);
1973         if (IS_ERR(tv_nexus->tvn_se_sess)) {
1974                 mutex_unlock(&tpg->tv_tpg_mutex);
1975                 kfree(tv_nexus);
1976                 return -ENOMEM;
1977         }
1978         tpg->tpg_nexus = tv_nexus;
1979
1980         mutex_unlock(&tpg->tv_tpg_mutex);
1981         return 0;
1982 }
1983
1984 static int vhost_scsi_drop_nexus(struct vhost_scsi_tpg *tpg)
1985 {
1986         struct se_session *se_sess;
1987         struct vhost_scsi_nexus *tv_nexus;
1988
1989         mutex_lock(&tpg->tv_tpg_mutex);
1990         tv_nexus = tpg->tpg_nexus;
1991         if (!tv_nexus) {
1992                 mutex_unlock(&tpg->tv_tpg_mutex);
1993                 return -ENODEV;
1994         }
1995
1996         se_sess = tv_nexus->tvn_se_sess;
1997         if (!se_sess) {
1998                 mutex_unlock(&tpg->tv_tpg_mutex);
1999                 return -ENODEV;
2000         }
2001
2002         if (tpg->tv_tpg_port_count != 0) {
2003                 mutex_unlock(&tpg->tv_tpg_mutex);
2004                 pr_err("Unable to remove TCM_vhost I_T Nexus with"
2005                         " active TPG port count: %d\n",
2006                         tpg->tv_tpg_port_count);
2007                 return -EBUSY;
2008         }
2009
2010         if (tpg->tv_tpg_vhost_count != 0) {
2011                 mutex_unlock(&tpg->tv_tpg_mutex);
2012                 pr_err("Unable to remove TCM_vhost I_T Nexus with"
2013                         " active TPG vhost count: %d\n",
2014                         tpg->tv_tpg_vhost_count);
2015                 return -EBUSY;
2016         }
2017
2018         pr_debug("TCM_vhost_ConfigFS: Removing I_T Nexus to emulated"
2019                 " %s Initiator Port: %s\n", vhost_scsi_dump_proto_id(tpg->tport),
2020                 tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
2021
2022         vhost_scsi_free_cmd_map_res(se_sess);
2023         /*
2024          * Release the SCSI I_T Nexus to the emulated vhost Target Port
2025          */
2026         target_remove_session(se_sess);
2027         tpg->tpg_nexus = NULL;
2028         mutex_unlock(&tpg->tv_tpg_mutex);
2029
2030         kfree(tv_nexus);
2031         return 0;
2032 }
2033
2034 static ssize_t vhost_scsi_tpg_nexus_show(struct config_item *item, char *page)
2035 {
2036         struct se_portal_group *se_tpg = to_tpg(item);
2037         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2038                                 struct vhost_scsi_tpg, se_tpg);
2039         struct vhost_scsi_nexus *tv_nexus;
2040         ssize_t ret;
2041
2042         mutex_lock(&tpg->tv_tpg_mutex);
2043         tv_nexus = tpg->tpg_nexus;
2044         if (!tv_nexus) {
2045                 mutex_unlock(&tpg->tv_tpg_mutex);
2046                 return -ENODEV;
2047         }
2048         ret = snprintf(page, PAGE_SIZE, "%s\n",
2049                         tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
2050         mutex_unlock(&tpg->tv_tpg_mutex);
2051
2052         return ret;
2053 }
2054
2055 static ssize_t vhost_scsi_tpg_nexus_store(struct config_item *item,
2056                 const char *page, size_t count)
2057 {
2058         struct se_portal_group *se_tpg = to_tpg(item);
2059         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2060                                 struct vhost_scsi_tpg, se_tpg);
2061         struct vhost_scsi_tport *tport_wwn = tpg->tport;
2062         unsigned char i_port[VHOST_SCSI_NAMELEN], *ptr, *port_ptr;
2063         int ret;
2064         /*
2065          * Shutdown the active I_T nexus if 'NULL' is passed..
2066          */
2067         if (!strncmp(page, "NULL", 4)) {
2068                 ret = vhost_scsi_drop_nexus(tpg);
2069                 return (!ret) ? count : ret;
2070         }
2071         /*
2072          * Otherwise make sure the passed virtual Initiator port WWN matches
2073          * the fabric protocol_id set in vhost_scsi_make_tport(), and call
2074          * vhost_scsi_make_nexus().
2075          */
2076         if (strlen(page) >= VHOST_SCSI_NAMELEN) {
2077                 pr_err("Emulated NAA Sas Address: %s, exceeds"
2078                                 " max: %d\n", page, VHOST_SCSI_NAMELEN);
2079                 return -EINVAL;
2080         }
2081         snprintf(&i_port[0], VHOST_SCSI_NAMELEN, "%s", page);
2082
2083         ptr = strstr(i_port, "naa.");
2084         if (ptr) {
2085                 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_SAS) {
2086                         pr_err("Passed SAS Initiator Port %s does not"
2087                                 " match target port protoid: %s\n", i_port,
2088                                 vhost_scsi_dump_proto_id(tport_wwn));
2089                         return -EINVAL;
2090                 }
2091                 port_ptr = &i_port[0];
2092                 goto check_newline;
2093         }
2094         ptr = strstr(i_port, "fc.");
2095         if (ptr) {
2096                 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_FCP) {
2097                         pr_err("Passed FCP Initiator Port %s does not"
2098                                 " match target port protoid: %s\n", i_port,
2099                                 vhost_scsi_dump_proto_id(tport_wwn));
2100                         return -EINVAL;
2101                 }
2102                 port_ptr = &i_port[3]; /* Skip over "fc." */
2103                 goto check_newline;
2104         }
2105         ptr = strstr(i_port, "iqn.");
2106         if (ptr) {
2107                 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_ISCSI) {
2108                         pr_err("Passed iSCSI Initiator Port %s does not"
2109                                 " match target port protoid: %s\n", i_port,
2110                                 vhost_scsi_dump_proto_id(tport_wwn));
2111                         return -EINVAL;
2112                 }
2113                 port_ptr = &i_port[0];
2114                 goto check_newline;
2115         }
2116         pr_err("Unable to locate prefix for emulated Initiator Port:"
2117                         " %s\n", i_port);
2118         return -EINVAL;
2119         /*
2120          * Clear any trailing newline for the NAA WWN
2121          */
2122 check_newline:
2123         if (i_port[strlen(i_port)-1] == '\n')
2124                 i_port[strlen(i_port)-1] = '\0';
2125
2126         ret = vhost_scsi_make_nexus(tpg, port_ptr);
2127         if (ret < 0)
2128                 return ret;
2129
2130         return count;
2131 }
2132
2133 CONFIGFS_ATTR(vhost_scsi_tpg_, nexus);
2134
2135 static struct configfs_attribute *vhost_scsi_tpg_attrs[] = {
2136         &vhost_scsi_tpg_attr_nexus,
2137         NULL,
2138 };
2139
2140 static struct se_portal_group *
2141 vhost_scsi_make_tpg(struct se_wwn *wwn, const char *name)
2142 {
2143         struct vhost_scsi_tport *tport = container_of(wwn,
2144                         struct vhost_scsi_tport, tport_wwn);
2145
2146         struct vhost_scsi_tpg *tpg;
2147         u16 tpgt;
2148         int ret;
2149
2150         if (strstr(name, "tpgt_") != name)
2151                 return ERR_PTR(-EINVAL);
2152         if (kstrtou16(name + 5, 10, &tpgt) || tpgt >= VHOST_SCSI_MAX_TARGET)
2153                 return ERR_PTR(-EINVAL);
2154
2155         tpg = kzalloc(sizeof(*tpg), GFP_KERNEL);
2156         if (!tpg) {
2157                 pr_err("Unable to allocate struct vhost_scsi_tpg");
2158                 return ERR_PTR(-ENOMEM);
2159         }
2160         mutex_init(&tpg->tv_tpg_mutex);
2161         INIT_LIST_HEAD(&tpg->tv_tpg_list);
2162         tpg->tport = tport;
2163         tpg->tport_tpgt = tpgt;
2164
2165         ret = core_tpg_register(wwn, &tpg->se_tpg, tport->tport_proto_id);
2166         if (ret < 0) {
2167                 kfree(tpg);
2168                 return NULL;
2169         }
2170         mutex_lock(&vhost_scsi_mutex);
2171         list_add_tail(&tpg->tv_tpg_list, &vhost_scsi_list);
2172         mutex_unlock(&vhost_scsi_mutex);
2173
2174         return &tpg->se_tpg;
2175 }
2176
2177 static void vhost_scsi_drop_tpg(struct se_portal_group *se_tpg)
2178 {
2179         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2180                                 struct vhost_scsi_tpg, se_tpg);
2181
2182         mutex_lock(&vhost_scsi_mutex);
2183         list_del(&tpg->tv_tpg_list);
2184         mutex_unlock(&vhost_scsi_mutex);
2185         /*
2186          * Release the virtual I_T Nexus for this vhost TPG
2187          */
2188         vhost_scsi_drop_nexus(tpg);
2189         /*
2190          * Deregister the se_tpg from TCM..
2191          */
2192         core_tpg_deregister(se_tpg);
2193         kfree(tpg);
2194 }
2195
2196 static struct se_wwn *
2197 vhost_scsi_make_tport(struct target_fabric_configfs *tf,
2198                      struct config_group *group,
2199                      const char *name)
2200 {
2201         struct vhost_scsi_tport *tport;
2202         char *ptr;
2203         u64 wwpn = 0;
2204         int off = 0;
2205
2206         /* if (vhost_scsi_parse_wwn(name, &wwpn, 1) < 0)
2207                 return ERR_PTR(-EINVAL); */
2208
2209         tport = kzalloc(sizeof(*tport), GFP_KERNEL);
2210         if (!tport) {
2211                 pr_err("Unable to allocate struct vhost_scsi_tport");
2212                 return ERR_PTR(-ENOMEM);
2213         }
2214         tport->tport_wwpn = wwpn;
2215         /*
2216          * Determine the emulated Protocol Identifier and Target Port Name
2217          * based on the incoming configfs directory name.
2218          */
2219         ptr = strstr(name, "naa.");
2220         if (ptr) {
2221                 tport->tport_proto_id = SCSI_PROTOCOL_SAS;
2222                 goto check_len;
2223         }
2224         ptr = strstr(name, "fc.");
2225         if (ptr) {
2226                 tport->tport_proto_id = SCSI_PROTOCOL_FCP;
2227                 off = 3; /* Skip over "fc." */
2228                 goto check_len;
2229         }
2230         ptr = strstr(name, "iqn.");
2231         if (ptr) {
2232                 tport->tport_proto_id = SCSI_PROTOCOL_ISCSI;
2233                 goto check_len;
2234         }
2235
2236         pr_err("Unable to locate prefix for emulated Target Port:"
2237                         " %s\n", name);
2238         kfree(tport);
2239         return ERR_PTR(-EINVAL);
2240
2241 check_len:
2242         if (strlen(name) >= VHOST_SCSI_NAMELEN) {
2243                 pr_err("Emulated %s Address: %s, exceeds"
2244                         " max: %d\n", name, vhost_scsi_dump_proto_id(tport),
2245                         VHOST_SCSI_NAMELEN);
2246                 kfree(tport);
2247                 return ERR_PTR(-EINVAL);
2248         }
2249         snprintf(&tport->tport_name[0], VHOST_SCSI_NAMELEN, "%s", &name[off]);
2250
2251         pr_debug("TCM_VHost_ConfigFS: Allocated emulated Target"
2252                 " %s Address: %s\n", vhost_scsi_dump_proto_id(tport), name);
2253
2254         return &tport->tport_wwn;
2255 }
2256
2257 static void vhost_scsi_drop_tport(struct se_wwn *wwn)
2258 {
2259         struct vhost_scsi_tport *tport = container_of(wwn,
2260                                 struct vhost_scsi_tport, tport_wwn);
2261
2262         pr_debug("TCM_VHost_ConfigFS: Deallocating emulated Target"
2263                 " %s Address: %s\n", vhost_scsi_dump_proto_id(tport),
2264                 tport->tport_name);
2265
2266         kfree(tport);
2267 }
2268
2269 static ssize_t
2270 vhost_scsi_wwn_version_show(struct config_item *item, char *page)
2271 {
2272         return sprintf(page, "TCM_VHOST fabric module %s on %s/%s"
2273                 "on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname,
2274                 utsname()->machine);
2275 }
2276
2277 CONFIGFS_ATTR_RO(vhost_scsi_wwn_, version);
2278
2279 static struct configfs_attribute *vhost_scsi_wwn_attrs[] = {
2280         &vhost_scsi_wwn_attr_version,
2281         NULL,
2282 };
2283
2284 static const struct target_core_fabric_ops vhost_scsi_ops = {
2285         .module                         = THIS_MODULE,
2286         .fabric_name                    = "vhost",
2287         .tpg_get_wwn                    = vhost_scsi_get_fabric_wwn,
2288         .tpg_get_tag                    = vhost_scsi_get_tpgt,
2289         .tpg_check_demo_mode            = vhost_scsi_check_true,
2290         .tpg_check_demo_mode_cache      = vhost_scsi_check_true,
2291         .tpg_check_demo_mode_write_protect = vhost_scsi_check_false,
2292         .tpg_check_prod_mode_write_protect = vhost_scsi_check_false,
2293         .tpg_check_prot_fabric_only     = vhost_scsi_check_prot_fabric_only,
2294         .tpg_get_inst_index             = vhost_scsi_tpg_get_inst_index,
2295         .release_cmd                    = vhost_scsi_release_cmd,
2296         .check_stop_free                = vhost_scsi_check_stop_free,
2297         .sess_get_index                 = vhost_scsi_sess_get_index,
2298         .sess_get_initiator_sid         = NULL,
2299         .write_pending                  = vhost_scsi_write_pending,
2300         .set_default_node_attributes    = vhost_scsi_set_default_node_attrs,
2301         .get_cmd_state                  = vhost_scsi_get_cmd_state,
2302         .queue_data_in                  = vhost_scsi_queue_data_in,
2303         .queue_status                   = vhost_scsi_queue_status,
2304         .queue_tm_rsp                   = vhost_scsi_queue_tm_rsp,
2305         .aborted_task                   = vhost_scsi_aborted_task,
2306         /*
2307          * Setup callers for generic logic in target_core_fabric_configfs.c
2308          */
2309         .fabric_make_wwn                = vhost_scsi_make_tport,
2310         .fabric_drop_wwn                = vhost_scsi_drop_tport,
2311         .fabric_make_tpg                = vhost_scsi_make_tpg,
2312         .fabric_drop_tpg                = vhost_scsi_drop_tpg,
2313         .fabric_post_link               = vhost_scsi_port_link,
2314         .fabric_pre_unlink              = vhost_scsi_port_unlink,
2315
2316         .tfc_wwn_attrs                  = vhost_scsi_wwn_attrs,
2317         .tfc_tpg_base_attrs             = vhost_scsi_tpg_attrs,
2318         .tfc_tpg_attrib_attrs           = vhost_scsi_tpg_attrib_attrs,
2319 };
2320
2321 static int __init vhost_scsi_init(void)
2322 {
2323         int ret = -ENOMEM;
2324
2325         pr_debug("TCM_VHOST fabric module %s on %s/%s"
2326                 " on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname,
2327                 utsname()->machine);
2328
2329         /*
2330          * Use our own dedicated workqueue for submitting I/O into
2331          * target core to avoid contention within system_wq.
2332          */
2333         vhost_scsi_workqueue = alloc_workqueue("vhost_scsi", 0, 0);
2334         if (!vhost_scsi_workqueue)
2335                 goto out;
2336
2337         ret = vhost_scsi_register();
2338         if (ret < 0)
2339                 goto out_destroy_workqueue;
2340
2341         ret = target_register_template(&vhost_scsi_ops);
2342         if (ret < 0)
2343                 goto out_vhost_scsi_deregister;
2344
2345         return 0;
2346
2347 out_vhost_scsi_deregister:
2348         vhost_scsi_deregister();
2349 out_destroy_workqueue:
2350         destroy_workqueue(vhost_scsi_workqueue);
2351 out:
2352         return ret;
2353 };
2354
2355 static void vhost_scsi_exit(void)
2356 {
2357         target_unregister_template(&vhost_scsi_ops);
2358         vhost_scsi_deregister();
2359         destroy_workqueue(vhost_scsi_workqueue);
2360 };
2361
2362 MODULE_DESCRIPTION("VHOST_SCSI series fabric driver");
2363 MODULE_ALIAS("tcm_vhost");
2364 MODULE_LICENSE("GPL");
2365 module_init(vhost_scsi_init);
2366 module_exit(vhost_scsi_exit);