2 * Copyright (c) 2016 Avago Technologies. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful.
9 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES,
10 * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
11 * PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED, EXCEPT TO
12 * THE EXTENT THAT SUCH DISCLAIMERS ARE HELD TO BE LEGALLY INVALID.
13 * See the GNU General Public License for more details, a copy of which
14 * can be found in the file COPYING included with this package
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18 #include <linux/module.h>
19 #include <linux/slab.h>
20 #include <linux/blk-mq.h>
21 #include <linux/parser.h>
22 #include <linux/random.h>
23 #include <uapi/scsi/fc/fc_fs.h>
24 #include <uapi/scsi/fc/fc_els.h>
27 #include <linux/nvme-fc-driver.h>
28 #include <linux/nvme-fc.h>
31 /* *************************** Data Structures/Defines ****************** */
34 #define NVMET_LS_CTX_COUNT 4
36 /* for this implementation, assume small single frame rqst/rsp */
37 #define NVME_FC_MAX_LS_BUFFER_SIZE 2048
39 struct nvmet_fc_tgtport;
40 struct nvmet_fc_tgt_assoc;
42 struct nvmet_fc_ls_iod {
43 struct nvmefc_tgt_ls_req *lsreq;
44 struct nvmefc_tgt_fcp_req *fcpreq; /* only if RS */
46 struct list_head ls_list; /* tgtport->ls_list */
48 struct nvmet_fc_tgtport *tgtport;
49 struct nvmet_fc_tgt_assoc *assoc;
56 struct scatterlist sg[2];
58 struct work_struct work;
59 } __aligned(sizeof(unsigned long long));
61 #define NVMET_FC_MAX_SEQ_LENGTH (256 * 1024)
62 #define NVMET_FC_MAX_XFR_SGENTS (NVMET_FC_MAX_SEQ_LENGTH / PAGE_SIZE)
64 enum nvmet_fcp_datadir {
71 struct nvmet_fc_fcp_iod {
72 struct nvmefc_tgt_fcp_req *fcpreq;
74 struct nvme_fc_cmd_iu cmdiubuf;
75 struct nvme_fc_ersp_iu rspiubuf;
77 struct scatterlist *data_sg;
81 enum nvmet_fcp_datadir io_dir;
89 struct work_struct work;
90 struct work_struct done_work;
92 struct nvmet_fc_tgtport *tgtport;
93 struct nvmet_fc_tgt_queue *queue;
95 struct list_head fcp_list; /* tgtport->fcp_list */
98 struct nvmet_fc_tgtport {
100 struct nvmet_fc_target_port fc_target_port;
102 struct list_head tgt_list; /* nvmet_fc_target_list */
103 struct device *dev; /* dev for dma mapping */
104 struct nvmet_fc_target_template *ops;
106 struct nvmet_fc_ls_iod *iod;
108 struct list_head ls_list;
109 struct list_head ls_busylist;
110 struct list_head assoc_list;
111 struct ida assoc_cnt;
112 struct nvmet_port *port;
117 struct nvmet_fc_defer_fcp_req {
118 struct list_head req_list;
119 struct nvmefc_tgt_fcp_req *fcp_req;
122 struct nvmet_fc_tgt_queue {
134 struct nvmet_port *port;
135 struct nvmet_cq nvme_cq;
136 struct nvmet_sq nvme_sq;
137 struct nvmet_fc_tgt_assoc *assoc;
138 struct nvmet_fc_fcp_iod *fod; /* array of fcp_iods */
139 struct list_head fod_list;
140 struct list_head pending_cmd_list;
141 struct list_head avail_defer_list;
142 struct workqueue_struct *work_q;
144 } __aligned(sizeof(unsigned long long));
146 struct nvmet_fc_tgt_assoc {
149 struct nvmet_fc_tgtport *tgtport;
150 struct list_head a_list;
151 struct nvmet_fc_tgt_queue *queues[NVMET_NR_QUEUES + 1];
157 nvmet_fc_iodnum(struct nvmet_fc_ls_iod *iodptr)
159 return (iodptr - iodptr->tgtport->iod);
163 nvmet_fc_fodnum(struct nvmet_fc_fcp_iod *fodptr)
165 return (fodptr - fodptr->queue->fod);
170 * Association and Connection IDs:
172 * Association ID will have random number in upper 6 bytes and zero
175 * Connection IDs will be Association ID with QID or'd in lower 2 bytes
177 * note: Association ID = Connection ID for queue 0
179 #define BYTES_FOR_QID sizeof(u16)
180 #define BYTES_FOR_QID_SHIFT (BYTES_FOR_QID * 8)
181 #define NVMET_FC_QUEUEID_MASK ((u64)((1 << BYTES_FOR_QID_SHIFT) - 1))
184 nvmet_fc_makeconnid(struct nvmet_fc_tgt_assoc *assoc, u16 qid)
186 return (assoc->association_id | qid);
190 nvmet_fc_getassociationid(u64 connectionid)
192 return connectionid & ~NVMET_FC_QUEUEID_MASK;
196 nvmet_fc_getqueueid(u64 connectionid)
198 return (u16)(connectionid & NVMET_FC_QUEUEID_MASK);
201 static inline struct nvmet_fc_tgtport *
202 targetport_to_tgtport(struct nvmet_fc_target_port *targetport)
204 return container_of(targetport, struct nvmet_fc_tgtport,
208 static inline struct nvmet_fc_fcp_iod *
209 nvmet_req_to_fod(struct nvmet_req *nvme_req)
211 return container_of(nvme_req, struct nvmet_fc_fcp_iod, req);
215 /* *************************** Globals **************************** */
218 static DEFINE_SPINLOCK(nvmet_fc_tgtlock);
220 static LIST_HEAD(nvmet_fc_target_list);
221 static DEFINE_IDA(nvmet_fc_tgtport_cnt);
224 static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work);
225 static void nvmet_fc_handle_fcp_rqst_work(struct work_struct *work);
226 static void nvmet_fc_fcp_rqst_op_done_work(struct work_struct *work);
227 static void nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc);
228 static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc);
229 static void nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue);
230 static int nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue);
231 static void nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport);
232 static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport);
233 static void nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
234 struct nvmet_fc_fcp_iod *fod);
237 /* *********************** FC-NVME DMA Handling **************************** */
240 * The fcloop device passes in a NULL device pointer. Real LLD's will
241 * pass in a valid device pointer. If NULL is passed to the dma mapping
242 * routines, depending on the platform, it may or may not succeed, and
246 * Wrapper all the dma routines and check the dev pointer.
248 * If simple mappings (return just a dma address, we'll noop them,
249 * returning a dma address of 0.
251 * On more complex mappings (dma_map_sg), a pseudo routine fills
252 * in the scatter list, setting all dma addresses to 0.
255 static inline dma_addr_t
256 fc_dma_map_single(struct device *dev, void *ptr, size_t size,
257 enum dma_data_direction dir)
259 return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
263 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
265 return dev ? dma_mapping_error(dev, dma_addr) : 0;
269 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
270 enum dma_data_direction dir)
273 dma_unmap_single(dev, addr, size, dir);
277 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
278 enum dma_data_direction dir)
281 dma_sync_single_for_cpu(dev, addr, size, dir);
285 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
286 enum dma_data_direction dir)
289 dma_sync_single_for_device(dev, addr, size, dir);
292 /* pseudo dma_map_sg call */
294 fc_map_sg(struct scatterlist *sg, int nents)
296 struct scatterlist *s;
299 WARN_ON(nents == 0 || sg[0].length == 0);
301 for_each_sg(sg, s, nents, i) {
303 #ifdef CONFIG_NEED_SG_DMA_LENGTH
304 s->dma_length = s->length;
311 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
312 enum dma_data_direction dir)
314 return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
318 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
319 enum dma_data_direction dir)
322 dma_unmap_sg(dev, sg, nents, dir);
326 /* *********************** FC-NVME Port Management ************************ */
330 nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
332 struct nvmet_fc_ls_iod *iod;
335 iod = kcalloc(NVMET_LS_CTX_COUNT, sizeof(struct nvmet_fc_ls_iod),
342 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
343 INIT_WORK(&iod->work, nvmet_fc_handle_ls_rqst_work);
344 iod->tgtport = tgtport;
345 list_add_tail(&iod->ls_list, &tgtport->ls_list);
347 iod->rqstbuf = kcalloc(2, NVME_FC_MAX_LS_BUFFER_SIZE,
352 iod->rspbuf = iod->rqstbuf + NVME_FC_MAX_LS_BUFFER_SIZE;
354 iod->rspdma = fc_dma_map_single(tgtport->dev, iod->rspbuf,
355 NVME_FC_MAX_LS_BUFFER_SIZE,
357 if (fc_dma_mapping_error(tgtport->dev, iod->rspdma))
365 list_del(&iod->ls_list);
366 for (iod--, i--; i >= 0; iod--, i--) {
367 fc_dma_unmap_single(tgtport->dev, iod->rspdma,
368 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
370 list_del(&iod->ls_list);
379 nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
381 struct nvmet_fc_ls_iod *iod = tgtport->iod;
384 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
385 fc_dma_unmap_single(tgtport->dev,
386 iod->rspdma, NVME_FC_MAX_LS_BUFFER_SIZE,
389 list_del(&iod->ls_list);
394 static struct nvmet_fc_ls_iod *
395 nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport *tgtport)
397 struct nvmet_fc_ls_iod *iod;
400 spin_lock_irqsave(&tgtport->lock, flags);
401 iod = list_first_entry_or_null(&tgtport->ls_list,
402 struct nvmet_fc_ls_iod, ls_list);
404 list_move_tail(&iod->ls_list, &tgtport->ls_busylist);
405 spin_unlock_irqrestore(&tgtport->lock, flags);
411 nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport *tgtport,
412 struct nvmet_fc_ls_iod *iod)
416 spin_lock_irqsave(&tgtport->lock, flags);
417 list_move(&iod->ls_list, &tgtport->ls_list);
418 spin_unlock_irqrestore(&tgtport->lock, flags);
422 nvmet_fc_prep_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
423 struct nvmet_fc_tgt_queue *queue)
425 struct nvmet_fc_fcp_iod *fod = queue->fod;
428 for (i = 0; i < queue->sqsize; fod++, i++) {
429 INIT_WORK(&fod->work, nvmet_fc_handle_fcp_rqst_work);
430 INIT_WORK(&fod->done_work, nvmet_fc_fcp_rqst_op_done_work);
431 fod->tgtport = tgtport;
435 fod->aborted = false;
437 list_add_tail(&fod->fcp_list, &queue->fod_list);
438 spin_lock_init(&fod->flock);
440 fod->rspdma = fc_dma_map_single(tgtport->dev, &fod->rspiubuf,
441 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
442 if (fc_dma_mapping_error(tgtport->dev, fod->rspdma)) {
443 list_del(&fod->fcp_list);
444 for (fod--, i--; i >= 0; fod--, i--) {
445 fc_dma_unmap_single(tgtport->dev, fod->rspdma,
446 sizeof(fod->rspiubuf),
449 list_del(&fod->fcp_list);
458 nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
459 struct nvmet_fc_tgt_queue *queue)
461 struct nvmet_fc_fcp_iod *fod = queue->fod;
464 for (i = 0; i < queue->sqsize; fod++, i++) {
466 fc_dma_unmap_single(tgtport->dev, fod->rspdma,
467 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
471 static struct nvmet_fc_fcp_iod *
472 nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue *queue)
474 struct nvmet_fc_fcp_iod *fod;
476 lockdep_assert_held(&queue->qlock);
478 fod = list_first_entry_or_null(&queue->fod_list,
479 struct nvmet_fc_fcp_iod, fcp_list);
481 list_del(&fod->fcp_list);
484 * no queue reference is taken, as it was taken by the
485 * queue lookup just prior to the allocation. The iod
486 * will "inherit" that reference.
494 nvmet_fc_queue_fcp_req(struct nvmet_fc_tgtport *tgtport,
495 struct nvmet_fc_tgt_queue *queue,
496 struct nvmefc_tgt_fcp_req *fcpreq)
498 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
501 * put all admin cmds on hw queue id 0. All io commands go to
502 * the respective hw queue based on a modulo basis
504 fcpreq->hwqid = queue->qid ?
505 ((queue->qid - 1) % tgtport->ops->max_hw_queues) : 0;
507 if (tgtport->ops->target_features & NVMET_FCTGTFEAT_CMD_IN_ISR)
508 queue_work_on(queue->cpu, queue->work_q, &fod->work);
510 nvmet_fc_handle_fcp_rqst(tgtport, fod);
514 nvmet_fc_free_fcp_iod(struct nvmet_fc_tgt_queue *queue,
515 struct nvmet_fc_fcp_iod *fod)
517 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
518 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
519 struct nvmet_fc_defer_fcp_req *deferfcp;
522 fc_dma_sync_single_for_cpu(tgtport->dev, fod->rspdma,
523 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
525 fcpreq->nvmet_fc_private = NULL;
529 fod->aborted = false;
530 fod->writedataactive = false;
533 tgtport->ops->fcp_req_release(&tgtport->fc_target_port, fcpreq);
535 spin_lock_irqsave(&queue->qlock, flags);
536 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
537 struct nvmet_fc_defer_fcp_req, req_list);
539 list_add_tail(&fod->fcp_list, &fod->queue->fod_list);
540 spin_unlock_irqrestore(&queue->qlock, flags);
542 /* Release reference taken at queue lookup and fod allocation */
543 nvmet_fc_tgt_q_put(queue);
547 /* Re-use the fod for the next pending cmd that was deferred */
548 list_del(&deferfcp->req_list);
550 fcpreq = deferfcp->fcp_req;
552 /* deferfcp can be reused for another IO at a later date */
553 list_add_tail(&deferfcp->req_list, &queue->avail_defer_list);
555 spin_unlock_irqrestore(&queue->qlock, flags);
557 /* Save NVME CMD IO in fod */
558 memcpy(&fod->cmdiubuf, fcpreq->rspaddr, fcpreq->rsplen);
560 /* Setup new fcpreq to be processed */
561 fcpreq->rspaddr = NULL;
563 fcpreq->nvmet_fc_private = fod;
564 fod->fcpreq = fcpreq;
567 /* inform LLDD IO is now being processed */
568 tgtport->ops->defer_rcv(&tgtport->fc_target_port, fcpreq);
570 /* Submit deferred IO for processing */
571 nvmet_fc_queue_fcp_req(tgtport, queue, fcpreq);
574 * Leave the queue lookup get reference taken when
575 * fod was originally allocated.
580 nvmet_fc_queue_to_cpu(struct nvmet_fc_tgtport *tgtport, int qid)
584 if (tgtport->ops->max_hw_queues == 1)
585 return WORK_CPU_UNBOUND;
587 /* Simple cpu selection based on qid modulo active cpu count */
588 idx = !qid ? 0 : (qid - 1) % num_active_cpus();
590 /* find the n'th active cpu */
591 for (cpu = 0, cnt = 0; ; ) {
592 if (cpu_active(cpu)) {
597 cpu = (cpu + 1) % num_possible_cpus();
603 static struct nvmet_fc_tgt_queue *
604 nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc,
607 struct nvmet_fc_tgt_queue *queue;
611 if (qid > NVMET_NR_QUEUES)
614 queue = kzalloc((sizeof(*queue) +
615 (sizeof(struct nvmet_fc_fcp_iod) * sqsize)),
620 if (!nvmet_fc_tgt_a_get(assoc))
623 queue->work_q = alloc_workqueue("ntfc%d.%d.%d", 0, 0,
624 assoc->tgtport->fc_target_port.port_num,
629 queue->fod = (struct nvmet_fc_fcp_iod *)&queue[1];
631 queue->sqsize = sqsize;
632 queue->assoc = assoc;
633 queue->port = assoc->tgtport->port;
634 queue->cpu = nvmet_fc_queue_to_cpu(assoc->tgtport, qid);
635 INIT_LIST_HEAD(&queue->fod_list);
636 INIT_LIST_HEAD(&queue->avail_defer_list);
637 INIT_LIST_HEAD(&queue->pending_cmd_list);
638 atomic_set(&queue->connected, 0);
639 atomic_set(&queue->sqtail, 0);
640 atomic_set(&queue->rsn, 1);
641 atomic_set(&queue->zrspcnt, 0);
642 spin_lock_init(&queue->qlock);
643 kref_init(&queue->ref);
645 nvmet_fc_prep_fcp_iodlist(assoc->tgtport, queue);
647 ret = nvmet_sq_init(&queue->nvme_sq);
649 goto out_fail_iodlist;
651 WARN_ON(assoc->queues[qid]);
652 spin_lock_irqsave(&assoc->tgtport->lock, flags);
653 assoc->queues[qid] = queue;
654 spin_unlock_irqrestore(&assoc->tgtport->lock, flags);
659 nvmet_fc_destroy_fcp_iodlist(assoc->tgtport, queue);
660 destroy_workqueue(queue->work_q);
662 nvmet_fc_tgt_a_put(assoc);
670 nvmet_fc_tgt_queue_free(struct kref *ref)
672 struct nvmet_fc_tgt_queue *queue =
673 container_of(ref, struct nvmet_fc_tgt_queue, ref);
676 spin_lock_irqsave(&queue->assoc->tgtport->lock, flags);
677 queue->assoc->queues[queue->qid] = NULL;
678 spin_unlock_irqrestore(&queue->assoc->tgtport->lock, flags);
680 nvmet_fc_destroy_fcp_iodlist(queue->assoc->tgtport, queue);
682 nvmet_fc_tgt_a_put(queue->assoc);
684 destroy_workqueue(queue->work_q);
690 nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue)
692 kref_put(&queue->ref, nvmet_fc_tgt_queue_free);
696 nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue)
698 return kref_get_unless_zero(&queue->ref);
703 nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue)
705 struct nvmet_fc_tgtport *tgtport = queue->assoc->tgtport;
706 struct nvmet_fc_fcp_iod *fod = queue->fod;
707 struct nvmet_fc_defer_fcp_req *deferfcp, *tempptr;
709 int i, writedataactive;
712 disconnect = atomic_xchg(&queue->connected, 0);
714 spin_lock_irqsave(&queue->qlock, flags);
715 /* about outstanding io's */
716 for (i = 0; i < queue->sqsize; fod++, i++) {
718 spin_lock(&fod->flock);
720 writedataactive = fod->writedataactive;
721 spin_unlock(&fod->flock);
723 * only call lldd abort routine if waiting for
724 * writedata. other outstanding ops should finish
727 if (writedataactive) {
728 spin_lock(&fod->flock);
730 spin_unlock(&fod->flock);
731 tgtport->ops->fcp_abort(
732 &tgtport->fc_target_port, fod->fcpreq);
737 /* Cleanup defer'ed IOs in queue */
738 list_for_each_entry_safe(deferfcp, tempptr, &queue->avail_defer_list,
740 list_del(&deferfcp->req_list);
745 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
746 struct nvmet_fc_defer_fcp_req, req_list);
750 list_del(&deferfcp->req_list);
751 spin_unlock_irqrestore(&queue->qlock, flags);
753 tgtport->ops->defer_rcv(&tgtport->fc_target_port,
756 tgtport->ops->fcp_abort(&tgtport->fc_target_port,
759 tgtport->ops->fcp_req_release(&tgtport->fc_target_port,
764 spin_lock_irqsave(&queue->qlock, flags);
766 spin_unlock_irqrestore(&queue->qlock, flags);
768 flush_workqueue(queue->work_q);
771 nvmet_sq_destroy(&queue->nvme_sq);
773 nvmet_fc_tgt_q_put(queue);
776 static struct nvmet_fc_tgt_queue *
777 nvmet_fc_find_target_queue(struct nvmet_fc_tgtport *tgtport,
780 struct nvmet_fc_tgt_assoc *assoc;
781 struct nvmet_fc_tgt_queue *queue;
782 u64 association_id = nvmet_fc_getassociationid(connection_id);
783 u16 qid = nvmet_fc_getqueueid(connection_id);
786 if (qid > NVMET_NR_QUEUES)
789 spin_lock_irqsave(&tgtport->lock, flags);
790 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
791 if (association_id == assoc->association_id) {
792 queue = assoc->queues[qid];
794 (!atomic_read(&queue->connected) ||
795 !nvmet_fc_tgt_q_get(queue)))
797 spin_unlock_irqrestore(&tgtport->lock, flags);
801 spin_unlock_irqrestore(&tgtport->lock, flags);
805 static struct nvmet_fc_tgt_assoc *
806 nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport)
808 struct nvmet_fc_tgt_assoc *assoc, *tmpassoc;
812 bool needrandom = true;
814 assoc = kzalloc(sizeof(*assoc), GFP_KERNEL);
818 idx = ida_simple_get(&tgtport->assoc_cnt, 0, 0, GFP_KERNEL);
822 if (!nvmet_fc_tgtport_get(tgtport))
825 assoc->tgtport = tgtport;
827 INIT_LIST_HEAD(&assoc->a_list);
828 kref_init(&assoc->ref);
831 get_random_bytes(&ran, sizeof(ran) - BYTES_FOR_QID);
832 ran = ran << BYTES_FOR_QID_SHIFT;
834 spin_lock_irqsave(&tgtport->lock, flags);
836 list_for_each_entry(tmpassoc, &tgtport->assoc_list, a_list)
837 if (ran == tmpassoc->association_id) {
842 assoc->association_id = ran;
843 list_add_tail(&assoc->a_list, &tgtport->assoc_list);
845 spin_unlock_irqrestore(&tgtport->lock, flags);
851 ida_simple_remove(&tgtport->assoc_cnt, idx);
858 nvmet_fc_target_assoc_free(struct kref *ref)
860 struct nvmet_fc_tgt_assoc *assoc =
861 container_of(ref, struct nvmet_fc_tgt_assoc, ref);
862 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
865 spin_lock_irqsave(&tgtport->lock, flags);
866 list_del(&assoc->a_list);
867 spin_unlock_irqrestore(&tgtport->lock, flags);
868 ida_simple_remove(&tgtport->assoc_cnt, assoc->a_id);
870 nvmet_fc_tgtport_put(tgtport);
874 nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc)
876 kref_put(&assoc->ref, nvmet_fc_target_assoc_free);
880 nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc)
882 return kref_get_unless_zero(&assoc->ref);
886 nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc)
888 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
889 struct nvmet_fc_tgt_queue *queue;
893 spin_lock_irqsave(&tgtport->lock, flags);
894 for (i = NVMET_NR_QUEUES; i >= 0; i--) {
895 queue = assoc->queues[i];
897 if (!nvmet_fc_tgt_q_get(queue))
899 spin_unlock_irqrestore(&tgtport->lock, flags);
900 nvmet_fc_delete_target_queue(queue);
901 nvmet_fc_tgt_q_put(queue);
902 spin_lock_irqsave(&tgtport->lock, flags);
905 spin_unlock_irqrestore(&tgtport->lock, flags);
907 nvmet_fc_tgt_a_put(assoc);
910 static struct nvmet_fc_tgt_assoc *
911 nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport *tgtport,
914 struct nvmet_fc_tgt_assoc *assoc;
915 struct nvmet_fc_tgt_assoc *ret = NULL;
918 spin_lock_irqsave(&tgtport->lock, flags);
919 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
920 if (association_id == assoc->association_id) {
922 nvmet_fc_tgt_a_get(assoc);
926 spin_unlock_irqrestore(&tgtport->lock, flags);
933 * nvme_fc_register_targetport - transport entry point called by an
934 * LLDD to register the existence of a local
935 * NVME subystem FC port.
936 * @pinfo: pointer to information about the port to be registered
937 * @template: LLDD entrypoints and operational parameters for the port
938 * @dev: physical hardware device node port corresponds to. Will be
939 * used for DMA mappings
940 * @portptr: pointer to a local port pointer. Upon success, the routine
941 * will allocate a nvme_fc_local_port structure and place its
942 * address in the local port pointer. Upon failure, local port
943 * pointer will be set to NULL.
946 * a completion status. Must be 0 upon success; a negative errno
947 * (ex: -ENXIO) upon failure.
950 nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo,
951 struct nvmet_fc_target_template *template,
953 struct nvmet_fc_target_port **portptr)
955 struct nvmet_fc_tgtport *newrec;
959 if (!template->xmt_ls_rsp || !template->fcp_op ||
960 !template->fcp_abort ||
961 !template->fcp_req_release || !template->targetport_delete ||
962 !template->max_hw_queues || !template->max_sgl_segments ||
963 !template->max_dif_sgl_segments || !template->dma_boundary) {
965 goto out_regtgt_failed;
968 newrec = kzalloc((sizeof(*newrec) + template->target_priv_sz),
972 goto out_regtgt_failed;
975 idx = ida_simple_get(&nvmet_fc_tgtport_cnt, 0, 0, GFP_KERNEL);
981 if (!get_device(dev) && dev) {
986 newrec->fc_target_port.node_name = pinfo->node_name;
987 newrec->fc_target_port.port_name = pinfo->port_name;
988 newrec->fc_target_port.private = &newrec[1];
989 newrec->fc_target_port.port_id = pinfo->port_id;
990 newrec->fc_target_port.port_num = idx;
991 INIT_LIST_HEAD(&newrec->tgt_list);
993 newrec->ops = template;
994 spin_lock_init(&newrec->lock);
995 INIT_LIST_HEAD(&newrec->ls_list);
996 INIT_LIST_HEAD(&newrec->ls_busylist);
997 INIT_LIST_HEAD(&newrec->assoc_list);
998 kref_init(&newrec->ref);
999 ida_init(&newrec->assoc_cnt);
1000 newrec->max_sg_cnt = min_t(u32, NVMET_FC_MAX_XFR_SGENTS,
1001 template->max_sgl_segments);
1003 ret = nvmet_fc_alloc_ls_iodlist(newrec);
1006 goto out_free_newrec;
1009 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1010 list_add_tail(&newrec->tgt_list, &nvmet_fc_target_list);
1011 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1013 *portptr = &newrec->fc_target_port;
1019 ida_simple_remove(&nvmet_fc_tgtport_cnt, idx);
1026 EXPORT_SYMBOL_GPL(nvmet_fc_register_targetport);
1030 nvmet_fc_free_tgtport(struct kref *ref)
1032 struct nvmet_fc_tgtport *tgtport =
1033 container_of(ref, struct nvmet_fc_tgtport, ref);
1034 struct device *dev = tgtport->dev;
1035 unsigned long flags;
1037 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1038 list_del(&tgtport->tgt_list);
1039 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1041 nvmet_fc_free_ls_iodlist(tgtport);
1043 /* let the LLDD know we've finished tearing it down */
1044 tgtport->ops->targetport_delete(&tgtport->fc_target_port);
1046 ida_simple_remove(&nvmet_fc_tgtport_cnt,
1047 tgtport->fc_target_port.port_num);
1049 ida_destroy(&tgtport->assoc_cnt);
1057 nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport)
1059 kref_put(&tgtport->ref, nvmet_fc_free_tgtport);
1063 nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport)
1065 return kref_get_unless_zero(&tgtport->ref);
1069 __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport)
1071 struct nvmet_fc_tgt_assoc *assoc, *next;
1072 unsigned long flags;
1074 spin_lock_irqsave(&tgtport->lock, flags);
1075 list_for_each_entry_safe(assoc, next,
1076 &tgtport->assoc_list, a_list) {
1077 if (!nvmet_fc_tgt_a_get(assoc))
1079 spin_unlock_irqrestore(&tgtport->lock, flags);
1080 nvmet_fc_delete_target_assoc(assoc);
1081 nvmet_fc_tgt_a_put(assoc);
1082 spin_lock_irqsave(&tgtport->lock, flags);
1084 spin_unlock_irqrestore(&tgtport->lock, flags);
1088 * nvmet layer has called to terminate an association
1091 nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl)
1093 struct nvmet_fc_tgtport *tgtport, *next;
1094 struct nvmet_fc_tgt_assoc *assoc;
1095 struct nvmet_fc_tgt_queue *queue;
1096 unsigned long flags;
1097 bool found_ctrl = false;
1099 /* this is a bit ugly, but don't want to make locks layered */
1100 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1101 list_for_each_entry_safe(tgtport, next, &nvmet_fc_target_list,
1103 if (!nvmet_fc_tgtport_get(tgtport))
1105 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1107 spin_lock_irqsave(&tgtport->lock, flags);
1108 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
1109 queue = assoc->queues[0];
1110 if (queue && queue->nvme_sq.ctrl == ctrl) {
1111 if (nvmet_fc_tgt_a_get(assoc))
1116 spin_unlock_irqrestore(&tgtport->lock, flags);
1118 nvmet_fc_tgtport_put(tgtport);
1121 nvmet_fc_delete_target_assoc(assoc);
1122 nvmet_fc_tgt_a_put(assoc);
1126 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1128 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1132 * nvme_fc_unregister_targetport - transport entry point called by an
1133 * LLDD to deregister/remove a previously
1134 * registered a local NVME subsystem FC port.
1135 * @tgtport: pointer to the (registered) target port that is to be
1139 * a completion status. Must be 0 upon success; a negative errno
1140 * (ex: -ENXIO) upon failure.
1143 nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port)
1145 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1147 /* terminate any outstanding associations */
1148 __nvmet_fc_free_assocs(tgtport);
1150 nvmet_fc_tgtport_put(tgtport);
1154 EXPORT_SYMBOL_GPL(nvmet_fc_unregister_targetport);
1157 /* *********************** FC-NVME LS Handling **************************** */
1161 nvmet_fc_format_rsp_hdr(void *buf, u8 ls_cmd, __be32 desc_len, u8 rqst_ls_cmd)
1163 struct fcnvme_ls_acc_hdr *acc = buf;
1165 acc->w0.ls_cmd = ls_cmd;
1166 acc->desc_list_len = desc_len;
1167 acc->rqst.desc_tag = cpu_to_be32(FCNVME_LSDESC_RQST);
1168 acc->rqst.desc_len =
1169 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst));
1170 acc->rqst.w0.ls_cmd = rqst_ls_cmd;
1174 nvmet_fc_format_rjt(void *buf, u16 buflen, u8 ls_cmd,
1175 u8 reason, u8 explanation, u8 vendor)
1177 struct fcnvme_ls_rjt *rjt = buf;
1179 nvmet_fc_format_rsp_hdr(buf, FCNVME_LSDESC_RQST,
1180 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_rjt)),
1182 rjt->rjt.desc_tag = cpu_to_be32(FCNVME_LSDESC_RJT);
1183 rjt->rjt.desc_len = fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rjt));
1184 rjt->rjt.reason_code = reason;
1185 rjt->rjt.reason_explanation = explanation;
1186 rjt->rjt.vendor = vendor;
1188 return sizeof(struct fcnvme_ls_rjt);
1191 /* Validation Error indexes into the string table below */
1194 VERR_CR_ASSOC_LEN = 1,
1195 VERR_CR_ASSOC_RQST_LEN = 2,
1196 VERR_CR_ASSOC_CMD = 3,
1197 VERR_CR_ASSOC_CMD_LEN = 4,
1198 VERR_ERSP_RATIO = 5,
1199 VERR_ASSOC_ALLOC_FAIL = 6,
1200 VERR_QUEUE_ALLOC_FAIL = 7,
1201 VERR_CR_CONN_LEN = 8,
1202 VERR_CR_CONN_RQST_LEN = 9,
1204 VERR_ASSOC_ID_LEN = 11,
1207 VERR_CONN_ID_LEN = 14,
1209 VERR_CR_CONN_CMD = 16,
1210 VERR_CR_CONN_CMD_LEN = 17,
1211 VERR_DISCONN_LEN = 18,
1212 VERR_DISCONN_RQST_LEN = 19,
1213 VERR_DISCONN_CMD = 20,
1214 VERR_DISCONN_CMD_LEN = 21,
1215 VERR_DISCONN_SCOPE = 22,
1217 VERR_RS_RQST_LEN = 24,
1219 VERR_RS_CMD_LEN = 26,
1224 static char *validation_errors[] = {
1226 "Bad CR_ASSOC Length",
1227 "Bad CR_ASSOC Rqst Length",
1229 "Bad CR_ASSOC Cmd Length",
1231 "Association Allocation Failed",
1232 "Queue Allocation Failed",
1233 "Bad CR_CONN Length",
1234 "Bad CR_CONN Rqst Length",
1235 "Not Association ID",
1236 "Bad Association ID Length",
1238 "Not Connection ID",
1239 "Bad Connection ID Length",
1242 "Bad CR_CONN Cmd Length",
1243 "Bad DISCONN Length",
1244 "Bad DISCONN Rqst Length",
1246 "Bad DISCONN Cmd Length",
1247 "Bad Disconnect Scope",
1249 "Bad RS Rqst Length",
1251 "Bad RS Cmd Length",
1253 "Bad RS Relative Offset",
1257 nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport,
1258 struct nvmet_fc_ls_iod *iod)
1260 struct fcnvme_ls_cr_assoc_rqst *rqst =
1261 (struct fcnvme_ls_cr_assoc_rqst *)iod->rqstbuf;
1262 struct fcnvme_ls_cr_assoc_acc *acc =
1263 (struct fcnvme_ls_cr_assoc_acc *)iod->rspbuf;
1264 struct nvmet_fc_tgt_queue *queue;
1267 memset(acc, 0, sizeof(*acc));
1270 * FC-NVME spec changes. There are initiators sending different
1271 * lengths as padding sizes for Create Association Cmd descriptor
1273 * Accept anything of "minimum" length. Assume format per 1.15
1274 * spec (with HOSTID reduced to 16 bytes), ignore how long the
1275 * trailing pad length is.
1277 if (iod->rqstdatalen < FCNVME_LSDESC_CRA_RQST_MINLEN)
1278 ret = VERR_CR_ASSOC_LEN;
1279 else if (be32_to_cpu(rqst->desc_list_len) <
1280 FCNVME_LSDESC_CRA_RQST_MIN_LISTLEN)
1281 ret = VERR_CR_ASSOC_RQST_LEN;
1282 else if (rqst->assoc_cmd.desc_tag !=
1283 cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD))
1284 ret = VERR_CR_ASSOC_CMD;
1285 else if (be32_to_cpu(rqst->assoc_cmd.desc_len) <
1286 FCNVME_LSDESC_CRA_CMD_DESC_MIN_DESCLEN)
1287 ret = VERR_CR_ASSOC_CMD_LEN;
1288 else if (!rqst->assoc_cmd.ersp_ratio ||
1289 (be16_to_cpu(rqst->assoc_cmd.ersp_ratio) >=
1290 be16_to_cpu(rqst->assoc_cmd.sqsize)))
1291 ret = VERR_ERSP_RATIO;
1294 /* new association w/ admin queue */
1295 iod->assoc = nvmet_fc_alloc_target_assoc(tgtport);
1297 ret = VERR_ASSOC_ALLOC_FAIL;
1299 queue = nvmet_fc_alloc_target_queue(iod->assoc, 0,
1300 be16_to_cpu(rqst->assoc_cmd.sqsize));
1302 ret = VERR_QUEUE_ALLOC_FAIL;
1307 dev_err(tgtport->dev,
1308 "Create Association LS failed: %s\n",
1309 validation_errors[ret]);
1310 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1311 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1312 FCNVME_RJT_RC_LOGIC,
1313 FCNVME_RJT_EXP_NONE, 0);
1317 queue->ersp_ratio = be16_to_cpu(rqst->assoc_cmd.ersp_ratio);
1318 atomic_set(&queue->connected, 1);
1319 queue->sqhd = 0; /* best place to init value */
1321 /* format a response */
1323 iod->lsreq->rsplen = sizeof(*acc);
1325 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1327 sizeof(struct fcnvme_ls_cr_assoc_acc)),
1328 FCNVME_LS_CREATE_ASSOCIATION);
1329 acc->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1330 acc->associd.desc_len =
1332 sizeof(struct fcnvme_lsdesc_assoc_id));
1333 acc->associd.association_id =
1334 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 0));
1335 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1336 acc->connectid.desc_len =
1338 sizeof(struct fcnvme_lsdesc_conn_id));
1339 acc->connectid.connection_id = acc->associd.association_id;
1343 nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport,
1344 struct nvmet_fc_ls_iod *iod)
1346 struct fcnvme_ls_cr_conn_rqst *rqst =
1347 (struct fcnvme_ls_cr_conn_rqst *)iod->rqstbuf;
1348 struct fcnvme_ls_cr_conn_acc *acc =
1349 (struct fcnvme_ls_cr_conn_acc *)iod->rspbuf;
1350 struct nvmet_fc_tgt_queue *queue;
1353 memset(acc, 0, sizeof(*acc));
1355 if (iod->rqstdatalen < sizeof(struct fcnvme_ls_cr_conn_rqst))
1356 ret = VERR_CR_CONN_LEN;
1357 else if (rqst->desc_list_len !=
1359 sizeof(struct fcnvme_ls_cr_conn_rqst)))
1360 ret = VERR_CR_CONN_RQST_LEN;
1361 else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1362 ret = VERR_ASSOC_ID;
1363 else if (rqst->associd.desc_len !=
1365 sizeof(struct fcnvme_lsdesc_assoc_id)))
1366 ret = VERR_ASSOC_ID_LEN;
1367 else if (rqst->connect_cmd.desc_tag !=
1368 cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD))
1369 ret = VERR_CR_CONN_CMD;
1370 else if (rqst->connect_cmd.desc_len !=
1372 sizeof(struct fcnvme_lsdesc_cr_conn_cmd)))
1373 ret = VERR_CR_CONN_CMD_LEN;
1374 else if (!rqst->connect_cmd.ersp_ratio ||
1375 (be16_to_cpu(rqst->connect_cmd.ersp_ratio) >=
1376 be16_to_cpu(rqst->connect_cmd.sqsize)))
1377 ret = VERR_ERSP_RATIO;
1381 iod->assoc = nvmet_fc_find_target_assoc(tgtport,
1382 be64_to_cpu(rqst->associd.association_id));
1384 ret = VERR_NO_ASSOC;
1386 queue = nvmet_fc_alloc_target_queue(iod->assoc,
1387 be16_to_cpu(rqst->connect_cmd.qid),
1388 be16_to_cpu(rqst->connect_cmd.sqsize));
1390 ret = VERR_QUEUE_ALLOC_FAIL;
1392 /* release get taken in nvmet_fc_find_target_assoc */
1393 nvmet_fc_tgt_a_put(iod->assoc);
1398 dev_err(tgtport->dev,
1399 "Create Connection LS failed: %s\n",
1400 validation_errors[ret]);
1401 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1402 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1403 (ret == VERR_NO_ASSOC) ?
1404 FCNVME_RJT_RC_INV_ASSOC :
1405 FCNVME_RJT_RC_LOGIC,
1406 FCNVME_RJT_EXP_NONE, 0);
1410 queue->ersp_ratio = be16_to_cpu(rqst->connect_cmd.ersp_ratio);
1411 atomic_set(&queue->connected, 1);
1412 queue->sqhd = 0; /* best place to init value */
1414 /* format a response */
1416 iod->lsreq->rsplen = sizeof(*acc);
1418 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1419 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)),
1420 FCNVME_LS_CREATE_CONNECTION);
1421 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1422 acc->connectid.desc_len =
1424 sizeof(struct fcnvme_lsdesc_conn_id));
1425 acc->connectid.connection_id =
1426 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc,
1427 be16_to_cpu(rqst->connect_cmd.qid)));
1431 nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport,
1432 struct nvmet_fc_ls_iod *iod)
1434 struct fcnvme_ls_disconnect_rqst *rqst =
1435 (struct fcnvme_ls_disconnect_rqst *)iod->rqstbuf;
1436 struct fcnvme_ls_disconnect_acc *acc =
1437 (struct fcnvme_ls_disconnect_acc *)iod->rspbuf;
1438 struct nvmet_fc_tgt_queue *queue = NULL;
1439 struct nvmet_fc_tgt_assoc *assoc;
1441 bool del_assoc = false;
1443 memset(acc, 0, sizeof(*acc));
1445 if (iod->rqstdatalen < sizeof(struct fcnvme_ls_disconnect_rqst))
1446 ret = VERR_DISCONN_LEN;
1447 else if (rqst->desc_list_len !=
1449 sizeof(struct fcnvme_ls_disconnect_rqst)))
1450 ret = VERR_DISCONN_RQST_LEN;
1451 else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1452 ret = VERR_ASSOC_ID;
1453 else if (rqst->associd.desc_len !=
1455 sizeof(struct fcnvme_lsdesc_assoc_id)))
1456 ret = VERR_ASSOC_ID_LEN;
1457 else if (rqst->discon_cmd.desc_tag !=
1458 cpu_to_be32(FCNVME_LSDESC_DISCONN_CMD))
1459 ret = VERR_DISCONN_CMD;
1460 else if (rqst->discon_cmd.desc_len !=
1462 sizeof(struct fcnvme_lsdesc_disconn_cmd)))
1463 ret = VERR_DISCONN_CMD_LEN;
1464 else if ((rqst->discon_cmd.scope != FCNVME_DISCONN_ASSOCIATION) &&
1465 (rqst->discon_cmd.scope != FCNVME_DISCONN_CONNECTION))
1466 ret = VERR_DISCONN_SCOPE;
1468 /* match an active association */
1469 assoc = nvmet_fc_find_target_assoc(tgtport,
1470 be64_to_cpu(rqst->associd.association_id));
1473 if (rqst->discon_cmd.scope ==
1474 FCNVME_DISCONN_CONNECTION) {
1475 queue = nvmet_fc_find_target_queue(tgtport,
1477 rqst->discon_cmd.id));
1479 nvmet_fc_tgt_a_put(assoc);
1484 ret = VERR_NO_ASSOC;
1488 dev_err(tgtport->dev,
1489 "Disconnect LS failed: %s\n",
1490 validation_errors[ret]);
1491 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1492 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1493 (ret == VERR_NO_ASSOC) ?
1494 FCNVME_RJT_RC_INV_ASSOC :
1495 (ret == VERR_NO_CONN) ?
1496 FCNVME_RJT_RC_INV_CONN :
1497 FCNVME_RJT_RC_LOGIC,
1498 FCNVME_RJT_EXP_NONE, 0);
1502 /* format a response */
1504 iod->lsreq->rsplen = sizeof(*acc);
1506 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1508 sizeof(struct fcnvme_ls_disconnect_acc)),
1509 FCNVME_LS_DISCONNECT);
1512 /* are we to delete a Connection ID (queue) */
1514 int qid = queue->qid;
1516 nvmet_fc_delete_target_queue(queue);
1518 /* release the get taken by find_target_queue */
1519 nvmet_fc_tgt_q_put(queue);
1521 /* tear association down if io queue terminated */
1526 /* release get taken in nvmet_fc_find_target_assoc */
1527 nvmet_fc_tgt_a_put(iod->assoc);
1530 nvmet_fc_delete_target_assoc(iod->assoc);
1534 /* *********************** NVME Ctrl Routines **************************** */
1537 static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req);
1539 static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops;
1542 nvmet_fc_xmt_ls_rsp_done(struct nvmefc_tgt_ls_req *lsreq)
1544 struct nvmet_fc_ls_iod *iod = lsreq->nvmet_fc_private;
1545 struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1547 fc_dma_sync_single_for_cpu(tgtport->dev, iod->rspdma,
1548 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
1549 nvmet_fc_free_ls_iod(tgtport, iod);
1550 nvmet_fc_tgtport_put(tgtport);
1554 nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport,
1555 struct nvmet_fc_ls_iod *iod)
1559 fc_dma_sync_single_for_device(tgtport->dev, iod->rspdma,
1560 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
1562 ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsreq);
1564 nvmet_fc_xmt_ls_rsp_done(iod->lsreq);
1568 * Actual processing routine for received FC-NVME LS Requests from the LLD
1571 nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport,
1572 struct nvmet_fc_ls_iod *iod)
1574 struct fcnvme_ls_rqst_w0 *w0 =
1575 (struct fcnvme_ls_rqst_w0 *)iod->rqstbuf;
1577 iod->lsreq->nvmet_fc_private = iod;
1578 iod->lsreq->rspbuf = iod->rspbuf;
1579 iod->lsreq->rspdma = iod->rspdma;
1580 iod->lsreq->done = nvmet_fc_xmt_ls_rsp_done;
1581 /* Be preventative. handlers will later set to valid length */
1582 iod->lsreq->rsplen = 0;
1588 * parse request input, execute the request, and format the
1591 switch (w0->ls_cmd) {
1592 case FCNVME_LS_CREATE_ASSOCIATION:
1593 /* Creates Association and initial Admin Queue/Connection */
1594 nvmet_fc_ls_create_association(tgtport, iod);
1596 case FCNVME_LS_CREATE_CONNECTION:
1597 /* Creates an IO Queue/Connection */
1598 nvmet_fc_ls_create_connection(tgtport, iod);
1600 case FCNVME_LS_DISCONNECT:
1601 /* Terminate a Queue/Connection or the Association */
1602 nvmet_fc_ls_disconnect(tgtport, iod);
1605 iod->lsreq->rsplen = nvmet_fc_format_rjt(iod->rspbuf,
1606 NVME_FC_MAX_LS_BUFFER_SIZE, w0->ls_cmd,
1607 FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0);
1610 nvmet_fc_xmt_ls_rsp(tgtport, iod);
1614 * Actual processing routine for received FC-NVME LS Requests from the LLD
1617 nvmet_fc_handle_ls_rqst_work(struct work_struct *work)
1619 struct nvmet_fc_ls_iod *iod =
1620 container_of(work, struct nvmet_fc_ls_iod, work);
1621 struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1623 nvmet_fc_handle_ls_rqst(tgtport, iod);
1628 * nvmet_fc_rcv_ls_req - transport entry point called by an LLDD
1629 * upon the reception of a NVME LS request.
1631 * The nvmet-fc layer will copy payload to an internal structure for
1632 * processing. As such, upon completion of the routine, the LLDD may
1633 * immediately free/reuse the LS request buffer passed in the call.
1635 * If this routine returns error, the LLDD should abort the exchange.
1637 * @tgtport: pointer to the (registered) target port the LS was
1639 * @lsreq: pointer to a lsreq request structure to be used to reference
1640 * the exchange corresponding to the LS.
1641 * @lsreqbuf: pointer to the buffer containing the LS Request
1642 * @lsreqbuf_len: length, in bytes, of the received LS request
1645 nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port,
1646 struct nvmefc_tgt_ls_req *lsreq,
1647 void *lsreqbuf, u32 lsreqbuf_len)
1649 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1650 struct nvmet_fc_ls_iod *iod;
1652 if (lsreqbuf_len > NVME_FC_MAX_LS_BUFFER_SIZE)
1655 if (!nvmet_fc_tgtport_get(tgtport))
1658 iod = nvmet_fc_alloc_ls_iod(tgtport);
1660 nvmet_fc_tgtport_put(tgtport);
1666 memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len);
1667 iod->rqstdatalen = lsreqbuf_len;
1669 schedule_work(&iod->work);
1673 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_ls_req);
1677 * **********************
1678 * Start of FCP handling
1679 * **********************
1683 nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
1685 struct scatterlist *sg;
1688 u32 page_len, length;
1691 length = fod->total_length;
1692 nent = DIV_ROUND_UP(length, PAGE_SIZE);
1693 sg = kmalloc_array(nent, sizeof(struct scatterlist), GFP_KERNEL);
1697 sg_init_table(sg, nent);
1700 page_len = min_t(u32, length, PAGE_SIZE);
1702 page = alloc_page(GFP_KERNEL);
1704 goto out_free_pages;
1706 sg_set_page(&sg[i], page, page_len, 0);
1712 fod->data_sg_cnt = nent;
1713 fod->data_sg_cnt = fc_dma_map_sg(fod->tgtport->dev, sg, nent,
1714 ((fod->io_dir == NVMET_FCP_WRITE) ?
1715 DMA_FROM_DEVICE : DMA_TO_DEVICE));
1716 /* note: write from initiator perspective */
1723 __free_page(sg_page(&sg[i]));
1726 fod->data_sg = NULL;
1727 fod->data_sg_cnt = 0;
1729 return NVME_SC_INTERNAL;
1733 nvmet_fc_free_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
1735 struct scatterlist *sg;
1738 if (!fod->data_sg || !fod->data_sg_cnt)
1741 fc_dma_unmap_sg(fod->tgtport->dev, fod->data_sg, fod->data_sg_cnt,
1742 ((fod->io_dir == NVMET_FCP_WRITE) ?
1743 DMA_FROM_DEVICE : DMA_TO_DEVICE));
1744 for_each_sg(fod->data_sg, sg, fod->data_sg_cnt, count)
1745 __free_page(sg_page(sg));
1746 kfree(fod->data_sg);
1747 fod->data_sg = NULL;
1748 fod->data_sg_cnt = 0;
1753 queue_90percent_full(struct nvmet_fc_tgt_queue *q, u32 sqhd)
1757 /* egad, this is ugly. And sqtail is just a best guess */
1758 sqtail = atomic_read(&q->sqtail) % q->sqsize;
1760 used = (sqtail < sqhd) ? (sqtail + q->sqsize - sqhd) : (sqtail - sqhd);
1761 return ((used * 10) >= (((u32)(q->sqsize - 1) * 9)));
1766 * May be a NVMET_FCOP_RSP or NVMET_FCOP_READDATA_RSP op
1769 nvmet_fc_prep_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
1770 struct nvmet_fc_fcp_iod *fod)
1772 struct nvme_fc_ersp_iu *ersp = &fod->rspiubuf;
1773 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
1774 struct nvme_completion *cqe = &ersp->cqe;
1775 u32 *cqewd = (u32 *)cqe;
1776 bool send_ersp = false;
1777 u32 rsn, rspcnt, xfr_length;
1779 if (fod->fcpreq->op == NVMET_FCOP_READDATA_RSP)
1780 xfr_length = fod->total_length;
1782 xfr_length = fod->offset;
1785 * check to see if we can send a 0's rsp.
1786 * Note: to send a 0's response, the NVME-FC host transport will
1787 * recreate the CQE. The host transport knows: sq id, SQHD (last
1788 * seen in an ersp), and command_id. Thus it will create a
1789 * zero-filled CQE with those known fields filled in. Transport
1790 * must send an ersp for any condition where the cqe won't match
1793 * Here are the FC-NVME mandated cases where we must send an ersp:
1794 * every N responses, where N=ersp_ratio
1795 * force fabric commands to send ersp's (not in FC-NVME but good
1797 * normal cmds: any time status is non-zero, or status is zero
1798 * but words 0 or 1 are non-zero.
1799 * the SQ is 90% or more full
1800 * the cmd is a fused command
1801 * transferred data length not equal to cmd iu length
1803 rspcnt = atomic_inc_return(&fod->queue->zrspcnt);
1804 if (!(rspcnt % fod->queue->ersp_ratio) ||
1805 sqe->opcode == nvme_fabrics_command ||
1806 xfr_length != fod->total_length ||
1807 (le16_to_cpu(cqe->status) & 0xFFFE) || cqewd[0] || cqewd[1] ||
1808 (sqe->flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND)) ||
1809 queue_90percent_full(fod->queue, le16_to_cpu(cqe->sq_head)))
1812 /* re-set the fields */
1813 fod->fcpreq->rspaddr = ersp;
1814 fod->fcpreq->rspdma = fod->rspdma;
1817 memset(ersp, 0, NVME_FC_SIZEOF_ZEROS_RSP);
1818 fod->fcpreq->rsplen = NVME_FC_SIZEOF_ZEROS_RSP;
1820 ersp->iu_len = cpu_to_be16(sizeof(*ersp)/sizeof(u32));
1821 rsn = atomic_inc_return(&fod->queue->rsn);
1822 ersp->rsn = cpu_to_be32(rsn);
1823 ersp->xfrd_len = cpu_to_be32(xfr_length);
1824 fod->fcpreq->rsplen = sizeof(*ersp);
1827 fc_dma_sync_single_for_device(tgtport->dev, fod->rspdma,
1828 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
1831 static void nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq);
1834 nvmet_fc_abort_op(struct nvmet_fc_tgtport *tgtport,
1835 struct nvmet_fc_fcp_iod *fod)
1837 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1839 /* data no longer needed */
1840 nvmet_fc_free_tgt_pgs(fod);
1843 * if an ABTS was received or we issued the fcp_abort early
1844 * don't call abort routine again.
1846 /* no need to take lock - lock was taken earlier to get here */
1848 tgtport->ops->fcp_abort(&tgtport->fc_target_port, fcpreq);
1850 nvmet_fc_free_fcp_iod(fod->queue, fod);
1854 nvmet_fc_xmt_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
1855 struct nvmet_fc_fcp_iod *fod)
1859 fod->fcpreq->op = NVMET_FCOP_RSP;
1860 fod->fcpreq->timeout = 0;
1862 nvmet_fc_prep_fcp_rsp(tgtport, fod);
1864 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
1866 nvmet_fc_abort_op(tgtport, fod);
1870 nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport *tgtport,
1871 struct nvmet_fc_fcp_iod *fod, u8 op)
1873 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1874 unsigned long flags;
1879 fcpreq->offset = fod->offset;
1880 fcpreq->timeout = NVME_FC_TGTOP_TIMEOUT_SEC;
1882 tlen = min_t(u32, tgtport->max_sg_cnt * PAGE_SIZE,
1883 (fod->total_length - fod->offset));
1884 fcpreq->transfer_length = tlen;
1885 fcpreq->transferred_length = 0;
1886 fcpreq->fcp_error = 0;
1889 fcpreq->sg = &fod->data_sg[fod->offset / PAGE_SIZE];
1890 fcpreq->sg_cnt = DIV_ROUND_UP(tlen, PAGE_SIZE);
1893 * If the last READDATA request: check if LLDD supports
1894 * combined xfr with response.
1896 if ((op == NVMET_FCOP_READDATA) &&
1897 ((fod->offset + fcpreq->transfer_length) == fod->total_length) &&
1898 (tgtport->ops->target_features & NVMET_FCTGTFEAT_READDATA_RSP)) {
1899 fcpreq->op = NVMET_FCOP_READDATA_RSP;
1900 nvmet_fc_prep_fcp_rsp(tgtport, fod);
1903 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
1906 * should be ok to set w/o lock as its in the thread of
1907 * execution (not an async timer routine) and doesn't
1908 * contend with any clearing action
1912 if (op == NVMET_FCOP_WRITEDATA) {
1913 spin_lock_irqsave(&fod->flock, flags);
1914 fod->writedataactive = false;
1915 spin_unlock_irqrestore(&fod->flock, flags);
1916 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
1917 } else /* NVMET_FCOP_READDATA or NVMET_FCOP_READDATA_RSP */ {
1918 fcpreq->fcp_error = ret;
1919 fcpreq->transferred_length = 0;
1920 nvmet_fc_xmt_fcp_op_done(fod->fcpreq);
1926 __nvmet_fc_fod_op_abort(struct nvmet_fc_fcp_iod *fod, bool abort)
1928 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1929 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
1931 /* if in the middle of an io and we need to tear down */
1933 if (fcpreq->op == NVMET_FCOP_WRITEDATA) {
1934 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
1938 nvmet_fc_abort_op(tgtport, fod);
1946 * actual done handler for FCP operations when completed by the lldd
1949 nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod)
1951 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1952 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
1953 unsigned long flags;
1956 spin_lock_irqsave(&fod->flock, flags);
1958 fod->writedataactive = false;
1959 spin_unlock_irqrestore(&fod->flock, flags);
1961 switch (fcpreq->op) {
1963 case NVMET_FCOP_WRITEDATA:
1964 if (__nvmet_fc_fod_op_abort(fod, abort))
1966 if (fcpreq->fcp_error ||
1967 fcpreq->transferred_length != fcpreq->transfer_length) {
1968 spin_lock(&fod->flock);
1970 spin_unlock(&fod->flock);
1972 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
1976 fod->offset += fcpreq->transferred_length;
1977 if (fod->offset != fod->total_length) {
1978 spin_lock_irqsave(&fod->flock, flags);
1979 fod->writedataactive = true;
1980 spin_unlock_irqrestore(&fod->flock, flags);
1982 /* transfer the next chunk */
1983 nvmet_fc_transfer_fcp_data(tgtport, fod,
1984 NVMET_FCOP_WRITEDATA);
1988 /* data transfer complete, resume with nvmet layer */
1990 fod->req.execute(&fod->req);
1994 case NVMET_FCOP_READDATA:
1995 case NVMET_FCOP_READDATA_RSP:
1996 if (__nvmet_fc_fod_op_abort(fod, abort))
1998 if (fcpreq->fcp_error ||
1999 fcpreq->transferred_length != fcpreq->transfer_length) {
2000 nvmet_fc_abort_op(tgtport, fod);
2006 if (fcpreq->op == NVMET_FCOP_READDATA_RSP) {
2007 /* data no longer needed */
2008 nvmet_fc_free_tgt_pgs(fod);
2009 nvmet_fc_free_fcp_iod(fod->queue, fod);
2013 fod->offset += fcpreq->transferred_length;
2014 if (fod->offset != fod->total_length) {
2015 /* transfer the next chunk */
2016 nvmet_fc_transfer_fcp_data(tgtport, fod,
2017 NVMET_FCOP_READDATA);
2021 /* data transfer complete, send response */
2023 /* data no longer needed */
2024 nvmet_fc_free_tgt_pgs(fod);
2026 nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2030 case NVMET_FCOP_RSP:
2031 if (__nvmet_fc_fod_op_abort(fod, abort))
2033 nvmet_fc_free_fcp_iod(fod->queue, fod);
2042 nvmet_fc_fcp_rqst_op_done_work(struct work_struct *work)
2044 struct nvmet_fc_fcp_iod *fod =
2045 container_of(work, struct nvmet_fc_fcp_iod, done_work);
2047 nvmet_fc_fod_op_done(fod);
2051 nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq)
2053 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2054 struct nvmet_fc_tgt_queue *queue = fod->queue;
2056 if (fod->tgtport->ops->target_features & NVMET_FCTGTFEAT_OPDONE_IN_ISR)
2057 /* context switch so completion is not in ISR context */
2058 queue_work_on(queue->cpu, queue->work_q, &fod->done_work);
2060 nvmet_fc_fod_op_done(fod);
2064 * actual completion handler after execution by the nvmet layer
2067 __nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport *tgtport,
2068 struct nvmet_fc_fcp_iod *fod, int status)
2070 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
2071 struct nvme_completion *cqe = &fod->rspiubuf.cqe;
2072 unsigned long flags;
2075 spin_lock_irqsave(&fod->flock, flags);
2077 spin_unlock_irqrestore(&fod->flock, flags);
2079 /* if we have a CQE, snoop the last sq_head value */
2081 fod->queue->sqhd = cqe->sq_head;
2084 nvmet_fc_abort_op(tgtport, fod);
2088 /* if an error handling the cmd post initial parsing */
2090 /* fudge up a failed CQE status for our transport error */
2091 memset(cqe, 0, sizeof(*cqe));
2092 cqe->sq_head = fod->queue->sqhd; /* echo last cqe sqhd */
2093 cqe->sq_id = cpu_to_le16(fod->queue->qid);
2094 cqe->command_id = sqe->command_id;
2095 cqe->status = cpu_to_le16(status);
2099 * try to push the data even if the SQE status is non-zero.
2100 * There may be a status where data still was intended to
2103 if ((fod->io_dir == NVMET_FCP_READ) && (fod->data_sg_cnt)) {
2104 /* push the data over before sending rsp */
2105 nvmet_fc_transfer_fcp_data(tgtport, fod,
2106 NVMET_FCOP_READDATA);
2110 /* writes & no data - fall thru */
2113 /* data no longer needed */
2114 nvmet_fc_free_tgt_pgs(fod);
2116 nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2121 nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req)
2123 struct nvmet_fc_fcp_iod *fod = nvmet_req_to_fod(nvme_req);
2124 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2126 __nvmet_fc_fcp_nvme_cmd_done(tgtport, fod, 0);
2131 * Actual processing routine for received FC-NVME LS Requests from the LLD
2134 nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
2135 struct nvmet_fc_fcp_iod *fod)
2137 struct nvme_fc_cmd_iu *cmdiu = &fod->cmdiubuf;
2141 * Fused commands are currently not supported in the linux
2144 * As such, the implementation of the FC transport does not
2145 * look at the fused commands and order delivery to the upper
2146 * layer until we have both based on csn.
2149 fod->fcpreq->done = nvmet_fc_xmt_fcp_op_done;
2151 fod->total_length = be32_to_cpu(cmdiu->data_len);
2152 if (cmdiu->flags & FCNVME_CMD_FLAGS_WRITE) {
2153 fod->io_dir = NVMET_FCP_WRITE;
2154 if (!nvme_is_write(&cmdiu->sqe))
2155 goto transport_error;
2156 } else if (cmdiu->flags & FCNVME_CMD_FLAGS_READ) {
2157 fod->io_dir = NVMET_FCP_READ;
2158 if (nvme_is_write(&cmdiu->sqe))
2159 goto transport_error;
2161 fod->io_dir = NVMET_FCP_NODATA;
2162 if (fod->total_length)
2163 goto transport_error;
2166 fod->req.cmd = &fod->cmdiubuf.sqe;
2167 fod->req.rsp = &fod->rspiubuf.cqe;
2168 fod->req.port = fod->queue->port;
2170 /* ensure nvmet handlers will set cmd handler callback */
2171 fod->req.execute = NULL;
2173 /* clear any response payload */
2174 memset(&fod->rspiubuf, 0, sizeof(fod->rspiubuf));
2176 fod->data_sg = NULL;
2177 fod->data_sg_cnt = 0;
2179 ret = nvmet_req_init(&fod->req,
2180 &fod->queue->nvme_cq,
2181 &fod->queue->nvme_sq,
2182 &nvmet_fc_tgt_fcp_ops);
2184 /* bad SQE content or invalid ctrl state */
2185 /* nvmet layer has already called op done to send rsp. */
2189 /* keep a running counter of tail position */
2190 atomic_inc(&fod->queue->sqtail);
2192 if (fod->total_length) {
2193 ret = nvmet_fc_alloc_tgt_pgs(fod);
2195 nvmet_req_complete(&fod->req, ret);
2199 fod->req.sg = fod->data_sg;
2200 fod->req.sg_cnt = fod->data_sg_cnt;
2203 if (fod->io_dir == NVMET_FCP_WRITE) {
2204 /* pull the data over before invoking nvmet layer */
2205 nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_WRITEDATA);
2212 * can invoke the nvmet_layer now. If read data, cmd completion will
2216 fod->req.execute(&fod->req);
2221 nvmet_fc_abort_op(tgtport, fod);
2225 * Actual processing routine for received FC-NVME LS Requests from the LLD
2228 nvmet_fc_handle_fcp_rqst_work(struct work_struct *work)
2230 struct nvmet_fc_fcp_iod *fod =
2231 container_of(work, struct nvmet_fc_fcp_iod, work);
2232 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2234 nvmet_fc_handle_fcp_rqst(tgtport, fod);
2238 * nvmet_fc_rcv_fcp_req - transport entry point called by an LLDD
2239 * upon the reception of a NVME FCP CMD IU.
2241 * Pass a FC-NVME FCP CMD IU received from the FC link to the nvmet-fc
2242 * layer for processing.
2244 * The nvmet_fc layer allocates a local job structure (struct
2245 * nvmet_fc_fcp_iod) from the queue for the io and copies the
2246 * CMD IU buffer to the job structure. As such, on a successful
2247 * completion (returns 0), the LLDD may immediately free/reuse
2248 * the CMD IU buffer passed in the call.
2250 * However, in some circumstances, due to the packetized nature of FC
2251 * and the api of the FC LLDD which may issue a hw command to send the
2252 * response, but the LLDD may not get the hw completion for that command
2253 * and upcall the nvmet_fc layer before a new command may be
2254 * asynchronously received - its possible for a command to be received
2255 * before the LLDD and nvmet_fc have recycled the job structure. It gives
2256 * the appearance of more commands received than fits in the sq.
2257 * To alleviate this scenario, a temporary queue is maintained in the
2258 * transport for pending LLDD requests waiting for a queue job structure.
2259 * In these "overrun" cases, a temporary queue element is allocated
2260 * the LLDD request and CMD iu buffer information remembered, and the
2261 * routine returns a -EOVERFLOW status. Subsequently, when a queue job
2262 * structure is freed, it is immediately reallocated for anything on the
2263 * pending request list. The LLDDs defer_rcv() callback is called,
2264 * informing the LLDD that it may reuse the CMD IU buffer, and the io
2265 * is then started normally with the transport.
2267 * The LLDD, when receiving an -EOVERFLOW completion status, is to treat
2268 * the completion as successful but must not reuse the CMD IU buffer
2269 * until the LLDD's defer_rcv() callback has been called for the
2270 * corresponding struct nvmefc_tgt_fcp_req pointer.
2272 * If there is any other condition in which an error occurs, the
2273 * transport will return a non-zero status indicating the error.
2274 * In all cases other than -EOVERFLOW, the transport has not accepted the
2275 * request and the LLDD should abort the exchange.
2277 * @target_port: pointer to the (registered) target port the FCP CMD IU
2279 * @fcpreq: pointer to a fcpreq request structure to be used to reference
2280 * the exchange corresponding to the FCP Exchange.
2281 * @cmdiubuf: pointer to the buffer containing the FCP CMD IU
2282 * @cmdiubuf_len: length, in bytes, of the received FCP CMD IU
2285 nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *target_port,
2286 struct nvmefc_tgt_fcp_req *fcpreq,
2287 void *cmdiubuf, u32 cmdiubuf_len)
2289 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
2290 struct nvme_fc_cmd_iu *cmdiu = cmdiubuf;
2291 struct nvmet_fc_tgt_queue *queue;
2292 struct nvmet_fc_fcp_iod *fod;
2293 struct nvmet_fc_defer_fcp_req *deferfcp;
2294 unsigned long flags;
2296 /* validate iu, so the connection id can be used to find the queue */
2297 if ((cmdiubuf_len != sizeof(*cmdiu)) ||
2298 (cmdiu->scsi_id != NVME_CMD_SCSI_ID) ||
2299 (cmdiu->fc_id != NVME_CMD_FC_ID) ||
2300 (be16_to_cpu(cmdiu->iu_len) != (sizeof(*cmdiu)/4)))
2303 queue = nvmet_fc_find_target_queue(tgtport,
2304 be64_to_cpu(cmdiu->connection_id));
2309 * note: reference taken by find_target_queue
2310 * After successful fod allocation, the fod will inherit the
2311 * ownership of that reference and will remove the reference
2312 * when the fod is freed.
2315 spin_lock_irqsave(&queue->qlock, flags);
2317 fod = nvmet_fc_alloc_fcp_iod(queue);
2319 spin_unlock_irqrestore(&queue->qlock, flags);
2321 fcpreq->nvmet_fc_private = fod;
2322 fod->fcpreq = fcpreq;
2324 memcpy(&fod->cmdiubuf, cmdiubuf, cmdiubuf_len);
2326 nvmet_fc_queue_fcp_req(tgtport, queue, fcpreq);
2331 if (!tgtport->ops->defer_rcv) {
2332 spin_unlock_irqrestore(&queue->qlock, flags);
2333 /* release the queue lookup reference */
2334 nvmet_fc_tgt_q_put(queue);
2338 deferfcp = list_first_entry_or_null(&queue->avail_defer_list,
2339 struct nvmet_fc_defer_fcp_req, req_list);
2341 /* Just re-use one that was previously allocated */
2342 list_del(&deferfcp->req_list);
2344 spin_unlock_irqrestore(&queue->qlock, flags);
2346 /* Now we need to dynamically allocate one */
2347 deferfcp = kmalloc(sizeof(*deferfcp), GFP_KERNEL);
2349 /* release the queue lookup reference */
2350 nvmet_fc_tgt_q_put(queue);
2353 spin_lock_irqsave(&queue->qlock, flags);
2356 /* For now, use rspaddr / rsplen to save payload information */
2357 fcpreq->rspaddr = cmdiubuf;
2358 fcpreq->rsplen = cmdiubuf_len;
2359 deferfcp->fcp_req = fcpreq;
2361 /* defer processing till a fod becomes available */
2362 list_add_tail(&deferfcp->req_list, &queue->pending_cmd_list);
2364 /* NOTE: the queue lookup reference is still valid */
2366 spin_unlock_irqrestore(&queue->qlock, flags);
2370 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_req);
2373 * nvmet_fc_rcv_fcp_abort - transport entry point called by an LLDD
2374 * upon the reception of an ABTS for a FCP command
2376 * Notify the transport that an ABTS has been received for a FCP command
2377 * that had been given to the transport via nvmet_fc_rcv_fcp_req(). The
2378 * LLDD believes the command is still being worked on
2379 * (template_ops->fcp_req_release() has not been called).
2381 * The transport will wait for any outstanding work (an op to the LLDD,
2382 * which the lldd should complete with error due to the ABTS; or the
2383 * completion from the nvmet layer of the nvme command), then will
2384 * stop processing and call the nvmet_fc_rcv_fcp_req() callback to
2385 * return the i/o context to the LLDD. The LLDD may send the BA_ACC
2386 * to the ABTS either after return from this function (assuming any
2387 * outstanding op work has been terminated) or upon the callback being
2390 * @target_port: pointer to the (registered) target port the FCP CMD IU
2392 * @fcpreq: pointer to the fcpreq request structure that corresponds
2393 * to the exchange that received the ABTS.
2396 nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port *target_port,
2397 struct nvmefc_tgt_fcp_req *fcpreq)
2399 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2400 struct nvmet_fc_tgt_queue *queue;
2401 unsigned long flags;
2403 if (!fod || fod->fcpreq != fcpreq)
2404 /* job appears to have already completed, ignore abort */
2409 spin_lock_irqsave(&queue->qlock, flags);
2412 * mark as abort. The abort handler, invoked upon completion
2413 * of any work, will detect the aborted status and do the
2416 spin_lock(&fod->flock);
2418 fod->aborted = true;
2419 spin_unlock(&fod->flock);
2421 spin_unlock_irqrestore(&queue->qlock, flags);
2423 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_abort);
2426 struct nvmet_fc_traddr {
2432 __nvme_fc_parse_u64(substring_t *sstr, u64 *val)
2436 if (match_u64(sstr, &token64))
2444 * This routine validates and extracts the WWN's from the TRADDR string.
2445 * As kernel parsers need the 0x to determine number base, universally
2446 * build string to parse with 0x prefix before parsing name strings.
2449 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
2451 char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
2452 substring_t wwn = { name, &name[sizeof(name)-1] };
2453 int nnoffset, pnoffset;
2455 /* validate it string one of the 2 allowed formats */
2456 if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH &&
2457 !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
2458 !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
2459 "pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
2460 nnoffset = NVME_FC_TRADDR_OXNNLEN;
2461 pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
2462 NVME_FC_TRADDR_OXNNLEN;
2463 } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH &&
2464 !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
2465 !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
2466 "pn-", NVME_FC_TRADDR_NNLEN))) {
2467 nnoffset = NVME_FC_TRADDR_NNLEN;
2468 pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
2474 name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
2476 memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2477 if (__nvme_fc_parse_u64(&wwn, &traddr->nn))
2480 memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2481 if (__nvme_fc_parse_u64(&wwn, &traddr->pn))
2487 pr_warn("%s: bad traddr string\n", __func__);
2492 nvmet_fc_add_port(struct nvmet_port *port)
2494 struct nvmet_fc_tgtport *tgtport;
2495 struct nvmet_fc_traddr traddr = { 0L, 0L };
2496 unsigned long flags;
2499 /* validate the address info */
2500 if ((port->disc_addr.trtype != NVMF_TRTYPE_FC) ||
2501 (port->disc_addr.adrfam != NVMF_ADDR_FAMILY_FC))
2504 /* map the traddr address info to a target port */
2506 ret = nvme_fc_parse_traddr(&traddr, port->disc_addr.traddr,
2507 sizeof(port->disc_addr.traddr));
2512 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
2513 list_for_each_entry(tgtport, &nvmet_fc_target_list, tgt_list) {
2514 if ((tgtport->fc_target_port.node_name == traddr.nn) &&
2515 (tgtport->fc_target_port.port_name == traddr.pn)) {
2516 /* a FC port can only be 1 nvmet port id */
2517 if (!tgtport->port) {
2518 tgtport->port = port;
2519 port->priv = tgtport;
2520 nvmet_fc_tgtport_get(tgtport);
2527 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
2532 nvmet_fc_remove_port(struct nvmet_port *port)
2534 struct nvmet_fc_tgtport *tgtport = port->priv;
2535 unsigned long flags;
2536 bool matched = false;
2538 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
2539 if (tgtport->port == port) {
2541 tgtport->port = NULL;
2543 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
2546 nvmet_fc_tgtport_put(tgtport);
2549 static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = {
2550 .owner = THIS_MODULE,
2551 .type = NVMF_TRTYPE_FC,
2553 .add_port = nvmet_fc_add_port,
2554 .remove_port = nvmet_fc_remove_port,
2555 .queue_response = nvmet_fc_fcp_nvme_cmd_done,
2556 .delete_ctrl = nvmet_fc_delete_ctrl,
2559 static int __init nvmet_fc_init_module(void)
2561 return nvmet_register_transport(&nvmet_fc_tgt_fcp_ops);
2564 static void __exit nvmet_fc_exit_module(void)
2566 /* sanity check - all lports should be removed */
2567 if (!list_empty(&nvmet_fc_target_list))
2568 pr_warn("%s: targetport list not empty\n", __func__);
2570 nvmet_unregister_transport(&nvmet_fc_tgt_fcp_ops);
2572 ida_destroy(&nvmet_fc_tgtport_cnt);
2575 module_init(nvmet_fc_init_module);
2576 module_exit(nvmet_fc_exit_module);
2578 MODULE_LICENSE("GPL v2");