1 // SPDX-License-Identifier: GPL-2.0
3 * NVMe over Fabrics TCP host.
4 * Copyright (c) 2018 Lightbits Labs. All rights reserved.
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 #include <linux/module.h>
8 #include <linux/init.h>
9 #include <linux/slab.h>
10 #include <linux/err.h>
11 #include <linux/nvme-tcp.h>
14 #include <linux/blk-mq.h>
15 #include <crypto/hash.h>
16 #include <net/busy_poll.h>
21 struct nvme_tcp_queue;
23 /* Define the socket priority to use for connections were it is desirable
24 * that the NIC consider performing optimized packet processing or filtering.
25 * A non-zero value being sufficient to indicate general consideration of any
26 * possible optimization. Making it a module param allows for alternative
27 * values that may be unique for some NIC implementations.
29 static int so_priority;
30 module_param(so_priority, int, 0644);
31 MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority");
33 enum nvme_tcp_send_state {
34 NVME_TCP_SEND_CMD_PDU = 0,
35 NVME_TCP_SEND_H2C_PDU,
40 struct nvme_tcp_request {
41 struct nvme_request req;
43 struct nvme_tcp_queue *queue;
48 struct list_head entry;
49 struct llist_node lentry;
58 enum nvme_tcp_send_state state;
61 enum nvme_tcp_queue_flags {
62 NVME_TCP_Q_ALLOCATED = 0,
64 NVME_TCP_Q_POLLING = 2,
67 enum nvme_tcp_recv_state {
68 NVME_TCP_RECV_PDU = 0,
74 struct nvme_tcp_queue {
76 struct work_struct io_work;
79 struct mutex send_mutex;
80 struct llist_head req_list;
81 struct list_head send_list;
88 size_t data_remaining;
89 size_t ddgst_remaining;
93 struct nvme_tcp_request *request;
96 size_t cmnd_capsule_len;
97 struct nvme_tcp_ctrl *ctrl;
103 struct ahash_request *rcv_hash;
104 struct ahash_request *snd_hash;
108 struct page_frag_cache pf_cache;
110 void (*state_change)(struct sock *);
111 void (*data_ready)(struct sock *);
112 void (*write_space)(struct sock *);
115 struct nvme_tcp_ctrl {
116 /* read only in the hot path */
117 struct nvme_tcp_queue *queues;
118 struct blk_mq_tag_set tag_set;
120 /* other member variables */
121 struct list_head list;
122 struct blk_mq_tag_set admin_tag_set;
123 struct sockaddr_storage addr;
124 struct sockaddr_storage src_addr;
125 struct nvme_ctrl ctrl;
127 struct mutex teardown_lock;
128 struct work_struct err_work;
129 struct delayed_work connect_work;
130 struct nvme_tcp_request async_req;
131 u32 io_queues[HCTX_MAX_TYPES];
134 static LIST_HEAD(nvme_tcp_ctrl_list);
135 static DEFINE_MUTEX(nvme_tcp_ctrl_mutex);
136 static struct workqueue_struct *nvme_tcp_wq;
137 static const struct blk_mq_ops nvme_tcp_mq_ops;
138 static const struct blk_mq_ops nvme_tcp_admin_mq_ops;
139 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue);
141 static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl)
143 return container_of(ctrl, struct nvme_tcp_ctrl, ctrl);
146 static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue)
148 return queue - queue->ctrl->queues;
151 static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue)
153 u32 queue_idx = nvme_tcp_queue_id(queue);
156 return queue->ctrl->admin_tag_set.tags[queue_idx];
157 return queue->ctrl->tag_set.tags[queue_idx - 1];
160 static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue)
162 return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0;
165 static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue)
167 return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0;
170 static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_queue *queue)
172 return queue->cmnd_capsule_len - sizeof(struct nvme_command);
175 static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req)
177 return req == &req->queue->ctrl->async_req;
180 static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req)
184 if (unlikely(nvme_tcp_async_req(req)))
185 return false; /* async events don't have a request */
187 rq = blk_mq_rq_from_pdu(req);
189 return rq_data_dir(rq) == WRITE && req->data_len &&
190 req->data_len <= nvme_tcp_inline_data_size(req->queue);
193 static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req)
195 return req->iter.bvec->bv_page;
198 static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req)
200 return req->iter.bvec->bv_offset + req->iter.iov_offset;
203 static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req)
205 return min_t(size_t, req->iter.bvec->bv_len - req->iter.iov_offset,
206 req->pdu_len - req->pdu_sent);
209 static inline size_t nvme_tcp_req_offset(struct nvme_tcp_request *req)
211 return req->iter.iov_offset;
214 static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req)
216 return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ?
217 req->pdu_len - req->pdu_sent : 0;
220 static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req,
223 return nvme_tcp_pdu_data_left(req) <= len;
226 static void nvme_tcp_init_iter(struct nvme_tcp_request *req,
229 struct request *rq = blk_mq_rq_from_pdu(req);
235 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) {
236 vec = &rq->special_vec;
238 size = blk_rq_payload_bytes(rq);
241 struct bio *bio = req->curr_bio;
243 vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
244 nsegs = bio_segments(bio);
245 size = bio->bi_iter.bi_size;
246 offset = bio->bi_iter.bi_bvec_done;
249 iov_iter_bvec(&req->iter, dir, vec, nsegs, size);
250 req->iter.iov_offset = offset;
253 static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req,
256 req->data_sent += len;
257 req->pdu_sent += len;
258 iov_iter_advance(&req->iter, len);
259 if (!iov_iter_count(&req->iter) &&
260 req->data_sent < req->data_len) {
261 req->curr_bio = req->curr_bio->bi_next;
262 nvme_tcp_init_iter(req, WRITE);
266 static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req,
267 bool sync, bool last)
269 struct nvme_tcp_queue *queue = req->queue;
272 empty = llist_add(&req->lentry, &queue->req_list) &&
273 list_empty(&queue->send_list) && !queue->request;
276 * if we're the first on the send_list and we can try to send
277 * directly, otherwise queue io_work. Also, only do that if we
278 * are on the same cpu, so we don't introduce contention.
280 if (queue->io_cpu == smp_processor_id() &&
281 sync && empty && mutex_trylock(&queue->send_mutex)) {
282 queue->more_requests = !last;
283 nvme_tcp_try_send(queue);
284 queue->more_requests = false;
285 mutex_unlock(&queue->send_mutex);
287 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
291 static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue)
293 struct nvme_tcp_request *req;
294 struct llist_node *node;
296 for (node = llist_del_all(&queue->req_list); node; node = node->next) {
297 req = llist_entry(node, struct nvme_tcp_request, lentry);
298 list_add(&req->entry, &queue->send_list);
302 static inline struct nvme_tcp_request *
303 nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)
305 struct nvme_tcp_request *req;
307 req = list_first_entry_or_null(&queue->send_list,
308 struct nvme_tcp_request, entry);
310 nvme_tcp_process_req_list(queue);
311 req = list_first_entry_or_null(&queue->send_list,
312 struct nvme_tcp_request, entry);
317 list_del(&req->entry);
321 static inline void nvme_tcp_ddgst_final(struct ahash_request *hash,
324 ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0);
325 crypto_ahash_final(hash);
328 static inline void nvme_tcp_ddgst_update(struct ahash_request *hash,
329 struct page *page, off_t off, size_t len)
331 struct scatterlist sg;
333 sg_init_marker(&sg, 1);
334 sg_set_page(&sg, page, len, off);
335 ahash_request_set_crypt(hash, &sg, NULL, len);
336 crypto_ahash_update(hash);
339 static inline void nvme_tcp_hdgst(struct ahash_request *hash,
340 void *pdu, size_t len)
342 struct scatterlist sg;
344 sg_init_one(&sg, pdu, len);
345 ahash_request_set_crypt(hash, &sg, pdu + len, len);
346 crypto_ahash_digest(hash);
349 static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
350 void *pdu, size_t pdu_len)
352 struct nvme_tcp_hdr *hdr = pdu;
356 if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) {
357 dev_err(queue->ctrl->ctrl.device,
358 "queue %d: header digest flag is cleared\n",
359 nvme_tcp_queue_id(queue));
363 recv_digest = *(__le32 *)(pdu + hdr->hlen);
364 nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len);
365 exp_digest = *(__le32 *)(pdu + hdr->hlen);
366 if (recv_digest != exp_digest) {
367 dev_err(queue->ctrl->ctrl.device,
368 "header digest error: recv %#x expected %#x\n",
369 le32_to_cpu(recv_digest), le32_to_cpu(exp_digest));
376 static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)
378 struct nvme_tcp_hdr *hdr = pdu;
379 u8 digest_len = nvme_tcp_hdgst_len(queue);
382 len = le32_to_cpu(hdr->plen) - hdr->hlen -
383 ((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0);
385 if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) {
386 dev_err(queue->ctrl->ctrl.device,
387 "queue %d: data digest flag is cleared\n",
388 nvme_tcp_queue_id(queue));
391 crypto_ahash_init(queue->rcv_hash);
396 static void nvme_tcp_exit_request(struct blk_mq_tag_set *set,
397 struct request *rq, unsigned int hctx_idx)
399 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
401 page_frag_free(req->pdu);
404 static int nvme_tcp_init_request(struct blk_mq_tag_set *set,
405 struct request *rq, unsigned int hctx_idx,
406 unsigned int numa_node)
408 struct nvme_tcp_ctrl *ctrl = set->driver_data;
409 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
410 int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
411 struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx];
412 u8 hdgst = nvme_tcp_hdgst_len(queue);
414 req->pdu = page_frag_alloc(&queue->pf_cache,
415 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
416 GFP_KERNEL | __GFP_ZERO);
421 nvme_req(rq)->ctrl = &ctrl->ctrl;
426 static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
427 unsigned int hctx_idx)
429 struct nvme_tcp_ctrl *ctrl = data;
430 struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1];
432 hctx->driver_data = queue;
436 static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
437 unsigned int hctx_idx)
439 struct nvme_tcp_ctrl *ctrl = data;
440 struct nvme_tcp_queue *queue = &ctrl->queues[0];
442 hctx->driver_data = queue;
446 static enum nvme_tcp_recv_state
447 nvme_tcp_recv_state(struct nvme_tcp_queue *queue)
449 return (queue->pdu_remaining) ? NVME_TCP_RECV_PDU :
450 (queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST :
454 static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue)
456 queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) +
457 nvme_tcp_hdgst_len(queue);
458 queue->pdu_offset = 0;
459 queue->data_remaining = -1;
460 queue->ddgst_remaining = 0;
463 static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl)
465 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
468 dev_warn(ctrl->device, "starting error recovery\n");
469 queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work);
472 static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue,
473 struct nvme_completion *cqe)
477 rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), cqe->command_id);
479 dev_err(queue->ctrl->ctrl.device,
480 "queue %d tag 0x%x not found\n",
481 nvme_tcp_queue_id(queue), cqe->command_id);
482 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
486 if (!nvme_try_complete_req(rq, cqe->status, cqe->result))
487 nvme_complete_rq(rq);
493 static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue,
494 struct nvme_tcp_data_pdu *pdu)
498 rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
500 dev_err(queue->ctrl->ctrl.device,
501 "queue %d tag %#x not found\n",
502 nvme_tcp_queue_id(queue), pdu->command_id);
506 if (!blk_rq_payload_bytes(rq)) {
507 dev_err(queue->ctrl->ctrl.device,
508 "queue %d tag %#x unexpected data\n",
509 nvme_tcp_queue_id(queue), rq->tag);
513 queue->data_remaining = le32_to_cpu(pdu->data_length);
515 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS &&
516 unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) {
517 dev_err(queue->ctrl->ctrl.device,
518 "queue %d tag %#x SUCCESS set but not last PDU\n",
519 nvme_tcp_queue_id(queue), rq->tag);
520 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
527 static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue,
528 struct nvme_tcp_rsp_pdu *pdu)
530 struct nvme_completion *cqe = &pdu->cqe;
534 * AEN requests are special as they don't time out and can
535 * survive any kind of queue freeze and often don't respond to
536 * aborts. We don't even bother to allocate a struct request
537 * for them but rather special case them here.
539 if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue),
541 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
544 ret = nvme_tcp_process_nvme_cqe(queue, cqe);
549 static int nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req,
550 struct nvme_tcp_r2t_pdu *pdu)
552 struct nvme_tcp_data_pdu *data = req->pdu;
553 struct nvme_tcp_queue *queue = req->queue;
554 struct request *rq = blk_mq_rq_from_pdu(req);
555 u8 hdgst = nvme_tcp_hdgst_len(queue);
556 u8 ddgst = nvme_tcp_ddgst_len(queue);
558 req->pdu_len = le32_to_cpu(pdu->r2t_length);
561 if (unlikely(req->data_sent + req->pdu_len > req->data_len)) {
562 dev_err(queue->ctrl->ctrl.device,
563 "req %d r2t len %u exceeded data len %u (%zu sent)\n",
564 rq->tag, req->pdu_len, req->data_len,
569 if (unlikely(le32_to_cpu(pdu->r2t_offset) < req->data_sent)) {
570 dev_err(queue->ctrl->ctrl.device,
571 "req %d unexpected r2t offset %u (expected %zu)\n",
572 rq->tag, le32_to_cpu(pdu->r2t_offset),
577 memset(data, 0, sizeof(*data));
578 data->hdr.type = nvme_tcp_h2c_data;
579 data->hdr.flags = NVME_TCP_F_DATA_LAST;
580 if (queue->hdr_digest)
581 data->hdr.flags |= NVME_TCP_F_HDGST;
582 if (queue->data_digest)
583 data->hdr.flags |= NVME_TCP_F_DDGST;
584 data->hdr.hlen = sizeof(*data);
585 data->hdr.pdo = data->hdr.hlen + hdgst;
587 cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst);
588 data->ttag = pdu->ttag;
589 data->command_id = rq->tag;
590 data->data_offset = cpu_to_le32(req->data_sent);
591 data->data_length = cpu_to_le32(req->pdu_len);
595 static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,
596 struct nvme_tcp_r2t_pdu *pdu)
598 struct nvme_tcp_request *req;
602 rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
604 dev_err(queue->ctrl->ctrl.device,
605 "queue %d tag %#x not found\n",
606 nvme_tcp_queue_id(queue), pdu->command_id);
609 req = blk_mq_rq_to_pdu(rq);
611 ret = nvme_tcp_setup_h2c_data_pdu(req, pdu);
615 req->state = NVME_TCP_SEND_H2C_PDU;
618 nvme_tcp_queue_request(req, false, true);
623 static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb,
624 unsigned int *offset, size_t *len)
626 struct nvme_tcp_hdr *hdr;
627 char *pdu = queue->pdu;
628 size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining);
631 ret = skb_copy_bits(skb, *offset,
632 &pdu[queue->pdu_offset], rcv_len);
636 queue->pdu_remaining -= rcv_len;
637 queue->pdu_offset += rcv_len;
640 if (queue->pdu_remaining)
644 if (queue->hdr_digest) {
645 ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen);
651 if (queue->data_digest) {
652 ret = nvme_tcp_check_ddgst(queue, queue->pdu);
658 case nvme_tcp_c2h_data:
659 return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu);
661 nvme_tcp_init_recv_ctx(queue);
662 return nvme_tcp_handle_comp(queue, (void *)queue->pdu);
664 nvme_tcp_init_recv_ctx(queue);
665 return nvme_tcp_handle_r2t(queue, (void *)queue->pdu);
667 dev_err(queue->ctrl->ctrl.device,
668 "unsupported pdu type (%d)\n", hdr->type);
673 static inline void nvme_tcp_end_request(struct request *rq, u16 status)
675 union nvme_result res = {};
677 if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res))
678 nvme_complete_rq(rq);
681 static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
682 unsigned int *offset, size_t *len)
684 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
685 struct nvme_tcp_request *req;
688 rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
690 dev_err(queue->ctrl->ctrl.device,
691 "queue %d tag %#x not found\n",
692 nvme_tcp_queue_id(queue), pdu->command_id);
695 req = blk_mq_rq_to_pdu(rq);
700 recv_len = min_t(size_t, *len, queue->data_remaining);
704 if (!iov_iter_count(&req->iter)) {
705 req->curr_bio = req->curr_bio->bi_next;
708 * If we don`t have any bios it means that controller
709 * sent more data than we requested, hence error
711 if (!req->curr_bio) {
712 dev_err(queue->ctrl->ctrl.device,
713 "queue %d no space in request %#x",
714 nvme_tcp_queue_id(queue), rq->tag);
715 nvme_tcp_init_recv_ctx(queue);
718 nvme_tcp_init_iter(req, READ);
721 /* we can read only from what is left in this bio */
722 recv_len = min_t(size_t, recv_len,
723 iov_iter_count(&req->iter));
725 if (queue->data_digest)
726 ret = skb_copy_and_hash_datagram_iter(skb, *offset,
727 &req->iter, recv_len, queue->rcv_hash);
729 ret = skb_copy_datagram_iter(skb, *offset,
730 &req->iter, recv_len);
732 dev_err(queue->ctrl->ctrl.device,
733 "queue %d failed to copy request %#x data",
734 nvme_tcp_queue_id(queue), rq->tag);
740 queue->data_remaining -= recv_len;
743 if (!queue->data_remaining) {
744 if (queue->data_digest) {
745 nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst);
746 queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH;
748 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
749 nvme_tcp_end_request(rq, NVME_SC_SUCCESS);
752 nvme_tcp_init_recv_ctx(queue);
759 static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue,
760 struct sk_buff *skb, unsigned int *offset, size_t *len)
762 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
763 char *ddgst = (char *)&queue->recv_ddgst;
764 size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining);
765 off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining;
768 ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len);
772 queue->ddgst_remaining -= recv_len;
775 if (queue->ddgst_remaining)
778 if (queue->recv_ddgst != queue->exp_ddgst) {
779 dev_err(queue->ctrl->ctrl.device,
780 "data digest error: recv %#x expected %#x\n",
781 le32_to_cpu(queue->recv_ddgst),
782 le32_to_cpu(queue->exp_ddgst));
786 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
787 struct request *rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue),
790 nvme_tcp_end_request(rq, NVME_SC_SUCCESS);
794 nvme_tcp_init_recv_ctx(queue);
798 static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
799 unsigned int offset, size_t len)
801 struct nvme_tcp_queue *queue = desc->arg.data;
802 size_t consumed = len;
806 switch (nvme_tcp_recv_state(queue)) {
807 case NVME_TCP_RECV_PDU:
808 result = nvme_tcp_recv_pdu(queue, skb, &offset, &len);
810 case NVME_TCP_RECV_DATA:
811 result = nvme_tcp_recv_data(queue, skb, &offset, &len);
813 case NVME_TCP_RECV_DDGST:
814 result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len);
820 dev_err(queue->ctrl->ctrl.device,
821 "receive failed: %d\n", result);
822 queue->rd_enabled = false;
823 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
831 static void nvme_tcp_data_ready(struct sock *sk)
833 struct nvme_tcp_queue *queue;
835 read_lock_bh(&sk->sk_callback_lock);
836 queue = sk->sk_user_data;
837 if (likely(queue && queue->rd_enabled) &&
838 !test_bit(NVME_TCP_Q_POLLING, &queue->flags))
839 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
840 read_unlock_bh(&sk->sk_callback_lock);
843 static void nvme_tcp_write_space(struct sock *sk)
845 struct nvme_tcp_queue *queue;
847 read_lock_bh(&sk->sk_callback_lock);
848 queue = sk->sk_user_data;
849 if (likely(queue && sk_stream_is_writeable(sk))) {
850 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
851 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
853 read_unlock_bh(&sk->sk_callback_lock);
856 static void nvme_tcp_state_change(struct sock *sk)
858 struct nvme_tcp_queue *queue;
860 read_lock(&sk->sk_callback_lock);
861 queue = sk->sk_user_data;
865 switch (sk->sk_state) {
871 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
874 dev_info(queue->ctrl->ctrl.device,
875 "queue %d socket state %d\n",
876 nvme_tcp_queue_id(queue), sk->sk_state);
879 queue->state_change(sk);
881 read_unlock(&sk->sk_callback_lock);
884 static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue)
886 return !list_empty(&queue->send_list) ||
887 !llist_empty(&queue->req_list) || queue->more_requests;
890 static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue)
892 queue->request = NULL;
895 static void nvme_tcp_fail_request(struct nvme_tcp_request *req)
897 nvme_tcp_end_request(blk_mq_rq_from_pdu(req), NVME_SC_HOST_PATH_ERROR);
900 static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
902 struct nvme_tcp_queue *queue = req->queue;
905 struct page *page = nvme_tcp_req_cur_page(req);
906 size_t offset = nvme_tcp_req_cur_offset(req);
907 size_t len = nvme_tcp_req_cur_length(req);
908 bool last = nvme_tcp_pdu_last_send(req, len);
909 int ret, flags = MSG_DONTWAIT;
911 if (last && !queue->data_digest && !nvme_tcp_queue_more(queue))
914 flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
916 if (sendpage_ok(page)) {
917 ret = kernel_sendpage(queue->sock, page, offset, len,
920 ret = sock_no_sendpage(queue->sock, page, offset, len,
926 nvme_tcp_advance_req(req, ret);
927 if (queue->data_digest)
928 nvme_tcp_ddgst_update(queue->snd_hash, page,
931 /* fully successful last write*/
932 if (last && ret == len) {
933 if (queue->data_digest) {
934 nvme_tcp_ddgst_final(queue->snd_hash,
936 req->state = NVME_TCP_SEND_DDGST;
939 nvme_tcp_done_send_req(queue);
947 static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
949 struct nvme_tcp_queue *queue = req->queue;
950 struct nvme_tcp_cmd_pdu *pdu = req->pdu;
951 bool inline_data = nvme_tcp_has_inline_data(req);
952 u8 hdgst = nvme_tcp_hdgst_len(queue);
953 int len = sizeof(*pdu) + hdgst - req->offset;
954 int flags = MSG_DONTWAIT;
957 if (inline_data || nvme_tcp_queue_more(queue))
958 flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
962 if (queue->hdr_digest && !req->offset)
963 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
965 ret = kernel_sendpage(queue->sock, virt_to_page(pdu),
966 offset_in_page(pdu) + req->offset, len, flags);
967 if (unlikely(ret <= 0))
973 req->state = NVME_TCP_SEND_DATA;
974 if (queue->data_digest)
975 crypto_ahash_init(queue->snd_hash);
976 nvme_tcp_init_iter(req, WRITE);
978 nvme_tcp_done_send_req(queue);
987 static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
989 struct nvme_tcp_queue *queue = req->queue;
990 struct nvme_tcp_data_pdu *pdu = req->pdu;
991 u8 hdgst = nvme_tcp_hdgst_len(queue);
992 int len = sizeof(*pdu) - req->offset + hdgst;
995 if (queue->hdr_digest && !req->offset)
996 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
998 ret = kernel_sendpage(queue->sock, virt_to_page(pdu),
999 offset_in_page(pdu) + req->offset, len,
1000 MSG_DONTWAIT | MSG_MORE | MSG_SENDPAGE_NOTLAST);
1001 if (unlikely(ret <= 0))
1006 req->state = NVME_TCP_SEND_DATA;
1007 if (queue->data_digest)
1008 crypto_ahash_init(queue->snd_hash);
1009 if (!req->data_sent)
1010 nvme_tcp_init_iter(req, WRITE);
1018 static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req)
1020 struct nvme_tcp_queue *queue = req->queue;
1022 struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
1024 .iov_base = &req->ddgst + req->offset,
1025 .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset
1028 if (nvme_tcp_queue_more(queue))
1029 msg.msg_flags |= MSG_MORE;
1031 msg.msg_flags |= MSG_EOR;
1033 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1034 if (unlikely(ret <= 0))
1037 if (req->offset + ret == NVME_TCP_DIGEST_LENGTH) {
1038 nvme_tcp_done_send_req(queue);
1046 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue)
1048 struct nvme_tcp_request *req;
1051 if (!queue->request) {
1052 queue->request = nvme_tcp_fetch_request(queue);
1053 if (!queue->request)
1056 req = queue->request;
1058 if (req->state == NVME_TCP_SEND_CMD_PDU) {
1059 ret = nvme_tcp_try_send_cmd_pdu(req);
1062 if (!nvme_tcp_has_inline_data(req))
1066 if (req->state == NVME_TCP_SEND_H2C_PDU) {
1067 ret = nvme_tcp_try_send_data_pdu(req);
1072 if (req->state == NVME_TCP_SEND_DATA) {
1073 ret = nvme_tcp_try_send_data(req);
1078 if (req->state == NVME_TCP_SEND_DDGST)
1079 ret = nvme_tcp_try_send_ddgst(req);
1081 if (ret == -EAGAIN) {
1083 } else if (ret < 0) {
1084 dev_err(queue->ctrl->ctrl.device,
1085 "failed to send request %d\n", ret);
1086 if (ret != -EPIPE && ret != -ECONNRESET)
1087 nvme_tcp_fail_request(queue->request);
1088 nvme_tcp_done_send_req(queue);
1093 static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue)
1095 struct socket *sock = queue->sock;
1096 struct sock *sk = sock->sk;
1097 read_descriptor_t rd_desc;
1100 rd_desc.arg.data = queue;
1104 consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb);
1109 static void nvme_tcp_io_work(struct work_struct *w)
1111 struct nvme_tcp_queue *queue =
1112 container_of(w, struct nvme_tcp_queue, io_work);
1113 unsigned long deadline = jiffies + msecs_to_jiffies(1);
1116 bool pending = false;
1119 if (mutex_trylock(&queue->send_mutex)) {
1120 result = nvme_tcp_try_send(queue);
1121 mutex_unlock(&queue->send_mutex);
1124 else if (unlikely(result < 0))
1128 result = nvme_tcp_try_recv(queue);
1131 else if (unlikely(result < 0))
1137 } while (!time_after(jiffies, deadline)); /* quota is exhausted */
1139 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
1142 static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue)
1144 struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash);
1146 ahash_request_free(queue->rcv_hash);
1147 ahash_request_free(queue->snd_hash);
1148 crypto_free_ahash(tfm);
1151 static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue)
1153 struct crypto_ahash *tfm;
1155 tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
1157 return PTR_ERR(tfm);
1159 queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1160 if (!queue->snd_hash)
1162 ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL);
1164 queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1165 if (!queue->rcv_hash)
1167 ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL);
1171 ahash_request_free(queue->snd_hash);
1173 crypto_free_ahash(tfm);
1177 static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl)
1179 struct nvme_tcp_request *async = &ctrl->async_req;
1181 page_frag_free(async->pdu);
1184 static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl)
1186 struct nvme_tcp_queue *queue = &ctrl->queues[0];
1187 struct nvme_tcp_request *async = &ctrl->async_req;
1188 u8 hdgst = nvme_tcp_hdgst_len(queue);
1190 async->pdu = page_frag_alloc(&queue->pf_cache,
1191 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
1192 GFP_KERNEL | __GFP_ZERO);
1196 async->queue = &ctrl->queues[0];
1200 static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
1202 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1203 struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1205 if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
1208 if (queue->hdr_digest || queue->data_digest)
1209 nvme_tcp_free_crypto(queue);
1211 sock_release(queue->sock);
1215 static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
1217 struct nvme_tcp_icreq_pdu *icreq;
1218 struct nvme_tcp_icresp_pdu *icresp;
1219 struct msghdr msg = {};
1221 bool ctrl_hdgst, ctrl_ddgst;
1224 icreq = kzalloc(sizeof(*icreq), GFP_KERNEL);
1228 icresp = kzalloc(sizeof(*icresp), GFP_KERNEL);
1234 icreq->hdr.type = nvme_tcp_icreq;
1235 icreq->hdr.hlen = sizeof(*icreq);
1237 icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen);
1238 icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0);
1239 icreq->maxr2t = 0; /* single inflight r2t supported */
1240 icreq->hpda = 0; /* no alignment constraint */
1241 if (queue->hdr_digest)
1242 icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE;
1243 if (queue->data_digest)
1244 icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE;
1246 iov.iov_base = icreq;
1247 iov.iov_len = sizeof(*icreq);
1248 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1252 memset(&msg, 0, sizeof(msg));
1253 iov.iov_base = icresp;
1254 iov.iov_len = sizeof(*icresp);
1255 ret = kernel_recvmsg(queue->sock, &msg, &iov, 1,
1256 iov.iov_len, msg.msg_flags);
1261 if (icresp->hdr.type != nvme_tcp_icresp) {
1262 pr_err("queue %d: bad type returned %d\n",
1263 nvme_tcp_queue_id(queue), icresp->hdr.type);
1267 if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) {
1268 pr_err("queue %d: bad pdu length returned %d\n",
1269 nvme_tcp_queue_id(queue), icresp->hdr.plen);
1273 if (icresp->pfv != NVME_TCP_PFV_1_0) {
1274 pr_err("queue %d: bad pfv returned %d\n",
1275 nvme_tcp_queue_id(queue), icresp->pfv);
1279 ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE);
1280 if ((queue->data_digest && !ctrl_ddgst) ||
1281 (!queue->data_digest && ctrl_ddgst)) {
1282 pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n",
1283 nvme_tcp_queue_id(queue),
1284 queue->data_digest ? "enabled" : "disabled",
1285 ctrl_ddgst ? "enabled" : "disabled");
1289 ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE);
1290 if ((queue->hdr_digest && !ctrl_hdgst) ||
1291 (!queue->hdr_digest && ctrl_hdgst)) {
1292 pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n",
1293 nvme_tcp_queue_id(queue),
1294 queue->hdr_digest ? "enabled" : "disabled",
1295 ctrl_hdgst ? "enabled" : "disabled");
1299 if (icresp->cpda != 0) {
1300 pr_err("queue %d: unsupported cpda returned %d\n",
1301 nvme_tcp_queue_id(queue), icresp->cpda);
1313 static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue)
1315 return nvme_tcp_queue_id(queue) == 0;
1318 static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue)
1320 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1321 int qid = nvme_tcp_queue_id(queue);
1323 return !nvme_tcp_admin_queue(queue) &&
1324 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT];
1327 static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue)
1329 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1330 int qid = nvme_tcp_queue_id(queue);
1332 return !nvme_tcp_admin_queue(queue) &&
1333 !nvme_tcp_default_queue(queue) &&
1334 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1335 ctrl->io_queues[HCTX_TYPE_READ];
1338 static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue)
1340 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1341 int qid = nvme_tcp_queue_id(queue);
1343 return !nvme_tcp_admin_queue(queue) &&
1344 !nvme_tcp_default_queue(queue) &&
1345 !nvme_tcp_read_queue(queue) &&
1346 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1347 ctrl->io_queues[HCTX_TYPE_READ] +
1348 ctrl->io_queues[HCTX_TYPE_POLL];
1351 static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
1353 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1354 int qid = nvme_tcp_queue_id(queue);
1357 if (nvme_tcp_default_queue(queue))
1359 else if (nvme_tcp_read_queue(queue))
1360 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1;
1361 else if (nvme_tcp_poll_queue(queue))
1362 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] -
1363 ctrl->io_queues[HCTX_TYPE_READ] - 1;
1364 queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false);
1367 static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl,
1368 int qid, size_t queue_size)
1370 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1371 struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1372 int ret, rcv_pdu_size;
1375 init_llist_head(&queue->req_list);
1376 INIT_LIST_HEAD(&queue->send_list);
1377 mutex_init(&queue->send_mutex);
1378 INIT_WORK(&queue->io_work, nvme_tcp_io_work);
1379 queue->queue_size = queue_size;
1382 queue->cmnd_capsule_len = nctrl->ioccsz * 16;
1384 queue->cmnd_capsule_len = sizeof(struct nvme_command) +
1385 NVME_TCP_ADMIN_CCSZ;
1387 ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM,
1388 IPPROTO_TCP, &queue->sock);
1390 dev_err(nctrl->device,
1391 "failed to create socket: %d\n", ret);
1395 /* Single syn retry */
1396 tcp_sock_set_syncnt(queue->sock->sk, 1);
1398 /* Set TCP no delay */
1399 tcp_sock_set_nodelay(queue->sock->sk);
1402 * Cleanup whatever is sitting in the TCP transmit queue on socket
1403 * close. This is done to prevent stale data from being sent should
1404 * the network connection be restored before TCP times out.
1406 sock_no_linger(queue->sock->sk);
1408 if (so_priority > 0)
1409 sock_set_priority(queue->sock->sk, so_priority);
1411 /* Set socket type of service */
1412 if (nctrl->opts->tos >= 0)
1413 ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos);
1415 /* Set 10 seconds timeout for icresp recvmsg */
1416 queue->sock->sk->sk_rcvtimeo = 10 * HZ;
1418 queue->sock->sk->sk_allocation = GFP_ATOMIC;
1419 nvme_tcp_set_queue_io_cpu(queue);
1420 queue->request = NULL;
1421 queue->data_remaining = 0;
1422 queue->ddgst_remaining = 0;
1423 queue->pdu_remaining = 0;
1424 queue->pdu_offset = 0;
1425 sk_set_memalloc(queue->sock->sk);
1427 if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) {
1428 ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr,
1429 sizeof(ctrl->src_addr));
1431 dev_err(nctrl->device,
1432 "failed to bind queue %d socket %d\n",
1438 queue->hdr_digest = nctrl->opts->hdr_digest;
1439 queue->data_digest = nctrl->opts->data_digest;
1440 if (queue->hdr_digest || queue->data_digest) {
1441 ret = nvme_tcp_alloc_crypto(queue);
1443 dev_err(nctrl->device,
1444 "failed to allocate queue %d crypto\n", qid);
1449 rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) +
1450 nvme_tcp_hdgst_len(queue);
1451 queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);
1457 dev_dbg(nctrl->device, "connecting queue %d\n",
1458 nvme_tcp_queue_id(queue));
1460 ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr,
1461 sizeof(ctrl->addr), 0);
1463 dev_err(nctrl->device,
1464 "failed to connect socket: %d\n", ret);
1468 ret = nvme_tcp_init_connection(queue);
1470 goto err_init_connect;
1472 queue->rd_enabled = true;
1473 set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags);
1474 nvme_tcp_init_recv_ctx(queue);
1476 write_lock_bh(&queue->sock->sk->sk_callback_lock);
1477 queue->sock->sk->sk_user_data = queue;
1478 queue->state_change = queue->sock->sk->sk_state_change;
1479 queue->data_ready = queue->sock->sk->sk_data_ready;
1480 queue->write_space = queue->sock->sk->sk_write_space;
1481 queue->sock->sk->sk_data_ready = nvme_tcp_data_ready;
1482 queue->sock->sk->sk_state_change = nvme_tcp_state_change;
1483 queue->sock->sk->sk_write_space = nvme_tcp_write_space;
1484 #ifdef CONFIG_NET_RX_BUSY_POLL
1485 queue->sock->sk->sk_ll_usec = 1;
1487 write_unlock_bh(&queue->sock->sk->sk_callback_lock);
1492 kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1496 if (queue->hdr_digest || queue->data_digest)
1497 nvme_tcp_free_crypto(queue);
1499 sock_release(queue->sock);
1504 static void nvme_tcp_restore_sock_calls(struct nvme_tcp_queue *queue)
1506 struct socket *sock = queue->sock;
1508 write_lock_bh(&sock->sk->sk_callback_lock);
1509 sock->sk->sk_user_data = NULL;
1510 sock->sk->sk_data_ready = queue->data_ready;
1511 sock->sk->sk_state_change = queue->state_change;
1512 sock->sk->sk_write_space = queue->write_space;
1513 write_unlock_bh(&sock->sk->sk_callback_lock);
1516 static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue)
1518 kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1519 nvme_tcp_restore_sock_calls(queue);
1520 cancel_work_sync(&queue->io_work);
1523 static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid)
1525 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1526 struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1528 if (!test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags))
1530 __nvme_tcp_stop_queue(queue);
1533 static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx)
1535 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1539 ret = nvmf_connect_io_queue(nctrl, idx, false);
1541 ret = nvmf_connect_admin_queue(nctrl);
1544 set_bit(NVME_TCP_Q_LIVE, &ctrl->queues[idx].flags);
1546 if (test_bit(NVME_TCP_Q_ALLOCATED, &ctrl->queues[idx].flags))
1547 __nvme_tcp_stop_queue(&ctrl->queues[idx]);
1548 dev_err(nctrl->device,
1549 "failed to connect queue: %d ret=%d\n", idx, ret);
1554 static struct blk_mq_tag_set *nvme_tcp_alloc_tagset(struct nvme_ctrl *nctrl,
1557 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1558 struct blk_mq_tag_set *set;
1562 set = &ctrl->admin_tag_set;
1563 memset(set, 0, sizeof(*set));
1564 set->ops = &nvme_tcp_admin_mq_ops;
1565 set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
1566 set->reserved_tags = 2; /* connect + keep-alive */
1567 set->numa_node = nctrl->numa_node;
1568 set->flags = BLK_MQ_F_BLOCKING;
1569 set->cmd_size = sizeof(struct nvme_tcp_request);
1570 set->driver_data = ctrl;
1571 set->nr_hw_queues = 1;
1572 set->timeout = ADMIN_TIMEOUT;
1574 set = &ctrl->tag_set;
1575 memset(set, 0, sizeof(*set));
1576 set->ops = &nvme_tcp_mq_ops;
1577 set->queue_depth = nctrl->sqsize + 1;
1578 set->reserved_tags = 1; /* fabric connect */
1579 set->numa_node = nctrl->numa_node;
1580 set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING;
1581 set->cmd_size = sizeof(struct nvme_tcp_request);
1582 set->driver_data = ctrl;
1583 set->nr_hw_queues = nctrl->queue_count - 1;
1584 set->timeout = NVME_IO_TIMEOUT;
1585 set->nr_maps = nctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2;
1588 ret = blk_mq_alloc_tag_set(set);
1590 return ERR_PTR(ret);
1595 static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl)
1597 if (to_tcp_ctrl(ctrl)->async_req.pdu) {
1598 cancel_work_sync(&ctrl->async_event_work);
1599 nvme_tcp_free_async_req(to_tcp_ctrl(ctrl));
1600 to_tcp_ctrl(ctrl)->async_req.pdu = NULL;
1603 nvme_tcp_free_queue(ctrl, 0);
1606 static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl)
1610 for (i = 1; i < ctrl->queue_count; i++)
1611 nvme_tcp_free_queue(ctrl, i);
1614 static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl)
1618 for (i = 1; i < ctrl->queue_count; i++)
1619 nvme_tcp_stop_queue(ctrl, i);
1622 static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl)
1626 for (i = 1; i < ctrl->queue_count; i++) {
1627 ret = nvme_tcp_start_queue(ctrl, i);
1629 goto out_stop_queues;
1635 for (i--; i >= 1; i--)
1636 nvme_tcp_stop_queue(ctrl, i);
1640 static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl)
1644 ret = nvme_tcp_alloc_queue(ctrl, 0, NVME_AQ_DEPTH);
1648 ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl));
1650 goto out_free_queue;
1655 nvme_tcp_free_queue(ctrl, 0);
1659 static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1663 for (i = 1; i < ctrl->queue_count; i++) {
1664 ret = nvme_tcp_alloc_queue(ctrl, i,
1667 goto out_free_queues;
1673 for (i--; i >= 1; i--)
1674 nvme_tcp_free_queue(ctrl, i);
1679 static unsigned int nvme_tcp_nr_io_queues(struct nvme_ctrl *ctrl)
1681 unsigned int nr_io_queues;
1683 nr_io_queues = min(ctrl->opts->nr_io_queues, num_online_cpus());
1684 nr_io_queues += min(ctrl->opts->nr_write_queues, num_online_cpus());
1685 nr_io_queues += min(ctrl->opts->nr_poll_queues, num_online_cpus());
1687 return nr_io_queues;
1690 static void nvme_tcp_set_io_queues(struct nvme_ctrl *nctrl,
1691 unsigned int nr_io_queues)
1693 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1694 struct nvmf_ctrl_options *opts = nctrl->opts;
1696 if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) {
1698 * separate read/write queues
1699 * hand out dedicated default queues only after we have
1700 * sufficient read queues.
1702 ctrl->io_queues[HCTX_TYPE_READ] = opts->nr_io_queues;
1703 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ];
1704 ctrl->io_queues[HCTX_TYPE_DEFAULT] =
1705 min(opts->nr_write_queues, nr_io_queues);
1706 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
1709 * shared read/write queues
1710 * either no write queues were requested, or we don't have
1711 * sufficient queue count to have dedicated default queues.
1713 ctrl->io_queues[HCTX_TYPE_DEFAULT] =
1714 min(opts->nr_io_queues, nr_io_queues);
1715 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
1718 if (opts->nr_poll_queues && nr_io_queues) {
1719 /* map dedicated poll queues only if we have queues left */
1720 ctrl->io_queues[HCTX_TYPE_POLL] =
1721 min(opts->nr_poll_queues, nr_io_queues);
1725 static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1727 unsigned int nr_io_queues;
1730 nr_io_queues = nvme_tcp_nr_io_queues(ctrl);
1731 ret = nvme_set_queue_count(ctrl, &nr_io_queues);
1735 ctrl->queue_count = nr_io_queues + 1;
1736 if (ctrl->queue_count < 2)
1739 dev_info(ctrl->device,
1740 "creating %d I/O queues.\n", nr_io_queues);
1742 nvme_tcp_set_io_queues(ctrl, nr_io_queues);
1744 return __nvme_tcp_alloc_io_queues(ctrl);
1747 static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove)
1749 nvme_tcp_stop_io_queues(ctrl);
1751 blk_cleanup_queue(ctrl->connect_q);
1752 blk_mq_free_tag_set(ctrl->tagset);
1754 nvme_tcp_free_io_queues(ctrl);
1757 static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
1761 ret = nvme_tcp_alloc_io_queues(ctrl);
1766 ctrl->tagset = nvme_tcp_alloc_tagset(ctrl, false);
1767 if (IS_ERR(ctrl->tagset)) {
1768 ret = PTR_ERR(ctrl->tagset);
1769 goto out_free_io_queues;
1772 ctrl->connect_q = blk_mq_init_queue(ctrl->tagset);
1773 if (IS_ERR(ctrl->connect_q)) {
1774 ret = PTR_ERR(ctrl->connect_q);
1775 goto out_free_tag_set;
1779 ret = nvme_tcp_start_io_queues(ctrl);
1781 goto out_cleanup_connect_q;
1784 nvme_start_queues(ctrl);
1785 if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) {
1787 * If we timed out waiting for freeze we are likely to
1788 * be stuck. Fail the controller initialization just
1792 goto out_wait_freeze_timed_out;
1794 blk_mq_update_nr_hw_queues(ctrl->tagset,
1795 ctrl->queue_count - 1);
1796 nvme_unfreeze(ctrl);
1801 out_wait_freeze_timed_out:
1802 nvme_stop_queues(ctrl);
1803 nvme_tcp_stop_io_queues(ctrl);
1804 out_cleanup_connect_q:
1806 blk_cleanup_queue(ctrl->connect_q);
1809 blk_mq_free_tag_set(ctrl->tagset);
1811 nvme_tcp_free_io_queues(ctrl);
1815 static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove)
1817 nvme_tcp_stop_queue(ctrl, 0);
1819 blk_cleanup_queue(ctrl->admin_q);
1820 blk_cleanup_queue(ctrl->fabrics_q);
1821 blk_mq_free_tag_set(ctrl->admin_tagset);
1823 nvme_tcp_free_admin_queue(ctrl);
1826 static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new)
1830 error = nvme_tcp_alloc_admin_queue(ctrl);
1835 ctrl->admin_tagset = nvme_tcp_alloc_tagset(ctrl, true);
1836 if (IS_ERR(ctrl->admin_tagset)) {
1837 error = PTR_ERR(ctrl->admin_tagset);
1838 goto out_free_queue;
1841 ctrl->fabrics_q = blk_mq_init_queue(ctrl->admin_tagset);
1842 if (IS_ERR(ctrl->fabrics_q)) {
1843 error = PTR_ERR(ctrl->fabrics_q);
1844 goto out_free_tagset;
1847 ctrl->admin_q = blk_mq_init_queue(ctrl->admin_tagset);
1848 if (IS_ERR(ctrl->admin_q)) {
1849 error = PTR_ERR(ctrl->admin_q);
1850 goto out_cleanup_fabrics_q;
1854 error = nvme_tcp_start_queue(ctrl, 0);
1856 goto out_cleanup_queue;
1858 error = nvme_enable_ctrl(ctrl);
1860 goto out_stop_queue;
1862 blk_mq_unquiesce_queue(ctrl->admin_q);
1864 error = nvme_init_identify(ctrl);
1866 goto out_stop_queue;
1871 nvme_tcp_stop_queue(ctrl, 0);
1874 blk_cleanup_queue(ctrl->admin_q);
1875 out_cleanup_fabrics_q:
1877 blk_cleanup_queue(ctrl->fabrics_q);
1880 blk_mq_free_tag_set(ctrl->admin_tagset);
1882 nvme_tcp_free_admin_queue(ctrl);
1886 static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl,
1889 mutex_lock(&to_tcp_ctrl(ctrl)->teardown_lock);
1890 blk_mq_quiesce_queue(ctrl->admin_q);
1891 nvme_tcp_stop_queue(ctrl, 0);
1892 if (ctrl->admin_tagset) {
1893 blk_mq_tagset_busy_iter(ctrl->admin_tagset,
1894 nvme_cancel_request, ctrl);
1895 blk_mq_tagset_wait_completed_request(ctrl->admin_tagset);
1898 blk_mq_unquiesce_queue(ctrl->admin_q);
1899 nvme_tcp_destroy_admin_queue(ctrl, remove);
1900 mutex_unlock(&to_tcp_ctrl(ctrl)->teardown_lock);
1903 static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl,
1906 mutex_lock(&to_tcp_ctrl(ctrl)->teardown_lock);
1907 if (ctrl->queue_count <= 1)
1909 blk_mq_quiesce_queue(ctrl->admin_q);
1910 nvme_start_freeze(ctrl);
1911 nvme_stop_queues(ctrl);
1912 nvme_tcp_stop_io_queues(ctrl);
1914 blk_mq_tagset_busy_iter(ctrl->tagset,
1915 nvme_cancel_request, ctrl);
1916 blk_mq_tagset_wait_completed_request(ctrl->tagset);
1919 nvme_start_queues(ctrl);
1920 nvme_tcp_destroy_io_queues(ctrl, remove);
1922 mutex_unlock(&to_tcp_ctrl(ctrl)->teardown_lock);
1925 static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)
1927 /* If we are resetting/deleting then do nothing */
1928 if (ctrl->state != NVME_CTRL_CONNECTING) {
1929 WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW ||
1930 ctrl->state == NVME_CTRL_LIVE);
1934 if (nvmf_should_reconnect(ctrl)) {
1935 dev_info(ctrl->device, "Reconnecting in %d seconds...\n",
1936 ctrl->opts->reconnect_delay);
1937 queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work,
1938 ctrl->opts->reconnect_delay * HZ);
1940 dev_info(ctrl->device, "Removing controller...\n");
1941 nvme_delete_ctrl(ctrl);
1945 static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new)
1947 struct nvmf_ctrl_options *opts = ctrl->opts;
1950 ret = nvme_tcp_configure_admin_queue(ctrl, new);
1955 dev_err(ctrl->device, "icdoff is not supported!\n");
1959 if (opts->queue_size > ctrl->sqsize + 1)
1960 dev_warn(ctrl->device,
1961 "queue_size %zu > ctrl sqsize %u, clamping down\n",
1962 opts->queue_size, ctrl->sqsize + 1);
1964 if (ctrl->sqsize + 1 > ctrl->maxcmd) {
1965 dev_warn(ctrl->device,
1966 "sqsize %u > ctrl maxcmd %u, clamping down\n",
1967 ctrl->sqsize + 1, ctrl->maxcmd);
1968 ctrl->sqsize = ctrl->maxcmd - 1;
1971 if (ctrl->queue_count > 1) {
1972 ret = nvme_tcp_configure_io_queues(ctrl, new);
1977 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) {
1979 * state change failure is ok if we started ctrl delete,
1980 * unless we're during creation of a new controller to
1981 * avoid races with teardown flow.
1983 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
1984 ctrl->state != NVME_CTRL_DELETING_NOIO);
1990 nvme_start_ctrl(ctrl);
1994 if (ctrl->queue_count > 1)
1995 nvme_tcp_destroy_io_queues(ctrl, new);
1997 nvme_tcp_stop_queue(ctrl, 0);
1998 nvme_tcp_destroy_admin_queue(ctrl, new);
2002 static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)
2004 struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work),
2005 struct nvme_tcp_ctrl, connect_work);
2006 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2008 ++ctrl->nr_reconnects;
2010 if (nvme_tcp_setup_ctrl(ctrl, false))
2013 dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n",
2014 ctrl->nr_reconnects);
2016 ctrl->nr_reconnects = 0;
2021 dev_info(ctrl->device, "Failed reconnect attempt %d\n",
2022 ctrl->nr_reconnects);
2023 nvme_tcp_reconnect_or_remove(ctrl);
2026 static void nvme_tcp_error_recovery_work(struct work_struct *work)
2028 struct nvme_tcp_ctrl *tcp_ctrl = container_of(work,
2029 struct nvme_tcp_ctrl, err_work);
2030 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2032 nvme_stop_keep_alive(ctrl);
2033 nvme_tcp_teardown_io_queues(ctrl, false);
2034 /* unquiesce to fail fast pending requests */
2035 nvme_start_queues(ctrl);
2036 nvme_tcp_teardown_admin_queue(ctrl, false);
2037 blk_mq_unquiesce_queue(ctrl->admin_q);
2039 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2040 /* state change failure is ok if we started ctrl delete */
2041 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2042 ctrl->state != NVME_CTRL_DELETING_NOIO);
2046 nvme_tcp_reconnect_or_remove(ctrl);
2049 static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
2051 cancel_work_sync(&to_tcp_ctrl(ctrl)->err_work);
2052 cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work);
2054 nvme_tcp_teardown_io_queues(ctrl, shutdown);
2055 blk_mq_quiesce_queue(ctrl->admin_q);
2057 nvme_shutdown_ctrl(ctrl);
2059 nvme_disable_ctrl(ctrl);
2060 nvme_tcp_teardown_admin_queue(ctrl, shutdown);
2063 static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl)
2065 nvme_tcp_teardown_ctrl(ctrl, true);
2068 static void nvme_reset_ctrl_work(struct work_struct *work)
2070 struct nvme_ctrl *ctrl =
2071 container_of(work, struct nvme_ctrl, reset_work);
2073 nvme_stop_ctrl(ctrl);
2074 nvme_tcp_teardown_ctrl(ctrl, false);
2076 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2077 /* state change failure is ok if we started ctrl delete */
2078 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2079 ctrl->state != NVME_CTRL_DELETING_NOIO);
2083 if (nvme_tcp_setup_ctrl(ctrl, false))
2089 ++ctrl->nr_reconnects;
2090 nvme_tcp_reconnect_or_remove(ctrl);
2093 static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
2095 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
2097 if (list_empty(&ctrl->list))
2100 mutex_lock(&nvme_tcp_ctrl_mutex);
2101 list_del(&ctrl->list);
2102 mutex_unlock(&nvme_tcp_ctrl_mutex);
2104 nvmf_free_options(nctrl->opts);
2106 kfree(ctrl->queues);
2110 static void nvme_tcp_set_sg_null(struct nvme_command *c)
2112 struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2116 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2117 NVME_SGL_FMT_TRANSPORT_A;
2120 static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue,
2121 struct nvme_command *c, u32 data_len)
2123 struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2125 sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
2126 sg->length = cpu_to_le32(data_len);
2127 sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
2130 static void nvme_tcp_set_sg_host_data(struct nvme_command *c,
2133 struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2136 sg->length = cpu_to_le32(data_len);
2137 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2138 NVME_SGL_FMT_TRANSPORT_A;
2141 static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg)
2143 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg);
2144 struct nvme_tcp_queue *queue = &ctrl->queues[0];
2145 struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu;
2146 struct nvme_command *cmd = &pdu->cmd;
2147 u8 hdgst = nvme_tcp_hdgst_len(queue);
2149 memset(pdu, 0, sizeof(*pdu));
2150 pdu->hdr.type = nvme_tcp_cmd;
2151 if (queue->hdr_digest)
2152 pdu->hdr.flags |= NVME_TCP_F_HDGST;
2153 pdu->hdr.hlen = sizeof(*pdu);
2154 pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
2156 cmd->common.opcode = nvme_admin_async_event;
2157 cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH;
2158 cmd->common.flags |= NVME_CMD_SGL_METABUF;
2159 nvme_tcp_set_sg_null(cmd);
2161 ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU;
2162 ctrl->async_req.offset = 0;
2163 ctrl->async_req.curr_bio = NULL;
2164 ctrl->async_req.data_len = 0;
2166 nvme_tcp_queue_request(&ctrl->async_req, true, true);
2169 static void nvme_tcp_complete_timed_out(struct request *rq)
2171 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2172 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2174 /* fence other contexts that may complete the command */
2175 mutex_lock(&to_tcp_ctrl(ctrl)->teardown_lock);
2176 nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue));
2177 if (!blk_mq_request_completed(rq)) {
2178 nvme_req(rq)->status = NVME_SC_HOST_ABORTED_CMD;
2179 blk_mq_complete_request(rq);
2181 mutex_unlock(&to_tcp_ctrl(ctrl)->teardown_lock);
2184 static enum blk_eh_timer_return
2185 nvme_tcp_timeout(struct request *rq, bool reserved)
2187 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2188 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2189 struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2191 dev_warn(ctrl->device,
2192 "queue %d: timeout request %#x type %d\n",
2193 nvme_tcp_queue_id(req->queue), rq->tag, pdu->hdr.type);
2195 if (ctrl->state != NVME_CTRL_LIVE) {
2197 * If we are resetting, connecting or deleting we should
2198 * complete immediately because we may block controller
2199 * teardown or setup sequence
2200 * - ctrl disable/shutdown fabrics requests
2201 * - connect requests
2202 * - initialization admin requests
2203 * - I/O requests that entered after unquiescing and
2204 * the controller stopped responding
2206 * All other requests should be cancelled by the error
2207 * recovery work, so it's fine that we fail it here.
2209 nvme_tcp_complete_timed_out(rq);
2214 * LIVE state should trigger the normal error recovery which will
2215 * handle completing this request.
2217 nvme_tcp_error_recovery(ctrl);
2218 return BLK_EH_RESET_TIMER;
2221 static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue,
2224 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2225 struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2226 struct nvme_command *c = &pdu->cmd;
2228 c->common.flags |= NVME_CMD_SGL_METABUF;
2230 if (!blk_rq_nr_phys_segments(rq))
2231 nvme_tcp_set_sg_null(c);
2232 else if (rq_data_dir(rq) == WRITE &&
2233 req->data_len <= nvme_tcp_inline_data_size(queue))
2234 nvme_tcp_set_sg_inline(queue, c, req->data_len);
2236 nvme_tcp_set_sg_host_data(c, req->data_len);
2241 static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns,
2244 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2245 struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2246 struct nvme_tcp_queue *queue = req->queue;
2247 u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0;
2250 ret = nvme_setup_cmd(ns, rq, &pdu->cmd);
2254 req->state = NVME_TCP_SEND_CMD_PDU;
2259 req->data_len = blk_rq_nr_phys_segments(rq) ?
2260 blk_rq_payload_bytes(rq) : 0;
2261 req->curr_bio = rq->bio;
2263 if (rq_data_dir(rq) == WRITE &&
2264 req->data_len <= nvme_tcp_inline_data_size(queue))
2265 req->pdu_len = req->data_len;
2266 else if (req->curr_bio)
2267 nvme_tcp_init_iter(req, READ);
2269 pdu->hdr.type = nvme_tcp_cmd;
2271 if (queue->hdr_digest)
2272 pdu->hdr.flags |= NVME_TCP_F_HDGST;
2273 if (queue->data_digest && req->pdu_len) {
2274 pdu->hdr.flags |= NVME_TCP_F_DDGST;
2275 ddgst = nvme_tcp_ddgst_len(queue);
2277 pdu->hdr.hlen = sizeof(*pdu);
2278 pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0;
2280 cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst);
2282 ret = nvme_tcp_map_data(queue, rq);
2283 if (unlikely(ret)) {
2284 nvme_cleanup_cmd(rq);
2285 dev_err(queue->ctrl->ctrl.device,
2286 "Failed to map data (%d)\n", ret);
2293 static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx)
2295 struct nvme_tcp_queue *queue = hctx->driver_data;
2297 if (!llist_empty(&queue->req_list))
2298 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
2301 static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx,
2302 const struct blk_mq_queue_data *bd)
2304 struct nvme_ns *ns = hctx->queue->queuedata;
2305 struct nvme_tcp_queue *queue = hctx->driver_data;
2306 struct request *rq = bd->rq;
2307 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2308 bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags);
2311 if (!nvmf_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2312 return nvmf_fail_nonready_command(&queue->ctrl->ctrl, rq);
2314 ret = nvme_tcp_setup_cmd_pdu(ns, rq);
2318 blk_mq_start_request(rq);
2320 nvme_tcp_queue_request(req, true, bd->last);
2325 static int nvme_tcp_map_queues(struct blk_mq_tag_set *set)
2327 struct nvme_tcp_ctrl *ctrl = set->driver_data;
2328 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2330 if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) {
2331 /* separate read/write queues */
2332 set->map[HCTX_TYPE_DEFAULT].nr_queues =
2333 ctrl->io_queues[HCTX_TYPE_DEFAULT];
2334 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
2335 set->map[HCTX_TYPE_READ].nr_queues =
2336 ctrl->io_queues[HCTX_TYPE_READ];
2337 set->map[HCTX_TYPE_READ].queue_offset =
2338 ctrl->io_queues[HCTX_TYPE_DEFAULT];
2340 /* shared read/write queues */
2341 set->map[HCTX_TYPE_DEFAULT].nr_queues =
2342 ctrl->io_queues[HCTX_TYPE_DEFAULT];
2343 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
2344 set->map[HCTX_TYPE_READ].nr_queues =
2345 ctrl->io_queues[HCTX_TYPE_DEFAULT];
2346 set->map[HCTX_TYPE_READ].queue_offset = 0;
2348 blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
2349 blk_mq_map_queues(&set->map[HCTX_TYPE_READ]);
2351 if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) {
2352 /* map dedicated poll queues only if we have queues left */
2353 set->map[HCTX_TYPE_POLL].nr_queues =
2354 ctrl->io_queues[HCTX_TYPE_POLL];
2355 set->map[HCTX_TYPE_POLL].queue_offset =
2356 ctrl->io_queues[HCTX_TYPE_DEFAULT] +
2357 ctrl->io_queues[HCTX_TYPE_READ];
2358 blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]);
2361 dev_info(ctrl->ctrl.device,
2362 "mapped %d/%d/%d default/read/poll queues.\n",
2363 ctrl->io_queues[HCTX_TYPE_DEFAULT],
2364 ctrl->io_queues[HCTX_TYPE_READ],
2365 ctrl->io_queues[HCTX_TYPE_POLL]);
2370 static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx)
2372 struct nvme_tcp_queue *queue = hctx->driver_data;
2373 struct sock *sk = queue->sock->sk;
2375 if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))
2378 set_bit(NVME_TCP_Q_POLLING, &queue->flags);
2379 if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue))
2380 sk_busy_loop(sk, true);
2381 nvme_tcp_try_recv(queue);
2382 clear_bit(NVME_TCP_Q_POLLING, &queue->flags);
2383 return queue->nr_cqe;
2386 static const struct blk_mq_ops nvme_tcp_mq_ops = {
2387 .queue_rq = nvme_tcp_queue_rq,
2388 .commit_rqs = nvme_tcp_commit_rqs,
2389 .complete = nvme_complete_rq,
2390 .init_request = nvme_tcp_init_request,
2391 .exit_request = nvme_tcp_exit_request,
2392 .init_hctx = nvme_tcp_init_hctx,
2393 .timeout = nvme_tcp_timeout,
2394 .map_queues = nvme_tcp_map_queues,
2395 .poll = nvme_tcp_poll,
2398 static const struct blk_mq_ops nvme_tcp_admin_mq_ops = {
2399 .queue_rq = nvme_tcp_queue_rq,
2400 .complete = nvme_complete_rq,
2401 .init_request = nvme_tcp_init_request,
2402 .exit_request = nvme_tcp_exit_request,
2403 .init_hctx = nvme_tcp_init_admin_hctx,
2404 .timeout = nvme_tcp_timeout,
2407 static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = {
2409 .module = THIS_MODULE,
2410 .flags = NVME_F_FABRICS,
2411 .reg_read32 = nvmf_reg_read32,
2412 .reg_read64 = nvmf_reg_read64,
2413 .reg_write32 = nvmf_reg_write32,
2414 .free_ctrl = nvme_tcp_free_ctrl,
2415 .submit_async_event = nvme_tcp_submit_async_event,
2416 .delete_ctrl = nvme_tcp_delete_ctrl,
2417 .get_address = nvmf_get_address,
2421 nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts)
2423 struct nvme_tcp_ctrl *ctrl;
2426 mutex_lock(&nvme_tcp_ctrl_mutex);
2427 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) {
2428 found = nvmf_ip_options_match(&ctrl->ctrl, opts);
2432 mutex_unlock(&nvme_tcp_ctrl_mutex);
2437 static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev,
2438 struct nvmf_ctrl_options *opts)
2440 struct nvme_tcp_ctrl *ctrl;
2443 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
2445 return ERR_PTR(-ENOMEM);
2447 INIT_LIST_HEAD(&ctrl->list);
2448 ctrl->ctrl.opts = opts;
2449 ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues +
2450 opts->nr_poll_queues + 1;
2451 ctrl->ctrl.sqsize = opts->queue_size - 1;
2452 ctrl->ctrl.kato = opts->kato;
2454 INIT_DELAYED_WORK(&ctrl->connect_work,
2455 nvme_tcp_reconnect_ctrl_work);
2456 INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work);
2457 INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work);
2458 mutex_init(&ctrl->teardown_lock);
2460 if (!(opts->mask & NVMF_OPT_TRSVCID)) {
2462 kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL);
2463 if (!opts->trsvcid) {
2467 opts->mask |= NVMF_OPT_TRSVCID;
2470 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2471 opts->traddr, opts->trsvcid, &ctrl->addr);
2473 pr_err("malformed address passed: %s:%s\n",
2474 opts->traddr, opts->trsvcid);
2478 if (opts->mask & NVMF_OPT_HOST_TRADDR) {
2479 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2480 opts->host_traddr, NULL, &ctrl->src_addr);
2482 pr_err("malformed src address passed: %s\n",
2488 if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) {
2493 ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues),
2495 if (!ctrl->queues) {
2500 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0);
2502 goto out_kfree_queues;
2504 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
2507 goto out_uninit_ctrl;
2510 ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true);
2512 goto out_uninit_ctrl;
2514 dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n",
2515 ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
2517 mutex_lock(&nvme_tcp_ctrl_mutex);
2518 list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list);
2519 mutex_unlock(&nvme_tcp_ctrl_mutex);
2524 nvme_uninit_ctrl(&ctrl->ctrl);
2525 nvme_put_ctrl(&ctrl->ctrl);
2528 return ERR_PTR(ret);
2530 kfree(ctrl->queues);
2533 return ERR_PTR(ret);
2536 static struct nvmf_transport_ops nvme_tcp_transport = {
2538 .module = THIS_MODULE,
2539 .required_opts = NVMF_OPT_TRADDR,
2540 .allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
2541 NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO |
2542 NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST |
2543 NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES |
2545 .create_ctrl = nvme_tcp_create_ctrl,
2548 static int __init nvme_tcp_init_module(void)
2550 nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq",
2551 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
2555 nvmf_register_transport(&nvme_tcp_transport);
2559 static void __exit nvme_tcp_cleanup_module(void)
2561 struct nvme_tcp_ctrl *ctrl;
2563 nvmf_unregister_transport(&nvme_tcp_transport);
2565 mutex_lock(&nvme_tcp_ctrl_mutex);
2566 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list)
2567 nvme_delete_ctrl(&ctrl->ctrl);
2568 mutex_unlock(&nvme_tcp_ctrl_mutex);
2569 flush_workqueue(nvme_delete_wq);
2571 destroy_workqueue(nvme_tcp_wq);
2574 module_init(nvme_tcp_init_module);
2575 module_exit(nvme_tcp_cleanup_module);
2577 MODULE_LICENSE("GPL v2");