treewide: Replace GPLv2 boilerplate/reference with SPDX - rule 157
[sfrench/cifs-2.6.git] / drivers / target / iscsi / iscsi_target_util.c
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*******************************************************************************
3  * This file contains the iSCSI Target specific utility functions.
4  *
5  * (c) Copyright 2007-2013 Datera, Inc.
6  *
7  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
8  *
9  ******************************************************************************/
10
11 #include <linux/list.h>
12 #include <linux/sched/signal.h>
13 #include <net/ipv6.h>         /* ipv6_addr_equal() */
14 #include <scsi/scsi_tcq.h>
15 #include <scsi/iscsi_proto.h>
16 #include <target/target_core_base.h>
17 #include <target/target_core_fabric.h>
18 #include <target/iscsi/iscsi_transport.h>
19
20 #include <target/iscsi/iscsi_target_core.h>
21 #include "iscsi_target_parameters.h"
22 #include "iscsi_target_seq_pdu_list.h"
23 #include "iscsi_target_datain_values.h"
24 #include "iscsi_target_erl0.h"
25 #include "iscsi_target_erl1.h"
26 #include "iscsi_target_erl2.h"
27 #include "iscsi_target_tpg.h"
28 #include "iscsi_target_util.h"
29 #include "iscsi_target.h"
30
31 #define PRINT_BUFF(buff, len)                                   \
32 {                                                               \
33         int zzz;                                                \
34                                                                 \
35         pr_debug("%d:\n", __LINE__);                            \
36         for (zzz = 0; zzz < len; zzz++) {                       \
37                 if (zzz % 16 == 0) {                            \
38                         if (zzz)                                \
39                                 pr_debug("\n");                 \
40                         pr_debug("%4i: ", zzz);                 \
41                 }                                               \
42                 pr_debug("%02x ", (unsigned char) (buff)[zzz]); \
43         }                                                       \
44         if ((len + 1) % 16)                                     \
45                 pr_debug("\n");                                 \
46 }
47
48 extern struct list_head g_tiqn_list;
49 extern spinlock_t tiqn_lock;
50
51 int iscsit_add_r2t_to_list(
52         struct iscsi_cmd *cmd,
53         u32 offset,
54         u32 xfer_len,
55         int recovery,
56         u32 r2t_sn)
57 {
58         struct iscsi_r2t *r2t;
59
60         lockdep_assert_held(&cmd->r2t_lock);
61
62         WARN_ON_ONCE((s32)xfer_len < 0);
63
64         r2t = kmem_cache_zalloc(lio_r2t_cache, GFP_ATOMIC);
65         if (!r2t) {
66                 pr_err("Unable to allocate memory for struct iscsi_r2t.\n");
67                 return -1;
68         }
69         INIT_LIST_HEAD(&r2t->r2t_list);
70
71         r2t->recovery_r2t = recovery;
72         r2t->r2t_sn = (!r2t_sn) ? cmd->r2t_sn++ : r2t_sn;
73         r2t->offset = offset;
74         r2t->xfer_len = xfer_len;
75         list_add_tail(&r2t->r2t_list, &cmd->cmd_r2t_list);
76         spin_unlock_bh(&cmd->r2t_lock);
77
78         iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, ISTATE_SEND_R2T);
79
80         spin_lock_bh(&cmd->r2t_lock);
81         return 0;
82 }
83
84 struct iscsi_r2t *iscsit_get_r2t_for_eos(
85         struct iscsi_cmd *cmd,
86         u32 offset,
87         u32 length)
88 {
89         struct iscsi_r2t *r2t;
90
91         spin_lock_bh(&cmd->r2t_lock);
92         list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {
93                 if ((r2t->offset <= offset) &&
94                     (r2t->offset + r2t->xfer_len) >= (offset + length)) {
95                         spin_unlock_bh(&cmd->r2t_lock);
96                         return r2t;
97                 }
98         }
99         spin_unlock_bh(&cmd->r2t_lock);
100
101         pr_err("Unable to locate R2T for Offset: %u, Length:"
102                         " %u\n", offset, length);
103         return NULL;
104 }
105
106 struct iscsi_r2t *iscsit_get_r2t_from_list(struct iscsi_cmd *cmd)
107 {
108         struct iscsi_r2t *r2t;
109
110         spin_lock_bh(&cmd->r2t_lock);
111         list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {
112                 if (!r2t->sent_r2t) {
113                         spin_unlock_bh(&cmd->r2t_lock);
114                         return r2t;
115                 }
116         }
117         spin_unlock_bh(&cmd->r2t_lock);
118
119         pr_err("Unable to locate next R2T to send for ITT:"
120                         " 0x%08x.\n", cmd->init_task_tag);
121         return NULL;
122 }
123
124 void iscsit_free_r2t(struct iscsi_r2t *r2t, struct iscsi_cmd *cmd)
125 {
126         lockdep_assert_held(&cmd->r2t_lock);
127
128         list_del(&r2t->r2t_list);
129         kmem_cache_free(lio_r2t_cache, r2t);
130 }
131
132 void iscsit_free_r2ts_from_list(struct iscsi_cmd *cmd)
133 {
134         struct iscsi_r2t *r2t, *r2t_tmp;
135
136         spin_lock_bh(&cmd->r2t_lock);
137         list_for_each_entry_safe(r2t, r2t_tmp, &cmd->cmd_r2t_list, r2t_list)
138                 iscsit_free_r2t(r2t, cmd);
139         spin_unlock_bh(&cmd->r2t_lock);
140 }
141
142 static int iscsit_wait_for_tag(struct se_session *se_sess, int state, int *cpup)
143 {
144         int tag = -1;
145         DEFINE_SBQ_WAIT(wait);
146         struct sbq_wait_state *ws;
147         struct sbitmap_queue *sbq;
148
149         if (state == TASK_RUNNING)
150                 return tag;
151
152         sbq = &se_sess->sess_tag_pool;
153         ws = &sbq->ws[0];
154         for (;;) {
155                 sbitmap_prepare_to_wait(sbq, ws, &wait, state);
156                 if (signal_pending_state(state, current))
157                         break;
158                 tag = sbitmap_queue_get(sbq, cpup);
159                 if (tag >= 0)
160                         break;
161                 schedule();
162         }
163
164         sbitmap_finish_wait(sbq, ws, &wait);
165         return tag;
166 }
167
168 /*
169  * May be called from software interrupt (timer) context for allocating
170  * iSCSI NopINs.
171  */
172 struct iscsi_cmd *iscsit_allocate_cmd(struct iscsi_conn *conn, int state)
173 {
174         struct iscsi_cmd *cmd;
175         struct se_session *se_sess = conn->sess->se_sess;
176         int size, tag, cpu;
177
178         tag = sbitmap_queue_get(&se_sess->sess_tag_pool, &cpu);
179         if (tag < 0)
180                 tag = iscsit_wait_for_tag(se_sess, state, &cpu);
181         if (tag < 0)
182                 return NULL;
183
184         size = sizeof(struct iscsi_cmd) + conn->conn_transport->priv_size;
185         cmd = (struct iscsi_cmd *)(se_sess->sess_cmd_map + (tag * size));
186         memset(cmd, 0, size);
187
188         cmd->se_cmd.map_tag = tag;
189         cmd->se_cmd.map_cpu = cpu;
190         cmd->conn = conn;
191         cmd->data_direction = DMA_NONE;
192         INIT_LIST_HEAD(&cmd->i_conn_node);
193         INIT_LIST_HEAD(&cmd->datain_list);
194         INIT_LIST_HEAD(&cmd->cmd_r2t_list);
195         spin_lock_init(&cmd->datain_lock);
196         spin_lock_init(&cmd->dataout_timeout_lock);
197         spin_lock_init(&cmd->istate_lock);
198         spin_lock_init(&cmd->error_lock);
199         spin_lock_init(&cmd->r2t_lock);
200         timer_setup(&cmd->dataout_timer, iscsit_handle_dataout_timeout, 0);
201
202         return cmd;
203 }
204 EXPORT_SYMBOL(iscsit_allocate_cmd);
205
206 struct iscsi_seq *iscsit_get_seq_holder_for_datain(
207         struct iscsi_cmd *cmd,
208         u32 seq_send_order)
209 {
210         u32 i;
211
212         for (i = 0; i < cmd->seq_count; i++)
213                 if (cmd->seq_list[i].seq_send_order == seq_send_order)
214                         return &cmd->seq_list[i];
215
216         return NULL;
217 }
218
219 struct iscsi_seq *iscsit_get_seq_holder_for_r2t(struct iscsi_cmd *cmd)
220 {
221         u32 i;
222
223         if (!cmd->seq_list) {
224                 pr_err("struct iscsi_cmd->seq_list is NULL!\n");
225                 return NULL;
226         }
227
228         for (i = 0; i < cmd->seq_count; i++) {
229                 if (cmd->seq_list[i].type != SEQTYPE_NORMAL)
230                         continue;
231                 if (cmd->seq_list[i].seq_send_order == cmd->seq_send_order) {
232                         cmd->seq_send_order++;
233                         return &cmd->seq_list[i];
234                 }
235         }
236
237         return NULL;
238 }
239
240 struct iscsi_r2t *iscsit_get_holder_for_r2tsn(
241         struct iscsi_cmd *cmd,
242         u32 r2t_sn)
243 {
244         struct iscsi_r2t *r2t;
245
246         spin_lock_bh(&cmd->r2t_lock);
247         list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {
248                 if (r2t->r2t_sn == r2t_sn) {
249                         spin_unlock_bh(&cmd->r2t_lock);
250                         return r2t;
251                 }
252         }
253         spin_unlock_bh(&cmd->r2t_lock);
254
255         return NULL;
256 }
257
258 static inline int iscsit_check_received_cmdsn(struct iscsi_session *sess, u32 cmdsn)
259 {
260         u32 max_cmdsn;
261         int ret;
262
263         /*
264          * This is the proper method of checking received CmdSN against
265          * ExpCmdSN and MaxCmdSN values, as well as accounting for out
266          * or order CmdSNs due to multiple connection sessions and/or
267          * CRC failures.
268          */
269         max_cmdsn = atomic_read(&sess->max_cmd_sn);
270         if (iscsi_sna_gt(cmdsn, max_cmdsn)) {
271                 pr_err("Received CmdSN: 0x%08x is greater than"
272                        " MaxCmdSN: 0x%08x, ignoring.\n", cmdsn, max_cmdsn);
273                 ret = CMDSN_MAXCMDSN_OVERRUN;
274
275         } else if (cmdsn == sess->exp_cmd_sn) {
276                 sess->exp_cmd_sn++;
277                 pr_debug("Received CmdSN matches ExpCmdSN,"
278                       " incremented ExpCmdSN to: 0x%08x\n",
279                       sess->exp_cmd_sn);
280                 ret = CMDSN_NORMAL_OPERATION;
281
282         } else if (iscsi_sna_gt(cmdsn, sess->exp_cmd_sn)) {
283                 pr_debug("Received CmdSN: 0x%08x is greater"
284                       " than ExpCmdSN: 0x%08x, not acknowledging.\n",
285                       cmdsn, sess->exp_cmd_sn);
286                 ret = CMDSN_HIGHER_THAN_EXP;
287
288         } else {
289                 pr_err("Received CmdSN: 0x%08x is less than"
290                        " ExpCmdSN: 0x%08x, ignoring.\n", cmdsn,
291                        sess->exp_cmd_sn);
292                 ret = CMDSN_LOWER_THAN_EXP;
293         }
294
295         return ret;
296 }
297
298 /*
299  * Commands may be received out of order if MC/S is in use.
300  * Ensure they are executed in CmdSN order.
301  */
302 int iscsit_sequence_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
303                         unsigned char *buf, __be32 cmdsn)
304 {
305         int ret, cmdsn_ret;
306         bool reject = false;
307         u8 reason = ISCSI_REASON_BOOKMARK_NO_RESOURCES;
308
309         mutex_lock(&conn->sess->cmdsn_mutex);
310
311         cmdsn_ret = iscsit_check_received_cmdsn(conn->sess, be32_to_cpu(cmdsn));
312         switch (cmdsn_ret) {
313         case CMDSN_NORMAL_OPERATION:
314                 ret = iscsit_execute_cmd(cmd, 0);
315                 if ((ret >= 0) && !list_empty(&conn->sess->sess_ooo_cmdsn_list))
316                         iscsit_execute_ooo_cmdsns(conn->sess);
317                 else if (ret < 0) {
318                         reject = true;
319                         ret = CMDSN_ERROR_CANNOT_RECOVER;
320                 }
321                 break;
322         case CMDSN_HIGHER_THAN_EXP:
323                 ret = iscsit_handle_ooo_cmdsn(conn->sess, cmd, be32_to_cpu(cmdsn));
324                 if (ret < 0) {
325                         reject = true;
326                         ret = CMDSN_ERROR_CANNOT_RECOVER;
327                         break;
328                 }
329                 ret = CMDSN_HIGHER_THAN_EXP;
330                 break;
331         case CMDSN_LOWER_THAN_EXP:
332         case CMDSN_MAXCMDSN_OVERRUN:
333         default:
334                 cmd->i_state = ISTATE_REMOVE;
335                 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
336                 /*
337                  * Existing callers for iscsit_sequence_cmd() will silently
338                  * ignore commands with CMDSN_LOWER_THAN_EXP, so force this
339                  * return for CMDSN_MAXCMDSN_OVERRUN as well..
340                  */
341                 ret = CMDSN_LOWER_THAN_EXP;
342                 break;
343         }
344         mutex_unlock(&conn->sess->cmdsn_mutex);
345
346         if (reject)
347                 iscsit_reject_cmd(cmd, reason, buf);
348
349         return ret;
350 }
351 EXPORT_SYMBOL(iscsit_sequence_cmd);
352
353 int iscsit_check_unsolicited_dataout(struct iscsi_cmd *cmd, unsigned char *buf)
354 {
355         struct iscsi_conn *conn = cmd->conn;
356         struct se_cmd *se_cmd = &cmd->se_cmd;
357         struct iscsi_data *hdr = (struct iscsi_data *) buf;
358         u32 payload_length = ntoh24(hdr->dlength);
359
360         if (conn->sess->sess_ops->InitialR2T) {
361                 pr_err("Received unexpected unsolicited data"
362                         " while InitialR2T=Yes, protocol error.\n");
363                 transport_send_check_condition_and_sense(se_cmd,
364                                 TCM_UNEXPECTED_UNSOLICITED_DATA, 0);
365                 return -1;
366         }
367
368         if ((cmd->first_burst_len + payload_length) >
369              conn->sess->sess_ops->FirstBurstLength) {
370                 pr_err("Total %u bytes exceeds FirstBurstLength: %u"
371                         " for this Unsolicited DataOut Burst.\n",
372                         (cmd->first_burst_len + payload_length),
373                                 conn->sess->sess_ops->FirstBurstLength);
374                 transport_send_check_condition_and_sense(se_cmd,
375                                 TCM_INCORRECT_AMOUNT_OF_DATA, 0);
376                 return -1;
377         }
378
379         if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))
380                 return 0;
381
382         if (((cmd->first_burst_len + payload_length) != cmd->se_cmd.data_length) &&
383             ((cmd->first_burst_len + payload_length) !=
384               conn->sess->sess_ops->FirstBurstLength)) {
385                 pr_err("Unsolicited non-immediate data received %u"
386                         " does not equal FirstBurstLength: %u, and does"
387                         " not equal ExpXferLen %u.\n",
388                         (cmd->first_burst_len + payload_length),
389                         conn->sess->sess_ops->FirstBurstLength, cmd->se_cmd.data_length);
390                 transport_send_check_condition_and_sense(se_cmd,
391                                 TCM_INCORRECT_AMOUNT_OF_DATA, 0);
392                 return -1;
393         }
394         return 0;
395 }
396
397 struct iscsi_cmd *iscsit_find_cmd_from_itt(
398         struct iscsi_conn *conn,
399         itt_t init_task_tag)
400 {
401         struct iscsi_cmd *cmd;
402
403         spin_lock_bh(&conn->cmd_lock);
404         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
405                 if (cmd->init_task_tag == init_task_tag) {
406                         spin_unlock_bh(&conn->cmd_lock);
407                         return cmd;
408                 }
409         }
410         spin_unlock_bh(&conn->cmd_lock);
411
412         pr_err("Unable to locate ITT: 0x%08x on CID: %hu",
413                         init_task_tag, conn->cid);
414         return NULL;
415 }
416 EXPORT_SYMBOL(iscsit_find_cmd_from_itt);
417
418 struct iscsi_cmd *iscsit_find_cmd_from_itt_or_dump(
419         struct iscsi_conn *conn,
420         itt_t init_task_tag,
421         u32 length)
422 {
423         struct iscsi_cmd *cmd;
424
425         spin_lock_bh(&conn->cmd_lock);
426         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
427                 if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT)
428                         continue;
429                 if (cmd->init_task_tag == init_task_tag) {
430                         spin_unlock_bh(&conn->cmd_lock);
431                         return cmd;
432                 }
433         }
434         spin_unlock_bh(&conn->cmd_lock);
435
436         pr_err("Unable to locate ITT: 0x%08x on CID: %hu,"
437                         " dumping payload\n", init_task_tag, conn->cid);
438         if (length)
439                 iscsit_dump_data_payload(conn, length, 1);
440
441         return NULL;
442 }
443 EXPORT_SYMBOL(iscsit_find_cmd_from_itt_or_dump);
444
445 struct iscsi_cmd *iscsit_find_cmd_from_ttt(
446         struct iscsi_conn *conn,
447         u32 targ_xfer_tag)
448 {
449         struct iscsi_cmd *cmd = NULL;
450
451         spin_lock_bh(&conn->cmd_lock);
452         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
453                 if (cmd->targ_xfer_tag == targ_xfer_tag) {
454                         spin_unlock_bh(&conn->cmd_lock);
455                         return cmd;
456                 }
457         }
458         spin_unlock_bh(&conn->cmd_lock);
459
460         pr_err("Unable to locate TTT: 0x%08x on CID: %hu\n",
461                         targ_xfer_tag, conn->cid);
462         return NULL;
463 }
464
465 int iscsit_find_cmd_for_recovery(
466         struct iscsi_session *sess,
467         struct iscsi_cmd **cmd_ptr,
468         struct iscsi_conn_recovery **cr_ptr,
469         itt_t init_task_tag)
470 {
471         struct iscsi_cmd *cmd = NULL;
472         struct iscsi_conn_recovery *cr;
473         /*
474          * Scan through the inactive connection recovery list's command list.
475          * If init_task_tag matches the command is still alligent.
476          */
477         spin_lock(&sess->cr_i_lock);
478         list_for_each_entry(cr, &sess->cr_inactive_list, cr_list) {
479                 spin_lock(&cr->conn_recovery_cmd_lock);
480                 list_for_each_entry(cmd, &cr->conn_recovery_cmd_list, i_conn_node) {
481                         if (cmd->init_task_tag == init_task_tag) {
482                                 spin_unlock(&cr->conn_recovery_cmd_lock);
483                                 spin_unlock(&sess->cr_i_lock);
484
485                                 *cr_ptr = cr;
486                                 *cmd_ptr = cmd;
487                                 return -2;
488                         }
489                 }
490                 spin_unlock(&cr->conn_recovery_cmd_lock);
491         }
492         spin_unlock(&sess->cr_i_lock);
493         /*
494          * Scan through the active connection recovery list's command list.
495          * If init_task_tag matches the command is ready to be reassigned.
496          */
497         spin_lock(&sess->cr_a_lock);
498         list_for_each_entry(cr, &sess->cr_active_list, cr_list) {
499                 spin_lock(&cr->conn_recovery_cmd_lock);
500                 list_for_each_entry(cmd, &cr->conn_recovery_cmd_list, i_conn_node) {
501                         if (cmd->init_task_tag == init_task_tag) {
502                                 spin_unlock(&cr->conn_recovery_cmd_lock);
503                                 spin_unlock(&sess->cr_a_lock);
504
505                                 *cr_ptr = cr;
506                                 *cmd_ptr = cmd;
507                                 return 0;
508                         }
509                 }
510                 spin_unlock(&cr->conn_recovery_cmd_lock);
511         }
512         spin_unlock(&sess->cr_a_lock);
513
514         return -1;
515 }
516
517 void iscsit_add_cmd_to_immediate_queue(
518         struct iscsi_cmd *cmd,
519         struct iscsi_conn *conn,
520         u8 state)
521 {
522         struct iscsi_queue_req *qr;
523
524         qr = kmem_cache_zalloc(lio_qr_cache, GFP_ATOMIC);
525         if (!qr) {
526                 pr_err("Unable to allocate memory for"
527                                 " struct iscsi_queue_req\n");
528                 return;
529         }
530         INIT_LIST_HEAD(&qr->qr_list);
531         qr->cmd = cmd;
532         qr->state = state;
533
534         spin_lock_bh(&conn->immed_queue_lock);
535         list_add_tail(&qr->qr_list, &conn->immed_queue_list);
536         atomic_inc(&cmd->immed_queue_count);
537         atomic_set(&conn->check_immediate_queue, 1);
538         spin_unlock_bh(&conn->immed_queue_lock);
539
540         wake_up(&conn->queues_wq);
541 }
542 EXPORT_SYMBOL(iscsit_add_cmd_to_immediate_queue);
543
544 struct iscsi_queue_req *iscsit_get_cmd_from_immediate_queue(struct iscsi_conn *conn)
545 {
546         struct iscsi_queue_req *qr;
547
548         spin_lock_bh(&conn->immed_queue_lock);
549         if (list_empty(&conn->immed_queue_list)) {
550                 spin_unlock_bh(&conn->immed_queue_lock);
551                 return NULL;
552         }
553         qr = list_first_entry(&conn->immed_queue_list,
554                               struct iscsi_queue_req, qr_list);
555
556         list_del(&qr->qr_list);
557         if (qr->cmd)
558                 atomic_dec(&qr->cmd->immed_queue_count);
559         spin_unlock_bh(&conn->immed_queue_lock);
560
561         return qr;
562 }
563
564 static void iscsit_remove_cmd_from_immediate_queue(
565         struct iscsi_cmd *cmd,
566         struct iscsi_conn *conn)
567 {
568         struct iscsi_queue_req *qr, *qr_tmp;
569
570         spin_lock_bh(&conn->immed_queue_lock);
571         if (!atomic_read(&cmd->immed_queue_count)) {
572                 spin_unlock_bh(&conn->immed_queue_lock);
573                 return;
574         }
575
576         list_for_each_entry_safe(qr, qr_tmp, &conn->immed_queue_list, qr_list) {
577                 if (qr->cmd != cmd)
578                         continue;
579
580                 atomic_dec(&qr->cmd->immed_queue_count);
581                 list_del(&qr->qr_list);
582                 kmem_cache_free(lio_qr_cache, qr);
583         }
584         spin_unlock_bh(&conn->immed_queue_lock);
585
586         if (atomic_read(&cmd->immed_queue_count)) {
587                 pr_err("ITT: 0x%08x immed_queue_count: %d\n",
588                         cmd->init_task_tag,
589                         atomic_read(&cmd->immed_queue_count));
590         }
591 }
592
593 int iscsit_add_cmd_to_response_queue(
594         struct iscsi_cmd *cmd,
595         struct iscsi_conn *conn,
596         u8 state)
597 {
598         struct iscsi_queue_req *qr;
599
600         qr = kmem_cache_zalloc(lio_qr_cache, GFP_ATOMIC);
601         if (!qr) {
602                 pr_err("Unable to allocate memory for"
603                         " struct iscsi_queue_req\n");
604                 return -ENOMEM;
605         }
606         INIT_LIST_HEAD(&qr->qr_list);
607         qr->cmd = cmd;
608         qr->state = state;
609
610         spin_lock_bh(&conn->response_queue_lock);
611         list_add_tail(&qr->qr_list, &conn->response_queue_list);
612         atomic_inc(&cmd->response_queue_count);
613         spin_unlock_bh(&conn->response_queue_lock);
614
615         wake_up(&conn->queues_wq);
616         return 0;
617 }
618
619 struct iscsi_queue_req *iscsit_get_cmd_from_response_queue(struct iscsi_conn *conn)
620 {
621         struct iscsi_queue_req *qr;
622
623         spin_lock_bh(&conn->response_queue_lock);
624         if (list_empty(&conn->response_queue_list)) {
625                 spin_unlock_bh(&conn->response_queue_lock);
626                 return NULL;
627         }
628
629         qr = list_first_entry(&conn->response_queue_list,
630                               struct iscsi_queue_req, qr_list);
631
632         list_del(&qr->qr_list);
633         if (qr->cmd)
634                 atomic_dec(&qr->cmd->response_queue_count);
635         spin_unlock_bh(&conn->response_queue_lock);
636
637         return qr;
638 }
639
640 static void iscsit_remove_cmd_from_response_queue(
641         struct iscsi_cmd *cmd,
642         struct iscsi_conn *conn)
643 {
644         struct iscsi_queue_req *qr, *qr_tmp;
645
646         spin_lock_bh(&conn->response_queue_lock);
647         if (!atomic_read(&cmd->response_queue_count)) {
648                 spin_unlock_bh(&conn->response_queue_lock);
649                 return;
650         }
651
652         list_for_each_entry_safe(qr, qr_tmp, &conn->response_queue_list,
653                                 qr_list) {
654                 if (qr->cmd != cmd)
655                         continue;
656
657                 atomic_dec(&qr->cmd->response_queue_count);
658                 list_del(&qr->qr_list);
659                 kmem_cache_free(lio_qr_cache, qr);
660         }
661         spin_unlock_bh(&conn->response_queue_lock);
662
663         if (atomic_read(&cmd->response_queue_count)) {
664                 pr_err("ITT: 0x%08x response_queue_count: %d\n",
665                         cmd->init_task_tag,
666                         atomic_read(&cmd->response_queue_count));
667         }
668 }
669
670 bool iscsit_conn_all_queues_empty(struct iscsi_conn *conn)
671 {
672         bool empty;
673
674         spin_lock_bh(&conn->immed_queue_lock);
675         empty = list_empty(&conn->immed_queue_list);
676         spin_unlock_bh(&conn->immed_queue_lock);
677
678         if (!empty)
679                 return empty;
680
681         spin_lock_bh(&conn->response_queue_lock);
682         empty = list_empty(&conn->response_queue_list);
683         spin_unlock_bh(&conn->response_queue_lock);
684
685         return empty;
686 }
687
688 void iscsit_free_queue_reqs_for_conn(struct iscsi_conn *conn)
689 {
690         struct iscsi_queue_req *qr, *qr_tmp;
691
692         spin_lock_bh(&conn->immed_queue_lock);
693         list_for_each_entry_safe(qr, qr_tmp, &conn->immed_queue_list, qr_list) {
694                 list_del(&qr->qr_list);
695                 if (qr->cmd)
696                         atomic_dec(&qr->cmd->immed_queue_count);
697
698                 kmem_cache_free(lio_qr_cache, qr);
699         }
700         spin_unlock_bh(&conn->immed_queue_lock);
701
702         spin_lock_bh(&conn->response_queue_lock);
703         list_for_each_entry_safe(qr, qr_tmp, &conn->response_queue_list,
704                         qr_list) {
705                 list_del(&qr->qr_list);
706                 if (qr->cmd)
707                         atomic_dec(&qr->cmd->response_queue_count);
708
709                 kmem_cache_free(lio_qr_cache, qr);
710         }
711         spin_unlock_bh(&conn->response_queue_lock);
712 }
713
714 void iscsit_release_cmd(struct iscsi_cmd *cmd)
715 {
716         struct iscsi_session *sess;
717         struct se_cmd *se_cmd = &cmd->se_cmd;
718
719         WARN_ON(!list_empty(&cmd->i_conn_node));
720
721         if (cmd->conn)
722                 sess = cmd->conn->sess;
723         else
724                 sess = cmd->sess;
725
726         BUG_ON(!sess || !sess->se_sess);
727
728         kfree(cmd->buf_ptr);
729         kfree(cmd->pdu_list);
730         kfree(cmd->seq_list);
731         kfree(cmd->tmr_req);
732         kfree(cmd->overflow_buf);
733         kfree(cmd->iov_data);
734         kfree(cmd->text_in_ptr);
735
736         target_free_tag(sess->se_sess, se_cmd);
737 }
738 EXPORT_SYMBOL(iscsit_release_cmd);
739
740 void __iscsit_free_cmd(struct iscsi_cmd *cmd, bool check_queues)
741 {
742         struct iscsi_conn *conn = cmd->conn;
743
744         WARN_ON(!list_empty(&cmd->i_conn_node));
745
746         if (cmd->data_direction == DMA_TO_DEVICE) {
747                 iscsit_stop_dataout_timer(cmd);
748                 iscsit_free_r2ts_from_list(cmd);
749         }
750         if (cmd->data_direction == DMA_FROM_DEVICE)
751                 iscsit_free_all_datain_reqs(cmd);
752
753         if (conn && check_queues) {
754                 iscsit_remove_cmd_from_immediate_queue(cmd, conn);
755                 iscsit_remove_cmd_from_response_queue(cmd, conn);
756         }
757
758         if (conn && conn->conn_transport->iscsit_unmap_cmd)
759                 conn->conn_transport->iscsit_unmap_cmd(conn, cmd);
760 }
761
762 void iscsit_free_cmd(struct iscsi_cmd *cmd, bool shutdown)
763 {
764         struct se_cmd *se_cmd = cmd->se_cmd.se_tfo ? &cmd->se_cmd : NULL;
765         int rc;
766
767         WARN_ON(!list_empty(&cmd->i_conn_node));
768
769         __iscsit_free_cmd(cmd, shutdown);
770         if (se_cmd) {
771                 rc = transport_generic_free_cmd(se_cmd, shutdown);
772                 if (!rc && shutdown && se_cmd->se_sess) {
773                         __iscsit_free_cmd(cmd, shutdown);
774                         target_put_sess_cmd(se_cmd);
775                 }
776         } else {
777                 iscsit_release_cmd(cmd);
778         }
779 }
780 EXPORT_SYMBOL(iscsit_free_cmd);
781
782 int iscsit_check_session_usage_count(struct iscsi_session *sess)
783 {
784         spin_lock_bh(&sess->session_usage_lock);
785         if (sess->session_usage_count != 0) {
786                 sess->session_waiting_on_uc = 1;
787                 spin_unlock_bh(&sess->session_usage_lock);
788                 if (in_interrupt())
789                         return 2;
790
791                 wait_for_completion(&sess->session_waiting_on_uc_comp);
792                 return 1;
793         }
794         spin_unlock_bh(&sess->session_usage_lock);
795
796         return 0;
797 }
798
799 void iscsit_dec_session_usage_count(struct iscsi_session *sess)
800 {
801         spin_lock_bh(&sess->session_usage_lock);
802         sess->session_usage_count--;
803
804         if (!sess->session_usage_count && sess->session_waiting_on_uc)
805                 complete(&sess->session_waiting_on_uc_comp);
806
807         spin_unlock_bh(&sess->session_usage_lock);
808 }
809
810 void iscsit_inc_session_usage_count(struct iscsi_session *sess)
811 {
812         spin_lock_bh(&sess->session_usage_lock);
813         sess->session_usage_count++;
814         spin_unlock_bh(&sess->session_usage_lock);
815 }
816
817 struct iscsi_conn *iscsit_get_conn_from_cid(struct iscsi_session *sess, u16 cid)
818 {
819         struct iscsi_conn *conn;
820
821         spin_lock_bh(&sess->conn_lock);
822         list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
823                 if ((conn->cid == cid) &&
824                     (conn->conn_state == TARG_CONN_STATE_LOGGED_IN)) {
825                         iscsit_inc_conn_usage_count(conn);
826                         spin_unlock_bh(&sess->conn_lock);
827                         return conn;
828                 }
829         }
830         spin_unlock_bh(&sess->conn_lock);
831
832         return NULL;
833 }
834
835 struct iscsi_conn *iscsit_get_conn_from_cid_rcfr(struct iscsi_session *sess, u16 cid)
836 {
837         struct iscsi_conn *conn;
838
839         spin_lock_bh(&sess->conn_lock);
840         list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
841                 if (conn->cid == cid) {
842                         iscsit_inc_conn_usage_count(conn);
843                         spin_lock(&conn->state_lock);
844                         atomic_set(&conn->connection_wait_rcfr, 1);
845                         spin_unlock(&conn->state_lock);
846                         spin_unlock_bh(&sess->conn_lock);
847                         return conn;
848                 }
849         }
850         spin_unlock_bh(&sess->conn_lock);
851
852         return NULL;
853 }
854
855 void iscsit_check_conn_usage_count(struct iscsi_conn *conn)
856 {
857         spin_lock_bh(&conn->conn_usage_lock);
858         if (conn->conn_usage_count != 0) {
859                 conn->conn_waiting_on_uc = 1;
860                 spin_unlock_bh(&conn->conn_usage_lock);
861
862                 wait_for_completion(&conn->conn_waiting_on_uc_comp);
863                 return;
864         }
865         spin_unlock_bh(&conn->conn_usage_lock);
866 }
867
868 void iscsit_dec_conn_usage_count(struct iscsi_conn *conn)
869 {
870         spin_lock_bh(&conn->conn_usage_lock);
871         conn->conn_usage_count--;
872
873         if (!conn->conn_usage_count && conn->conn_waiting_on_uc)
874                 complete(&conn->conn_waiting_on_uc_comp);
875
876         spin_unlock_bh(&conn->conn_usage_lock);
877 }
878
879 void iscsit_inc_conn_usage_count(struct iscsi_conn *conn)
880 {
881         spin_lock_bh(&conn->conn_usage_lock);
882         conn->conn_usage_count++;
883         spin_unlock_bh(&conn->conn_usage_lock);
884 }
885
886 static int iscsit_add_nopin(struct iscsi_conn *conn, int want_response)
887 {
888         u8 state;
889         struct iscsi_cmd *cmd;
890
891         cmd = iscsit_allocate_cmd(conn, TASK_RUNNING);
892         if (!cmd)
893                 return -1;
894
895         cmd->iscsi_opcode = ISCSI_OP_NOOP_IN;
896         state = (want_response) ? ISTATE_SEND_NOPIN_WANT_RESPONSE :
897                                 ISTATE_SEND_NOPIN_NO_RESPONSE;
898         cmd->init_task_tag = RESERVED_ITT;
899         cmd->targ_xfer_tag = (want_response) ?
900                              session_get_next_ttt(conn->sess) : 0xFFFFFFFF;
901         spin_lock_bh(&conn->cmd_lock);
902         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
903         spin_unlock_bh(&conn->cmd_lock);
904
905         if (want_response)
906                 iscsit_start_nopin_response_timer(conn);
907         iscsit_add_cmd_to_immediate_queue(cmd, conn, state);
908
909         return 0;
910 }
911
912 void iscsit_handle_nopin_response_timeout(struct timer_list *t)
913 {
914         struct iscsi_conn *conn = from_timer(conn, t, nopin_response_timer);
915         struct iscsi_session *sess = conn->sess;
916
917         iscsit_inc_conn_usage_count(conn);
918
919         spin_lock_bh(&conn->nopin_timer_lock);
920         if (conn->nopin_response_timer_flags & ISCSI_TF_STOP) {
921                 spin_unlock_bh(&conn->nopin_timer_lock);
922                 iscsit_dec_conn_usage_count(conn);
923                 return;
924         }
925
926         pr_err("Did not receive response to NOPIN on CID: %hu, failing"
927                 " connection for I_T Nexus %s,i,0x%6phN,%s,t,0x%02x\n",
928                 conn->cid, sess->sess_ops->InitiatorName, sess->isid,
929                 sess->tpg->tpg_tiqn->tiqn, (u32)sess->tpg->tpgt);
930         conn->nopin_response_timer_flags &= ~ISCSI_TF_RUNNING;
931         spin_unlock_bh(&conn->nopin_timer_lock);
932
933         iscsit_fill_cxn_timeout_err_stats(sess);
934         iscsit_cause_connection_reinstatement(conn, 0);
935         iscsit_dec_conn_usage_count(conn);
936 }
937
938 void iscsit_mod_nopin_response_timer(struct iscsi_conn *conn)
939 {
940         struct iscsi_session *sess = conn->sess;
941         struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
942
943         spin_lock_bh(&conn->nopin_timer_lock);
944         if (!(conn->nopin_response_timer_flags & ISCSI_TF_RUNNING)) {
945                 spin_unlock_bh(&conn->nopin_timer_lock);
946                 return;
947         }
948
949         mod_timer(&conn->nopin_response_timer,
950                 (get_jiffies_64() + na->nopin_response_timeout * HZ));
951         spin_unlock_bh(&conn->nopin_timer_lock);
952 }
953
954 void iscsit_start_nopin_response_timer(struct iscsi_conn *conn)
955 {
956         struct iscsi_session *sess = conn->sess;
957         struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
958
959         spin_lock_bh(&conn->nopin_timer_lock);
960         if (conn->nopin_response_timer_flags & ISCSI_TF_RUNNING) {
961                 spin_unlock_bh(&conn->nopin_timer_lock);
962                 return;
963         }
964
965         conn->nopin_response_timer_flags &= ~ISCSI_TF_STOP;
966         conn->nopin_response_timer_flags |= ISCSI_TF_RUNNING;
967         mod_timer(&conn->nopin_response_timer,
968                   jiffies + na->nopin_response_timeout * HZ);
969
970         pr_debug("Started NOPIN Response Timer on CID: %d to %u"
971                 " seconds\n", conn->cid, na->nopin_response_timeout);
972         spin_unlock_bh(&conn->nopin_timer_lock);
973 }
974
975 void iscsit_stop_nopin_response_timer(struct iscsi_conn *conn)
976 {
977         spin_lock_bh(&conn->nopin_timer_lock);
978         if (!(conn->nopin_response_timer_flags & ISCSI_TF_RUNNING)) {
979                 spin_unlock_bh(&conn->nopin_timer_lock);
980                 return;
981         }
982         conn->nopin_response_timer_flags |= ISCSI_TF_STOP;
983         spin_unlock_bh(&conn->nopin_timer_lock);
984
985         del_timer_sync(&conn->nopin_response_timer);
986
987         spin_lock_bh(&conn->nopin_timer_lock);
988         conn->nopin_response_timer_flags &= ~ISCSI_TF_RUNNING;
989         spin_unlock_bh(&conn->nopin_timer_lock);
990 }
991
992 void iscsit_handle_nopin_timeout(struct timer_list *t)
993 {
994         struct iscsi_conn *conn = from_timer(conn, t, nopin_timer);
995
996         iscsit_inc_conn_usage_count(conn);
997
998         spin_lock_bh(&conn->nopin_timer_lock);
999         if (conn->nopin_timer_flags & ISCSI_TF_STOP) {
1000                 spin_unlock_bh(&conn->nopin_timer_lock);
1001                 iscsit_dec_conn_usage_count(conn);
1002                 return;
1003         }
1004         conn->nopin_timer_flags &= ~ISCSI_TF_RUNNING;
1005         spin_unlock_bh(&conn->nopin_timer_lock);
1006
1007         iscsit_add_nopin(conn, 1);
1008         iscsit_dec_conn_usage_count(conn);
1009 }
1010
1011 void __iscsit_start_nopin_timer(struct iscsi_conn *conn)
1012 {
1013         struct iscsi_session *sess = conn->sess;
1014         struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
1015
1016         lockdep_assert_held(&conn->nopin_timer_lock);
1017
1018         /*
1019         * NOPIN timeout is disabled.
1020          */
1021         if (!na->nopin_timeout)
1022                 return;
1023
1024         if (conn->nopin_timer_flags & ISCSI_TF_RUNNING)
1025                 return;
1026
1027         conn->nopin_timer_flags &= ~ISCSI_TF_STOP;
1028         conn->nopin_timer_flags |= ISCSI_TF_RUNNING;
1029         mod_timer(&conn->nopin_timer, jiffies + na->nopin_timeout * HZ);
1030
1031         pr_debug("Started NOPIN Timer on CID: %d at %u second"
1032                 " interval\n", conn->cid, na->nopin_timeout);
1033 }
1034
1035 void iscsit_start_nopin_timer(struct iscsi_conn *conn)
1036 {
1037         spin_lock_bh(&conn->nopin_timer_lock);
1038         __iscsit_start_nopin_timer(conn);
1039         spin_unlock_bh(&conn->nopin_timer_lock);
1040 }
1041
1042 void iscsit_stop_nopin_timer(struct iscsi_conn *conn)
1043 {
1044         spin_lock_bh(&conn->nopin_timer_lock);
1045         if (!(conn->nopin_timer_flags & ISCSI_TF_RUNNING)) {
1046                 spin_unlock_bh(&conn->nopin_timer_lock);
1047                 return;
1048         }
1049         conn->nopin_timer_flags |= ISCSI_TF_STOP;
1050         spin_unlock_bh(&conn->nopin_timer_lock);
1051
1052         del_timer_sync(&conn->nopin_timer);
1053
1054         spin_lock_bh(&conn->nopin_timer_lock);
1055         conn->nopin_timer_flags &= ~ISCSI_TF_RUNNING;
1056         spin_unlock_bh(&conn->nopin_timer_lock);
1057 }
1058
1059 int iscsit_send_tx_data(
1060         struct iscsi_cmd *cmd,
1061         struct iscsi_conn *conn,
1062         int use_misc)
1063 {
1064         int tx_sent, tx_size;
1065         u32 iov_count;
1066         struct kvec *iov;
1067
1068 send_data:
1069         tx_size = cmd->tx_size;
1070
1071         if (!use_misc) {
1072                 iov = &cmd->iov_data[0];
1073                 iov_count = cmd->iov_data_count;
1074         } else {
1075                 iov = &cmd->iov_misc[0];
1076                 iov_count = cmd->iov_misc_count;
1077         }
1078
1079         tx_sent = tx_data(conn, &iov[0], iov_count, tx_size);
1080         if (tx_size != tx_sent) {
1081                 if (tx_sent == -EAGAIN) {
1082                         pr_err("tx_data() returned -EAGAIN\n");
1083                         goto send_data;
1084                 } else
1085                         return -1;
1086         }
1087         cmd->tx_size = 0;
1088
1089         return 0;
1090 }
1091
1092 int iscsit_fe_sendpage_sg(
1093         struct iscsi_cmd *cmd,
1094         struct iscsi_conn *conn)
1095 {
1096         struct scatterlist *sg = cmd->first_data_sg;
1097         struct kvec iov;
1098         u32 tx_hdr_size, data_len;
1099         u32 offset = cmd->first_data_sg_off;
1100         int tx_sent, iov_off;
1101
1102 send_hdr:
1103         tx_hdr_size = ISCSI_HDR_LEN;
1104         if (conn->conn_ops->HeaderDigest)
1105                 tx_hdr_size += ISCSI_CRC_LEN;
1106
1107         iov.iov_base = cmd->pdu;
1108         iov.iov_len = tx_hdr_size;
1109
1110         tx_sent = tx_data(conn, &iov, 1, tx_hdr_size);
1111         if (tx_hdr_size != tx_sent) {
1112                 if (tx_sent == -EAGAIN) {
1113                         pr_err("tx_data() returned -EAGAIN\n");
1114                         goto send_hdr;
1115                 }
1116                 return -1;
1117         }
1118
1119         data_len = cmd->tx_size - tx_hdr_size - cmd->padding;
1120         /*
1121          * Set iov_off used by padding and data digest tx_data() calls below
1122          * in order to determine proper offset into cmd->iov_data[]
1123          */
1124         if (conn->conn_ops->DataDigest) {
1125                 data_len -= ISCSI_CRC_LEN;
1126                 if (cmd->padding)
1127                         iov_off = (cmd->iov_data_count - 2);
1128                 else
1129                         iov_off = (cmd->iov_data_count - 1);
1130         } else {
1131                 iov_off = (cmd->iov_data_count - 1);
1132         }
1133         /*
1134          * Perform sendpage() for each page in the scatterlist
1135          */
1136         while (data_len) {
1137                 u32 space = (sg->length - offset);
1138                 u32 sub_len = min_t(u32, data_len, space);
1139 send_pg:
1140                 tx_sent = conn->sock->ops->sendpage(conn->sock,
1141                                         sg_page(sg), sg->offset + offset, sub_len, 0);
1142                 if (tx_sent != sub_len) {
1143                         if (tx_sent == -EAGAIN) {
1144                                 pr_err("tcp_sendpage() returned"
1145                                                 " -EAGAIN\n");
1146                                 goto send_pg;
1147                         }
1148
1149                         pr_err("tcp_sendpage() failure: %d\n",
1150                                         tx_sent);
1151                         return -1;
1152                 }
1153
1154                 data_len -= sub_len;
1155                 offset = 0;
1156                 sg = sg_next(sg);
1157         }
1158
1159 send_padding:
1160         if (cmd->padding) {
1161                 struct kvec *iov_p = &cmd->iov_data[iov_off++];
1162
1163                 tx_sent = tx_data(conn, iov_p, 1, cmd->padding);
1164                 if (cmd->padding != tx_sent) {
1165                         if (tx_sent == -EAGAIN) {
1166                                 pr_err("tx_data() returned -EAGAIN\n");
1167                                 goto send_padding;
1168                         }
1169                         return -1;
1170                 }
1171         }
1172
1173 send_datacrc:
1174         if (conn->conn_ops->DataDigest) {
1175                 struct kvec *iov_d = &cmd->iov_data[iov_off];
1176
1177                 tx_sent = tx_data(conn, iov_d, 1, ISCSI_CRC_LEN);
1178                 if (ISCSI_CRC_LEN != tx_sent) {
1179                         if (tx_sent == -EAGAIN) {
1180                                 pr_err("tx_data() returned -EAGAIN\n");
1181                                 goto send_datacrc;
1182                         }
1183                         return -1;
1184                 }
1185         }
1186
1187         return 0;
1188 }
1189
1190 /*
1191  *      This function is used for mainly sending a ISCSI_TARG_LOGIN_RSP PDU
1192  *      back to the Initiator when an expection condition occurs with the
1193  *      errors set in status_class and status_detail.
1194  *
1195  *      Parameters:     iSCSI Connection, Status Class, Status Detail.
1196  *      Returns:        0 on success, -1 on error.
1197  */
1198 int iscsit_tx_login_rsp(struct iscsi_conn *conn, u8 status_class, u8 status_detail)
1199 {
1200         struct iscsi_login_rsp *hdr;
1201         struct iscsi_login *login = conn->conn_login;
1202
1203         login->login_failed = 1;
1204         iscsit_collect_login_stats(conn, status_class, status_detail);
1205
1206         memset(&login->rsp[0], 0, ISCSI_HDR_LEN);
1207
1208         hdr     = (struct iscsi_login_rsp *)&login->rsp[0];
1209         hdr->opcode             = ISCSI_OP_LOGIN_RSP;
1210         hdr->status_class       = status_class;
1211         hdr->status_detail      = status_detail;
1212         hdr->itt                = conn->login_itt;
1213
1214         return conn->conn_transport->iscsit_put_login_tx(conn, login, 0);
1215 }
1216
1217 void iscsit_print_session_params(struct iscsi_session *sess)
1218 {
1219         struct iscsi_conn *conn;
1220
1221         pr_debug("-----------------------------[Session Params for"
1222                 " SID: %u]-----------------------------\n", sess->sid);
1223         spin_lock_bh(&sess->conn_lock);
1224         list_for_each_entry(conn, &sess->sess_conn_list, conn_list)
1225                 iscsi_dump_conn_ops(conn->conn_ops);
1226         spin_unlock_bh(&sess->conn_lock);
1227
1228         iscsi_dump_sess_ops(sess->sess_ops);
1229 }
1230
1231 static int iscsit_do_rx_data(
1232         struct iscsi_conn *conn,
1233         struct iscsi_data_count *count)
1234 {
1235         int data = count->data_length, rx_loop = 0, total_rx = 0;
1236         struct msghdr msg;
1237
1238         if (!conn || !conn->sock || !conn->conn_ops)
1239                 return -1;
1240
1241         memset(&msg, 0, sizeof(struct msghdr));
1242         iov_iter_kvec(&msg.msg_iter, READ, count->iov, count->iov_count, data);
1243
1244         while (msg_data_left(&msg)) {
1245                 rx_loop = sock_recvmsg(conn->sock, &msg, MSG_WAITALL);
1246                 if (rx_loop <= 0) {
1247                         pr_debug("rx_loop: %d total_rx: %d\n",
1248                                 rx_loop, total_rx);
1249                         return rx_loop;
1250                 }
1251                 total_rx += rx_loop;
1252                 pr_debug("rx_loop: %d, total_rx: %d, data: %d\n",
1253                                 rx_loop, total_rx, data);
1254         }
1255
1256         return total_rx;
1257 }
1258
1259 int rx_data(
1260         struct iscsi_conn *conn,
1261         struct kvec *iov,
1262         int iov_count,
1263         int data)
1264 {
1265         struct iscsi_data_count c;
1266
1267         if (!conn || !conn->sock || !conn->conn_ops)
1268                 return -1;
1269
1270         memset(&c, 0, sizeof(struct iscsi_data_count));
1271         c.iov = iov;
1272         c.iov_count = iov_count;
1273         c.data_length = data;
1274         c.type = ISCSI_RX_DATA;
1275
1276         return iscsit_do_rx_data(conn, &c);
1277 }
1278
1279 int tx_data(
1280         struct iscsi_conn *conn,
1281         struct kvec *iov,
1282         int iov_count,
1283         int data)
1284 {
1285         struct msghdr msg;
1286         int total_tx = 0;
1287
1288         if (!conn || !conn->sock || !conn->conn_ops)
1289                 return -1;
1290
1291         if (data <= 0) {
1292                 pr_err("Data length is: %d\n", data);
1293                 return -1;
1294         }
1295
1296         memset(&msg, 0, sizeof(struct msghdr));
1297
1298         iov_iter_kvec(&msg.msg_iter, WRITE, iov, iov_count, data);
1299
1300         while (msg_data_left(&msg)) {
1301                 int tx_loop = sock_sendmsg(conn->sock, &msg);
1302                 if (tx_loop <= 0) {
1303                         pr_debug("tx_loop: %d total_tx %d\n",
1304                                 tx_loop, total_tx);
1305                         return tx_loop;
1306                 }
1307                 total_tx += tx_loop;
1308                 pr_debug("tx_loop: %d, total_tx: %d, data: %d\n",
1309                                         tx_loop, total_tx, data);
1310         }
1311
1312         return total_tx;
1313 }
1314
1315 void iscsit_collect_login_stats(
1316         struct iscsi_conn *conn,
1317         u8 status_class,
1318         u8 status_detail)
1319 {
1320         struct iscsi_param *intrname = NULL;
1321         struct iscsi_tiqn *tiqn;
1322         struct iscsi_login_stats *ls;
1323
1324         tiqn = iscsit_snmp_get_tiqn(conn);
1325         if (!tiqn)
1326                 return;
1327
1328         ls = &tiqn->login_stats;
1329
1330         spin_lock(&ls->lock);
1331         if (status_class == ISCSI_STATUS_CLS_SUCCESS)
1332                 ls->accepts++;
1333         else if (status_class == ISCSI_STATUS_CLS_REDIRECT) {
1334                 ls->redirects++;
1335                 ls->last_fail_type = ISCSI_LOGIN_FAIL_REDIRECT;
1336         } else if ((status_class == ISCSI_STATUS_CLS_INITIATOR_ERR)  &&
1337                  (status_detail == ISCSI_LOGIN_STATUS_AUTH_FAILED)) {
1338                 ls->authenticate_fails++;
1339                 ls->last_fail_type =  ISCSI_LOGIN_FAIL_AUTHENTICATE;
1340         } else if ((status_class == ISCSI_STATUS_CLS_INITIATOR_ERR)  &&
1341                  (status_detail == ISCSI_LOGIN_STATUS_TGT_FORBIDDEN)) {
1342                 ls->authorize_fails++;
1343                 ls->last_fail_type = ISCSI_LOGIN_FAIL_AUTHORIZE;
1344         } else if ((status_class == ISCSI_STATUS_CLS_INITIATOR_ERR) &&
1345                  (status_detail == ISCSI_LOGIN_STATUS_INIT_ERR)) {
1346                 ls->negotiate_fails++;
1347                 ls->last_fail_type = ISCSI_LOGIN_FAIL_NEGOTIATE;
1348         } else {
1349                 ls->other_fails++;
1350                 ls->last_fail_type = ISCSI_LOGIN_FAIL_OTHER;
1351         }
1352
1353         /* Save initiator name, ip address and time, if it is a failed login */
1354         if (status_class != ISCSI_STATUS_CLS_SUCCESS) {
1355                 if (conn->param_list)
1356                         intrname = iscsi_find_param_from_key(INITIATORNAME,
1357                                                              conn->param_list);
1358                 strlcpy(ls->last_intr_fail_name,
1359                        (intrname ? intrname->value : "Unknown"),
1360                        sizeof(ls->last_intr_fail_name));
1361
1362                 ls->last_intr_fail_ip_family = conn->login_family;
1363
1364                 ls->last_intr_fail_sockaddr = conn->login_sockaddr;
1365                 ls->last_fail_time = get_jiffies_64();
1366         }
1367
1368         spin_unlock(&ls->lock);
1369 }
1370
1371 struct iscsi_tiqn *iscsit_snmp_get_tiqn(struct iscsi_conn *conn)
1372 {
1373         struct iscsi_portal_group *tpg;
1374
1375         if (!conn)
1376                 return NULL;
1377
1378         tpg = conn->tpg;
1379         if (!tpg)
1380                 return NULL;
1381
1382         if (!tpg->tpg_tiqn)
1383                 return NULL;
1384
1385         return tpg->tpg_tiqn;
1386 }
1387
1388 void iscsit_fill_cxn_timeout_err_stats(struct iscsi_session *sess)
1389 {
1390         struct iscsi_portal_group *tpg = sess->tpg;
1391         struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
1392
1393         if (!tiqn)
1394                 return;
1395
1396         spin_lock_bh(&tiqn->sess_err_stats.lock);
1397         strlcpy(tiqn->sess_err_stats.last_sess_fail_rem_name,
1398                         sess->sess_ops->InitiatorName,
1399                         sizeof(tiqn->sess_err_stats.last_sess_fail_rem_name));
1400         tiqn->sess_err_stats.last_sess_failure_type =
1401                         ISCSI_SESS_ERR_CXN_TIMEOUT;
1402         tiqn->sess_err_stats.cxn_timeout_errors++;
1403         atomic_long_inc(&sess->conn_timeout_errors);
1404         spin_unlock_bh(&tiqn->sess_err_stats.lock);
1405 }