2 * iSCSI Initiator over TCP/IP Data-Path
4 * Copyright (C) 2004 Dmitry Yusupov
5 * Copyright (C) 2004 Alex Aizman
6 * Copyright (C) 2005 - 2006 Mike Christie
7 * Copyright (C) 2006 Red Hat, Inc. All rights reserved.
8 * maintained by open-iscsi@googlegroups.com
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published
12 * by the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
15 * This program is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * General Public License for more details.
20 * See the file COPYING included with this distribution for more details.
29 #include <linux/types.h>
30 #include <linux/list.h>
31 #include <linux/inet.h>
32 #include <linux/file.h>
33 #include <linux/blkdev.h>
34 #include <linux/crypto.h>
35 #include <linux/delay.h>
36 #include <linux/kfifo.h>
37 #include <linux/scatterlist.h>
39 #include <scsi/scsi_cmnd.h>
40 #include <scsi/scsi_device.h>
41 #include <scsi/scsi_host.h>
42 #include <scsi/scsi.h>
43 #include <scsi/scsi_transport_iscsi.h>
45 #include "iscsi_tcp.h"
47 MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
48 "Alex Aizman <itn780@yahoo.com>");
49 MODULE_DESCRIPTION("iSCSI/TCP data-path");
50 MODULE_LICENSE("GPL");
55 #define debug_tcp(fmt...) printk(KERN_INFO "tcp: " fmt)
57 #define debug_tcp(fmt...)
67 static struct scsi_transport_template
*iscsi_tcp_scsi_transport
;
68 static struct scsi_host_template iscsi_sht
;
69 static struct iscsi_transport iscsi_tcp_transport
;
71 static unsigned int iscsi_max_lun
= 512;
72 module_param_named(max_lun
, iscsi_max_lun
, uint
, S_IRUGO
);
74 static int iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn
*tcp_conn
,
75 struct iscsi_segment
*segment
);
78 * Scatterlist handling: inside the iscsi_segment, we
79 * remember an index into the scatterlist, and set data/size
80 * to the current scatterlist entry. For highmem pages, we
83 * Note that the page is unmapped when we return from
84 * TCP's data_ready handler, so we may end up mapping and
85 * unmapping the same page repeatedly. The whole reason
86 * for this is that we shouldn't keep the page mapped
87 * outside the softirq.
91 * iscsi_tcp_segment_init_sg - init indicated scatterlist entry
92 * @segment: the buffer object
94 * @offset: byte offset into that sg entry
96 * This function sets up the segment so that subsequent
97 * data is copied to the indicated sg entry, at the given
101 iscsi_tcp_segment_init_sg(struct iscsi_segment
*segment
,
102 struct scatterlist
*sg
, unsigned int offset
)
105 segment
->sg_offset
= offset
;
106 segment
->size
= min(sg
->length
- offset
,
107 segment
->total_size
- segment
->total_copied
);
108 segment
->data
= NULL
;
112 * iscsi_tcp_segment_map - map the current S/G page
113 * @segment: iscsi_segment
114 * @recv: 1 if called from recv path
116 * We only need to possibly kmap data if scatter lists are being used,
117 * because the iscsi passthrough and internal IO paths will never use high
121 iscsi_tcp_segment_map(struct iscsi_segment
*segment
, int recv
)
123 struct scatterlist
*sg
;
125 if (segment
->data
!= NULL
|| !segment
->sg
)
129 BUG_ON(segment
->sg_mapped
);
130 BUG_ON(sg
->length
== 0);
133 * If the page count is greater than one it is ok to send
134 * to the network layer's zero copy send path. If not we
135 * have to go the slow sendmsg path. We always map for the
138 if (page_count(sg_page(sg
)) >= 1 && !recv
)
141 debug_tcp("iscsi_tcp_segment_map %s %p\n", recv
? "recv" : "xmit",
143 segment
->sg_mapped
= kmap_atomic(sg_page(sg
), KM_SOFTIRQ0
);
144 segment
->data
= segment
->sg_mapped
+ sg
->offset
+ segment
->sg_offset
;
148 iscsi_tcp_segment_unmap(struct iscsi_segment
*segment
)
150 debug_tcp("iscsi_tcp_segment_unmap %p\n", segment
);
152 if (segment
->sg_mapped
) {
153 debug_tcp("iscsi_tcp_segment_unmap valid\n");
154 kunmap_atomic(segment
->sg_mapped
, KM_SOFTIRQ0
);
155 segment
->sg_mapped
= NULL
;
156 segment
->data
= NULL
;
161 * Splice the digest buffer into the buffer
164 iscsi_tcp_segment_splice_digest(struct iscsi_segment
*segment
, void *digest
)
166 segment
->data
= digest
;
167 segment
->digest_len
= ISCSI_DIGEST_SIZE
;
168 segment
->total_size
+= ISCSI_DIGEST_SIZE
;
169 segment
->size
= ISCSI_DIGEST_SIZE
;
172 segment
->hash
= NULL
;
176 * iscsi_tcp_segment_done - check whether the segment is complete
177 * @segment: iscsi segment to check
178 * @recv: set to one of this is called from the recv path
179 * @copied: number of bytes copied
181 * Check if we're done receiving this segment. If the receive
182 * buffer is full but we expect more data, move on to the
183 * next entry in the scatterlist.
185 * If the amount of data we received isn't a multiple of 4,
186 * we will transparently receive the pad bytes, too.
188 * This function must be re-entrant.
191 iscsi_tcp_segment_done(struct iscsi_segment
*segment
, int recv
, unsigned copied
)
193 static unsigned char padbuf
[ISCSI_PAD_LEN
];
194 struct scatterlist sg
;
197 debug_tcp("copied %u %u size %u %s\n", segment
->copied
, copied
,
198 segment
->size
, recv
? "recv" : "xmit");
199 if (segment
->hash
&& copied
) {
201 * If a segment is kmapd we must unmap it before sending
202 * to the crypto layer since that will try to kmap it again.
204 iscsi_tcp_segment_unmap(segment
);
206 if (!segment
->data
) {
207 sg_init_table(&sg
, 1);
208 sg_set_page(&sg
, sg_page(segment
->sg
), copied
,
209 segment
->copied
+ segment
->sg_offset
+
210 segment
->sg
->offset
);
212 sg_init_one(&sg
, segment
->data
+ segment
->copied
,
214 crypto_hash_update(segment
->hash
, &sg
, copied
);
217 segment
->copied
+= copied
;
218 if (segment
->copied
< segment
->size
) {
219 iscsi_tcp_segment_map(segment
, recv
);
223 segment
->total_copied
+= segment
->copied
;
227 /* Unmap the current scatterlist page, if there is one. */
228 iscsi_tcp_segment_unmap(segment
);
230 /* Do we have more scatterlist entries? */
231 debug_tcp("total copied %u total size %u\n", segment
->total_copied
,
232 segment
->total_size
);
233 if (segment
->total_copied
< segment
->total_size
) {
234 /* Proceed to the next entry in the scatterlist. */
235 iscsi_tcp_segment_init_sg(segment
, sg_next(segment
->sg
),
237 iscsi_tcp_segment_map(segment
, recv
);
238 BUG_ON(segment
->size
== 0);
242 /* Do we need to handle padding? */
243 pad
= iscsi_padding(segment
->total_copied
);
245 debug_tcp("consume %d pad bytes\n", pad
);
246 segment
->total_size
+= pad
;
248 segment
->data
= padbuf
;
253 * Set us up for transferring the data digest. hdr digest
254 * is completely handled in hdr done function.
257 crypto_hash_final(segment
->hash
, segment
->digest
);
258 iscsi_tcp_segment_splice_digest(segment
,
259 recv
? segment
->recv_digest
: segment
->digest
);
267 * iscsi_tcp_xmit_segment - transmit segment
268 * @tcp_conn: the iSCSI TCP connection
269 * @segment: the buffer to transmnit
271 * This function transmits as much of the buffer as
272 * the network layer will accept, and returns the number of
275 * If CRC hashing is enabled, the function will compute the
276 * hash as it goes. When the entire segment has been transmitted,
277 * it will retrieve the hash value and send it as well.
280 iscsi_tcp_xmit_segment(struct iscsi_tcp_conn
*tcp_conn
,
281 struct iscsi_segment
*segment
)
283 struct socket
*sk
= tcp_conn
->sock
;
284 unsigned int copied
= 0;
287 while (!iscsi_tcp_segment_done(segment
, 0, r
)) {
288 struct scatterlist
*sg
;
289 unsigned int offset
, copy
;
293 offset
= segment
->copied
;
294 copy
= segment
->size
- offset
;
296 if (segment
->total_copied
+ segment
->size
< segment
->total_size
)
299 /* Use sendpage if we can; else fall back to sendmsg */
300 if (!segment
->data
) {
302 offset
+= segment
->sg_offset
+ sg
->offset
;
303 r
= tcp_conn
->sendpage(sk
, sg_page(sg
), offset
, copy
,
306 struct msghdr msg
= { .msg_flags
= flags
};
308 .iov_base
= segment
->data
+ offset
,
312 r
= kernel_sendmsg(sk
, &msg
, &iov
, 1, copy
);
316 iscsi_tcp_segment_unmap(segment
);
317 if (copied
|| r
== -EAGAIN
)
327 * iscsi_tcp_segment_recv - copy data to segment
328 * @tcp_conn: the iSCSI TCP connection
329 * @segment: the buffer to copy to
331 * @len: amount of data available
333 * This function copies up to @len bytes to the
334 * given buffer, and returns the number of bytes
335 * consumed, which can actually be less than @len.
337 * If hash digest is enabled, the function will update the
338 * hash while copying.
339 * Combining these two operations doesn't buy us a lot (yet),
340 * but in the future we could implement combined copy+crc,
341 * just way we do for network layer checksums.
344 iscsi_tcp_segment_recv(struct iscsi_tcp_conn
*tcp_conn
,
345 struct iscsi_segment
*segment
, const void *ptr
,
348 unsigned int copy
= 0, copied
= 0;
350 while (!iscsi_tcp_segment_done(segment
, 1, copy
)) {
352 debug_tcp("iscsi_tcp_segment_recv copied %d bytes\n",
357 copy
= min(len
- copied
, segment
->size
- segment
->copied
);
358 debug_tcp("iscsi_tcp_segment_recv copying %d\n", copy
);
359 memcpy(segment
->data
+ segment
->copied
, ptr
+ copied
, copy
);
366 iscsi_tcp_dgst_header(struct hash_desc
*hash
, const void *hdr
, size_t hdrlen
,
367 unsigned char digest
[ISCSI_DIGEST_SIZE
])
369 struct scatterlist sg
;
371 sg_init_one(&sg
, hdr
, hdrlen
);
372 crypto_hash_digest(hash
, &sg
, hdrlen
, digest
);
376 iscsi_tcp_dgst_verify(struct iscsi_tcp_conn
*tcp_conn
,
377 struct iscsi_segment
*segment
)
379 if (!segment
->digest_len
)
382 if (memcmp(segment
->recv_digest
, segment
->digest
,
383 segment
->digest_len
)) {
384 debug_scsi("digest mismatch\n");
392 * Helper function to set up segment buffer
395 __iscsi_segment_init(struct iscsi_segment
*segment
, size_t size
,
396 iscsi_segment_done_fn_t
*done
, struct hash_desc
*hash
)
398 memset(segment
, 0, sizeof(*segment
));
399 segment
->total_size
= size
;
400 segment
->done
= done
;
403 segment
->hash
= hash
;
404 crypto_hash_init(hash
);
409 iscsi_segment_init_linear(struct iscsi_segment
*segment
, void *data
,
410 size_t size
, iscsi_segment_done_fn_t
*done
,
411 struct hash_desc
*hash
)
413 __iscsi_segment_init(segment
, size
, done
, hash
);
414 segment
->data
= data
;
415 segment
->size
= size
;
419 iscsi_segment_seek_sg(struct iscsi_segment
*segment
,
420 struct scatterlist
*sg_list
, unsigned int sg_count
,
421 unsigned int offset
, size_t size
,
422 iscsi_segment_done_fn_t
*done
, struct hash_desc
*hash
)
424 struct scatterlist
*sg
;
427 debug_scsi("iscsi_segment_seek_sg offset %u size %llu\n",
429 __iscsi_segment_init(segment
, size
, done
, hash
);
430 for_each_sg(sg_list
, sg
, sg_count
, i
) {
431 debug_scsi("sg %d, len %u offset %u\n", i
, sg
->length
,
433 if (offset
< sg
->length
) {
434 iscsi_tcp_segment_init_sg(segment
, sg
, offset
);
437 offset
-= sg
->length
;
440 return ISCSI_ERR_DATA_OFFSET
;
444 * iscsi_tcp_hdr_recv_prep - prep segment for hdr reception
445 * @tcp_conn: iscsi connection to prep for
447 * This function always passes NULL for the hash argument, because when this
448 * function is called we do not yet know the final size of the header and want
449 * to delay the digest processing until we know that.
452 iscsi_tcp_hdr_recv_prep(struct iscsi_tcp_conn
*tcp_conn
)
454 debug_tcp("iscsi_tcp_hdr_recv_prep(%p%s)\n", tcp_conn
,
455 tcp_conn
->iscsi_conn
->hdrdgst_en
? ", digest enabled" : "");
456 iscsi_segment_init_linear(&tcp_conn
->in
.segment
,
457 tcp_conn
->in
.hdr_buf
, sizeof(struct iscsi_hdr
),
458 iscsi_tcp_hdr_recv_done
, NULL
);
462 * Handle incoming reply to any other type of command
465 iscsi_tcp_data_recv_done(struct iscsi_tcp_conn
*tcp_conn
,
466 struct iscsi_segment
*segment
)
468 struct iscsi_conn
*conn
= tcp_conn
->iscsi_conn
;
471 if (!iscsi_tcp_dgst_verify(tcp_conn
, segment
))
472 return ISCSI_ERR_DATA_DGST
;
474 rc
= iscsi_complete_pdu(conn
, tcp_conn
->in
.hdr
,
475 conn
->data
, tcp_conn
->in
.datalen
);
479 iscsi_tcp_hdr_recv_prep(tcp_conn
);
484 iscsi_tcp_data_recv_prep(struct iscsi_tcp_conn
*tcp_conn
)
486 struct iscsi_conn
*conn
= tcp_conn
->iscsi_conn
;
487 struct hash_desc
*rx_hash
= NULL
;
489 if (conn
->datadgst_en
)
490 rx_hash
= &tcp_conn
->rx_hash
;
492 iscsi_segment_init_linear(&tcp_conn
->in
.segment
,
493 conn
->data
, tcp_conn
->in
.datalen
,
494 iscsi_tcp_data_recv_done
, rx_hash
);
498 * must be called with session lock
501 iscsi_tcp_cleanup_task(struct iscsi_conn
*conn
, struct iscsi_task
*task
)
503 struct iscsi_tcp_task
*tcp_task
= task
->dd_data
;
504 struct iscsi_r2t_info
*r2t
;
506 /* nothing to do for mgmt tasks */
510 /* flush task's r2t queues */
511 while (__kfifo_get(tcp_task
->r2tqueue
, (void*)&r2t
, sizeof(void*))) {
512 __kfifo_put(tcp_task
->r2tpool
.queue
, (void*)&r2t
,
514 debug_scsi("iscsi_tcp_cleanup_task pending r2t dropped\n");
519 __kfifo_put(tcp_task
->r2tpool
.queue
, (void*)&r2t
,
521 tcp_task
->r2t
= NULL
;
526 * iscsi_data_in - SCSI Data-In Response processing
527 * @conn: iscsi connection
528 * @task: scsi command task
531 iscsi_data_in(struct iscsi_conn
*conn
, struct iscsi_task
*task
)
533 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
534 struct iscsi_tcp_task
*tcp_task
= task
->dd_data
;
535 struct iscsi_data_rsp
*rhdr
= (struct iscsi_data_rsp
*)tcp_conn
->in
.hdr
;
536 int datasn
= be32_to_cpu(rhdr
->datasn
);
537 unsigned total_in_length
= scsi_in(task
->sc
)->length
;
539 iscsi_update_cmdsn(conn
->session
, (struct iscsi_nopin
*)rhdr
);
540 if (tcp_conn
->in
.datalen
== 0)
543 if (tcp_task
->exp_datasn
!= datasn
) {
544 debug_tcp("%s: task->exp_datasn(%d) != rhdr->datasn(%d)\n",
545 __func__
, tcp_task
->exp_datasn
, datasn
);
546 return ISCSI_ERR_DATASN
;
549 tcp_task
->exp_datasn
++;
551 tcp_task
->data_offset
= be32_to_cpu(rhdr
->offset
);
552 if (tcp_task
->data_offset
+ tcp_conn
->in
.datalen
> total_in_length
) {
553 debug_tcp("%s: data_offset(%d) + data_len(%d) > total_length_in(%d)\n",
554 __func__
, tcp_task
->data_offset
,
555 tcp_conn
->in
.datalen
, total_in_length
);
556 return ISCSI_ERR_DATA_OFFSET
;
559 conn
->datain_pdus_cnt
++;
564 * iscsi_solicit_data_init - initialize first Data-Out
565 * @conn: iscsi connection
566 * @task: scsi command task
570 * Initialize first Data-Out within this R2T sequence and finds
571 * proper data_offset within this SCSI command.
573 * This function is called with connection lock taken.
576 iscsi_solicit_data_init(struct iscsi_conn
*conn
, struct iscsi_task
*task
,
577 struct iscsi_r2t_info
*r2t
)
579 struct iscsi_data
*hdr
;
581 hdr
= &r2t
->dtask
.hdr
;
582 memset(hdr
, 0, sizeof(struct iscsi_data
));
584 hdr
->datasn
= cpu_to_be32(r2t
->solicit_datasn
);
585 r2t
->solicit_datasn
++;
586 hdr
->opcode
= ISCSI_OP_SCSI_DATA_OUT
;
587 memcpy(hdr
->lun
, task
->hdr
->lun
, sizeof(hdr
->lun
));
588 hdr
->itt
= task
->hdr
->itt
;
589 hdr
->exp_statsn
= r2t
->exp_statsn
;
590 hdr
->offset
= cpu_to_be32(r2t
->data_offset
);
591 if (r2t
->data_length
> conn
->max_xmit_dlength
) {
592 hton24(hdr
->dlength
, conn
->max_xmit_dlength
);
593 r2t
->data_count
= conn
->max_xmit_dlength
;
596 hton24(hdr
->dlength
, r2t
->data_length
);
597 r2t
->data_count
= r2t
->data_length
;
598 hdr
->flags
= ISCSI_FLAG_CMD_FINAL
;
600 conn
->dataout_pdus_cnt
++;
606 * iscsi_r2t_rsp - iSCSI R2T Response processing
607 * @conn: iscsi connection
608 * @task: scsi command task
611 iscsi_r2t_rsp(struct iscsi_conn
*conn
, struct iscsi_task
*task
)
613 struct iscsi_r2t_info
*r2t
;
614 struct iscsi_session
*session
= conn
->session
;
615 struct iscsi_tcp_task
*tcp_task
= task
->dd_data
;
616 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
617 struct iscsi_r2t_rsp
*rhdr
= (struct iscsi_r2t_rsp
*)tcp_conn
->in
.hdr
;
618 int r2tsn
= be32_to_cpu(rhdr
->r2tsn
);
621 if (tcp_conn
->in
.datalen
) {
622 iscsi_conn_printk(KERN_ERR
, conn
,
623 "invalid R2t with datalen %d\n",
624 tcp_conn
->in
.datalen
);
625 return ISCSI_ERR_DATALEN
;
628 if (tcp_task
->exp_datasn
!= r2tsn
){
629 debug_tcp("%s: task->exp_datasn(%d) != rhdr->r2tsn(%d)\n",
630 __func__
, tcp_task
->exp_datasn
, r2tsn
);
631 return ISCSI_ERR_R2TSN
;
634 /* fill-in new R2T associated with the task */
635 iscsi_update_cmdsn(session
, (struct iscsi_nopin
*)rhdr
);
637 if (!task
->sc
|| session
->state
!= ISCSI_STATE_LOGGED_IN
) {
638 iscsi_conn_printk(KERN_INFO
, conn
,
639 "dropping R2T itt %d in recovery.\n",
644 rc
= __kfifo_get(tcp_task
->r2tpool
.queue
, (void*)&r2t
, sizeof(void*));
647 r2t
->exp_statsn
= rhdr
->statsn
;
648 r2t
->data_length
= be32_to_cpu(rhdr
->data_length
);
649 if (r2t
->data_length
== 0) {
650 iscsi_conn_printk(KERN_ERR
, conn
,
651 "invalid R2T with zero data len\n");
652 __kfifo_put(tcp_task
->r2tpool
.queue
, (void*)&r2t
,
654 return ISCSI_ERR_DATALEN
;
657 if (r2t
->data_length
> session
->max_burst
)
658 debug_scsi("invalid R2T with data len %u and max burst %u."
659 "Attempting to execute request.\n",
660 r2t
->data_length
, session
->max_burst
);
662 r2t
->data_offset
= be32_to_cpu(rhdr
->data_offset
);
663 if (r2t
->data_offset
+ r2t
->data_length
> scsi_out(task
->sc
)->length
) {
664 iscsi_conn_printk(KERN_ERR
, conn
,
665 "invalid R2T with data len %u at offset %u "
666 "and total length %d\n", r2t
->data_length
,
667 r2t
->data_offset
, scsi_out(task
->sc
)->length
);
668 __kfifo_put(tcp_task
->r2tpool
.queue
, (void*)&r2t
,
670 return ISCSI_ERR_DATALEN
;
673 r2t
->ttt
= rhdr
->ttt
; /* no flip */
674 r2t
->solicit_datasn
= 0;
676 iscsi_solicit_data_init(conn
, task
, r2t
);
678 tcp_task
->exp_datasn
= r2tsn
+ 1;
679 __kfifo_put(tcp_task
->r2tqueue
, (void*)&r2t
, sizeof(void*));
680 conn
->r2t_pdus_cnt
++;
682 iscsi_requeue_task(task
);
687 * Handle incoming reply to DataIn command
690 iscsi_tcp_process_data_in(struct iscsi_tcp_conn
*tcp_conn
,
691 struct iscsi_segment
*segment
)
693 struct iscsi_conn
*conn
= tcp_conn
->iscsi_conn
;
694 struct iscsi_hdr
*hdr
= tcp_conn
->in
.hdr
;
697 if (!iscsi_tcp_dgst_verify(tcp_conn
, segment
))
698 return ISCSI_ERR_DATA_DGST
;
700 /* check for non-exceptional status */
701 if (hdr
->flags
& ISCSI_FLAG_DATA_STATUS
) {
702 rc
= iscsi_complete_pdu(conn
, tcp_conn
->in
.hdr
, NULL
, 0);
707 iscsi_tcp_hdr_recv_prep(tcp_conn
);
712 * iscsi_tcp_hdr_dissect - process PDU header
713 * @conn: iSCSI connection
716 * This function analyzes the header of the PDU received,
717 * and performs several sanity checks. If the PDU is accompanied
718 * by data, the receive buffer is set up to copy the incoming data
719 * to the correct location.
722 iscsi_tcp_hdr_dissect(struct iscsi_conn
*conn
, struct iscsi_hdr
*hdr
)
724 int rc
= 0, opcode
, ahslen
;
725 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
726 struct iscsi_task
*task
;
728 /* verify PDU length */
729 tcp_conn
->in
.datalen
= ntoh24(hdr
->dlength
);
730 if (tcp_conn
->in
.datalen
> conn
->max_recv_dlength
) {
731 iscsi_conn_printk(KERN_ERR
, conn
,
732 "iscsi_tcp: datalen %d > %d\n",
733 tcp_conn
->in
.datalen
, conn
->max_recv_dlength
);
734 return ISCSI_ERR_DATALEN
;
737 /* Additional header segments. So far, we don't
738 * process additional headers.
740 ahslen
= hdr
->hlength
<< 2;
742 opcode
= hdr
->opcode
& ISCSI_OPCODE_MASK
;
743 /* verify itt (itt encoding: age+cid+itt) */
744 rc
= iscsi_verify_itt(conn
, hdr
->itt
);
748 debug_tcp("opcode 0x%x ahslen %d datalen %d\n",
749 opcode
, ahslen
, tcp_conn
->in
.datalen
);
752 case ISCSI_OP_SCSI_DATA_IN
:
753 spin_lock(&conn
->session
->lock
);
754 task
= iscsi_itt_to_ctask(conn
, hdr
->itt
);
756 rc
= ISCSI_ERR_BAD_ITT
;
758 rc
= iscsi_data_in(conn
, task
);
760 spin_unlock(&conn
->session
->lock
);
764 if (tcp_conn
->in
.datalen
) {
765 struct iscsi_tcp_task
*tcp_task
= task
->dd_data
;
766 struct hash_desc
*rx_hash
= NULL
;
767 struct scsi_data_buffer
*sdb
= scsi_in(task
->sc
);
770 * Setup copy of Data-In into the Scsi_Cmnd
772 * We set up the iscsi_segment to point to the next
773 * scatterlist entry to copy to. As we go along,
774 * we move on to the next scatterlist entry and
775 * update the digest per-entry.
777 if (conn
->datadgst_en
)
778 rx_hash
= &tcp_conn
->rx_hash
;
780 debug_tcp("iscsi_tcp_begin_data_in(%p, offset=%d, "
781 "datalen=%d)\n", tcp_conn
,
782 tcp_task
->data_offset
,
783 tcp_conn
->in
.datalen
);
784 rc
= iscsi_segment_seek_sg(&tcp_conn
->in
.segment
,
787 tcp_task
->data_offset
,
788 tcp_conn
->in
.datalen
,
789 iscsi_tcp_process_data_in
,
791 spin_unlock(&conn
->session
->lock
);
794 rc
= __iscsi_complete_pdu(conn
, hdr
, NULL
, 0);
795 spin_unlock(&conn
->session
->lock
);
797 case ISCSI_OP_SCSI_CMD_RSP
:
798 if (tcp_conn
->in
.datalen
) {
799 iscsi_tcp_data_recv_prep(tcp_conn
);
802 rc
= iscsi_complete_pdu(conn
, hdr
, NULL
, 0);
805 spin_lock(&conn
->session
->lock
);
806 task
= iscsi_itt_to_ctask(conn
, hdr
->itt
);
808 rc
= ISCSI_ERR_BAD_ITT
;
810 rc
= ISCSI_ERR_AHSLEN
;
811 else if (task
->sc
->sc_data_direction
== DMA_TO_DEVICE
)
812 rc
= iscsi_r2t_rsp(conn
, task
);
814 rc
= ISCSI_ERR_PROTO
;
815 spin_unlock(&conn
->session
->lock
);
817 case ISCSI_OP_LOGIN_RSP
:
818 case ISCSI_OP_TEXT_RSP
:
819 case ISCSI_OP_REJECT
:
820 case ISCSI_OP_ASYNC_EVENT
:
822 * It is possible that we could get a PDU with a buffer larger
823 * than 8K, but there are no targets that currently do this.
824 * For now we fail until we find a vendor that needs it
826 if (ISCSI_DEF_MAX_RECV_SEG_LEN
< tcp_conn
->in
.datalen
) {
827 iscsi_conn_printk(KERN_ERR
, conn
,
828 "iscsi_tcp: received buffer of "
829 "len %u but conn buffer is only %u "
831 tcp_conn
->in
.datalen
,
832 ISCSI_DEF_MAX_RECV_SEG_LEN
, opcode
);
833 rc
= ISCSI_ERR_PROTO
;
837 /* If there's data coming in with the response,
838 * receive it to the connection's buffer.
840 if (tcp_conn
->in
.datalen
) {
841 iscsi_tcp_data_recv_prep(tcp_conn
);
845 case ISCSI_OP_LOGOUT_RSP
:
846 case ISCSI_OP_NOOP_IN
:
847 case ISCSI_OP_SCSI_TMFUNC_RSP
:
848 rc
= iscsi_complete_pdu(conn
, hdr
, NULL
, 0);
851 rc
= ISCSI_ERR_BAD_OPCODE
;
856 /* Anything that comes with data should have
857 * been handled above. */
858 if (tcp_conn
->in
.datalen
)
859 return ISCSI_ERR_PROTO
;
860 iscsi_tcp_hdr_recv_prep(tcp_conn
);
867 * iscsi_tcp_hdr_recv_done - process PDU header
869 * This is the callback invoked when the PDU header has
870 * been received. If the header is followed by additional
871 * header segments, we go back for more data.
874 iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn
*tcp_conn
,
875 struct iscsi_segment
*segment
)
877 struct iscsi_conn
*conn
= tcp_conn
->iscsi_conn
;
878 struct iscsi_hdr
*hdr
;
880 /* Check if there are additional header segments
881 * *prior* to computing the digest, because we
882 * may need to go back to the caller for more.
884 hdr
= (struct iscsi_hdr
*) tcp_conn
->in
.hdr_buf
;
885 if (segment
->copied
== sizeof(struct iscsi_hdr
) && hdr
->hlength
) {
886 /* Bump the header length - the caller will
887 * just loop around and get the AHS for us, and
889 unsigned int ahslen
= hdr
->hlength
<< 2;
891 /* Make sure we don't overflow */
892 if (sizeof(*hdr
) + ahslen
> sizeof(tcp_conn
->in
.hdr_buf
))
893 return ISCSI_ERR_AHSLEN
;
895 segment
->total_size
+= ahslen
;
896 segment
->size
+= ahslen
;
900 /* We're done processing the header. See if we're doing
901 * header digests; if so, set up the recv_digest buffer
902 * and go back for more. */
903 if (conn
->hdrdgst_en
) {
904 if (segment
->digest_len
== 0) {
905 iscsi_tcp_segment_splice_digest(segment
,
906 segment
->recv_digest
);
909 iscsi_tcp_dgst_header(&tcp_conn
->rx_hash
, hdr
,
910 segment
->total_copied
- ISCSI_DIGEST_SIZE
,
913 if (!iscsi_tcp_dgst_verify(tcp_conn
, segment
))
914 return ISCSI_ERR_HDR_DGST
;
917 tcp_conn
->in
.hdr
= hdr
;
918 return iscsi_tcp_hdr_dissect(conn
, hdr
);
922 * iscsi_tcp_recv - TCP receive in sendfile fashion
923 * @rd_desc: read descriptor
924 * @skb: socket buffer
925 * @offset: offset in skb
926 * @len: skb->len - offset
929 iscsi_tcp_recv(read_descriptor_t
*rd_desc
, struct sk_buff
*skb
,
930 unsigned int offset
, size_t len
)
932 struct iscsi_conn
*conn
= rd_desc
->arg
.data
;
933 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
934 struct iscsi_segment
*segment
= &tcp_conn
->in
.segment
;
935 struct skb_seq_state seq
;
936 unsigned int consumed
= 0;
939 debug_tcp("in %d bytes\n", skb
->len
- offset
);
941 if (unlikely(conn
->suspend_rx
)) {
942 debug_tcp("conn %d Rx suspended!\n", conn
->id
);
946 skb_prepare_seq_read(skb
, offset
, skb
->len
, &seq
);
951 avail
= skb_seq_read(consumed
, &ptr
, &seq
);
953 debug_tcp("no more data avail. Consumed %d\n",
957 BUG_ON(segment
->copied
>= segment
->size
);
959 debug_tcp("skb %p ptr=%p avail=%u\n", skb
, ptr
, avail
);
960 rc
= iscsi_tcp_segment_recv(tcp_conn
, segment
, ptr
, avail
);
964 if (segment
->total_copied
>= segment
->total_size
) {
965 debug_tcp("segment done\n");
966 rc
= segment
->done(tcp_conn
, segment
);
968 skb_abort_seq_read(&seq
);
972 /* The done() functions sets up the
976 skb_abort_seq_read(&seq
);
977 conn
->rxdata_octets
+= consumed
;
981 debug_tcp("Error receiving PDU, errno=%d\n", rc
);
982 iscsi_conn_failure(conn
, rc
);
987 iscsi_tcp_data_ready(struct sock
*sk
, int flag
)
989 struct iscsi_conn
*conn
= sk
->sk_user_data
;
990 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
991 read_descriptor_t rd_desc
;
993 read_lock(&sk
->sk_callback_lock
);
996 * Use rd_desc to pass 'conn' to iscsi_tcp_recv.
997 * We set count to 1 because we want the network layer to
998 * hand us all the skbs that are available. iscsi_tcp_recv
999 * handled pdus that cross buffers or pdus that still need data.
1001 rd_desc
.arg
.data
= conn
;
1003 tcp_read_sock(sk
, &rd_desc
, iscsi_tcp_recv
);
1005 read_unlock(&sk
->sk_callback_lock
);
1007 /* If we had to (atomically) map a highmem page,
1009 iscsi_tcp_segment_unmap(&tcp_conn
->in
.segment
);
1013 iscsi_tcp_state_change(struct sock
*sk
)
1015 struct iscsi_tcp_conn
*tcp_conn
;
1016 struct iscsi_conn
*conn
;
1017 struct iscsi_session
*session
;
1018 void (*old_state_change
)(struct sock
*);
1020 read_lock(&sk
->sk_callback_lock
);
1022 conn
= (struct iscsi_conn
*)sk
->sk_user_data
;
1023 session
= conn
->session
;
1025 if ((sk
->sk_state
== TCP_CLOSE_WAIT
||
1026 sk
->sk_state
== TCP_CLOSE
) &&
1027 !atomic_read(&sk
->sk_rmem_alloc
)) {
1028 debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n");
1029 iscsi_conn_failure(conn
, ISCSI_ERR_CONN_FAILED
);
1032 tcp_conn
= conn
->dd_data
;
1033 old_state_change
= tcp_conn
->old_state_change
;
1035 read_unlock(&sk
->sk_callback_lock
);
1037 old_state_change(sk
);
1041 * iscsi_write_space - Called when more output buffer space is available
1042 * @sk: socket space is available for
1045 iscsi_write_space(struct sock
*sk
)
1047 struct iscsi_conn
*conn
= (struct iscsi_conn
*)sk
->sk_user_data
;
1048 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1050 tcp_conn
->old_write_space(sk
);
1051 debug_tcp("iscsi_write_space: cid %d\n", conn
->id
);
1052 scsi_queue_work(conn
->session
->host
, &conn
->xmitwork
);
1056 iscsi_conn_set_callbacks(struct iscsi_conn
*conn
)
1058 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1059 struct sock
*sk
= tcp_conn
->sock
->sk
;
1061 /* assign new callbacks */
1062 write_lock_bh(&sk
->sk_callback_lock
);
1063 sk
->sk_user_data
= conn
;
1064 tcp_conn
->old_data_ready
= sk
->sk_data_ready
;
1065 tcp_conn
->old_state_change
= sk
->sk_state_change
;
1066 tcp_conn
->old_write_space
= sk
->sk_write_space
;
1067 sk
->sk_data_ready
= iscsi_tcp_data_ready
;
1068 sk
->sk_state_change
= iscsi_tcp_state_change
;
1069 sk
->sk_write_space
= iscsi_write_space
;
1070 write_unlock_bh(&sk
->sk_callback_lock
);
1074 iscsi_conn_restore_callbacks(struct iscsi_tcp_conn
*tcp_conn
)
1076 struct sock
*sk
= tcp_conn
->sock
->sk
;
1078 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
1079 write_lock_bh(&sk
->sk_callback_lock
);
1080 sk
->sk_user_data
= NULL
;
1081 sk
->sk_data_ready
= tcp_conn
->old_data_ready
;
1082 sk
->sk_state_change
= tcp_conn
->old_state_change
;
1083 sk
->sk_write_space
= tcp_conn
->old_write_space
;
1084 sk
->sk_no_check
= 0;
1085 write_unlock_bh(&sk
->sk_callback_lock
);
1089 * iscsi_xmit - TCP transmit
1092 iscsi_xmit(struct iscsi_conn
*conn
)
1094 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1095 struct iscsi_segment
*segment
= &tcp_conn
->out
.segment
;
1096 unsigned int consumed
= 0;
1100 rc
= iscsi_tcp_xmit_segment(tcp_conn
, segment
);
1102 rc
= ISCSI_ERR_XMIT_FAILED
;
1110 if (segment
->total_copied
>= segment
->total_size
) {
1111 if (segment
->done
!= NULL
) {
1112 rc
= segment
->done(tcp_conn
, segment
);
1119 debug_tcp("xmit %d bytes\n", consumed
);
1121 conn
->txdata_octets
+= consumed
;
1125 /* Transmit error. We could initiate error recovery
1127 debug_tcp("Error sending PDU, errno=%d\n", rc
);
1128 iscsi_conn_failure(conn
, rc
);
1133 * iscsi_tcp_xmit_qlen - return the number of bytes queued for xmit
1136 iscsi_tcp_xmit_qlen(struct iscsi_conn
*conn
)
1138 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1139 struct iscsi_segment
*segment
= &tcp_conn
->out
.segment
;
1141 return segment
->total_copied
- segment
->total_size
;
1145 iscsi_tcp_flush(struct iscsi_conn
*conn
)
1149 while (iscsi_tcp_xmit_qlen(conn
)) {
1150 rc
= iscsi_xmit(conn
);
1161 * This is called when we're done sending the header.
1162 * Simply copy the data_segment to the send segment, and return.
1165 iscsi_tcp_send_hdr_done(struct iscsi_tcp_conn
*tcp_conn
,
1166 struct iscsi_segment
*segment
)
1168 tcp_conn
->out
.segment
= tcp_conn
->out
.data_segment
;
1169 debug_tcp("Header done. Next segment size %u total_size %u\n",
1170 tcp_conn
->out
.segment
.size
, tcp_conn
->out
.segment
.total_size
);
1175 iscsi_tcp_send_hdr_prep(struct iscsi_conn
*conn
, void *hdr
, size_t hdrlen
)
1177 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1179 debug_tcp("%s(%p%s)\n", __func__
, tcp_conn
,
1180 conn
->hdrdgst_en
? ", digest enabled" : "");
1182 /* Clear the data segment - needs to be filled in by the
1183 * caller using iscsi_tcp_send_data_prep() */
1184 memset(&tcp_conn
->out
.data_segment
, 0, sizeof(struct iscsi_segment
));
1186 /* If header digest is enabled, compute the CRC and
1187 * place the digest into the same buffer. We make
1188 * sure that both iscsi_tcp_task and mtask have
1191 if (conn
->hdrdgst_en
) {
1192 iscsi_tcp_dgst_header(&tcp_conn
->tx_hash
, hdr
, hdrlen
,
1194 hdrlen
+= ISCSI_DIGEST_SIZE
;
1197 /* Remember header pointer for later, when we need
1198 * to decide whether there's a payload to go along
1199 * with the header. */
1200 tcp_conn
->out
.hdr
= hdr
;
1202 iscsi_segment_init_linear(&tcp_conn
->out
.segment
, hdr
, hdrlen
,
1203 iscsi_tcp_send_hdr_done
, NULL
);
1207 * Prepare the send buffer for the payload data.
1208 * Padding and checksumming will all be taken care
1209 * of by the iscsi_segment routines.
1212 iscsi_tcp_send_data_prep(struct iscsi_conn
*conn
, struct scatterlist
*sg
,
1213 unsigned int count
, unsigned int offset
,
1216 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1217 struct hash_desc
*tx_hash
= NULL
;
1218 unsigned int hdr_spec_len
;
1220 debug_tcp("%s(%p, offset=%d, datalen=%d%s)\n", __func__
,
1221 tcp_conn
, offset
, len
,
1222 conn
->datadgst_en
? ", digest enabled" : "");
1224 /* Make sure the datalen matches what the caller
1225 said he would send. */
1226 hdr_spec_len
= ntoh24(tcp_conn
->out
.hdr
->dlength
);
1227 WARN_ON(iscsi_padded(len
) != iscsi_padded(hdr_spec_len
));
1229 if (conn
->datadgst_en
)
1230 tx_hash
= &tcp_conn
->tx_hash
;
1232 return iscsi_segment_seek_sg(&tcp_conn
->out
.data_segment
,
1233 sg
, count
, offset
, len
,
1238 iscsi_tcp_send_linear_data_prepare(struct iscsi_conn
*conn
, void *data
,
1241 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1242 struct hash_desc
*tx_hash
= NULL
;
1243 unsigned int hdr_spec_len
;
1245 debug_tcp("%s(%p, datalen=%d%s)\n", __func__
, tcp_conn
, len
,
1246 conn
->datadgst_en
? ", digest enabled" : "");
1248 /* Make sure the datalen matches what the caller
1249 said he would send. */
1250 hdr_spec_len
= ntoh24(tcp_conn
->out
.hdr
->dlength
);
1251 WARN_ON(iscsi_padded(len
) != iscsi_padded(hdr_spec_len
));
1253 if (conn
->datadgst_en
)
1254 tx_hash
= &tcp_conn
->tx_hash
;
1256 iscsi_segment_init_linear(&tcp_conn
->out
.data_segment
,
1257 data
, len
, NULL
, tx_hash
);
1261 * iscsi_solicit_data_cont - initialize next Data-Out
1262 * @conn: iscsi connection
1263 * @task: scsi command task
1265 * @left: bytes left to transfer
1268 * Initialize next Data-Out within this R2T sequence and continue
1269 * to process next Scatter-Gather element(if any) of this SCSI command.
1271 * Called under connection lock.
1274 iscsi_solicit_data_cont(struct iscsi_conn
*conn
, struct iscsi_task
*task
,
1275 struct iscsi_r2t_info
*r2t
)
1277 struct iscsi_data
*hdr
;
1278 int new_offset
, left
;
1280 BUG_ON(r2t
->data_length
- r2t
->sent
< 0);
1281 left
= r2t
->data_length
- r2t
->sent
;
1285 hdr
= &r2t
->dtask
.hdr
;
1286 memset(hdr
, 0, sizeof(struct iscsi_data
));
1287 hdr
->ttt
= r2t
->ttt
;
1288 hdr
->datasn
= cpu_to_be32(r2t
->solicit_datasn
);
1289 r2t
->solicit_datasn
++;
1290 hdr
->opcode
= ISCSI_OP_SCSI_DATA_OUT
;
1291 memcpy(hdr
->lun
, task
->hdr
->lun
, sizeof(hdr
->lun
));
1292 hdr
->itt
= task
->hdr
->itt
;
1293 hdr
->exp_statsn
= r2t
->exp_statsn
;
1294 new_offset
= r2t
->data_offset
+ r2t
->sent
;
1295 hdr
->offset
= cpu_to_be32(new_offset
);
1296 if (left
> conn
->max_xmit_dlength
) {
1297 hton24(hdr
->dlength
, conn
->max_xmit_dlength
);
1298 r2t
->data_count
= conn
->max_xmit_dlength
;
1300 hton24(hdr
->dlength
, left
);
1301 r2t
->data_count
= left
;
1302 hdr
->flags
= ISCSI_FLAG_CMD_FINAL
;
1305 conn
->dataout_pdus_cnt
++;
1310 * iscsi_tcp_task - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
1311 * @conn: iscsi connection
1312 * @task: scsi command task
1316 iscsi_tcp_task_init(struct iscsi_task
*task
)
1318 struct iscsi_tcp_task
*tcp_task
= task
->dd_data
;
1319 struct iscsi_conn
*conn
= task
->conn
;
1320 struct scsi_cmnd
*sc
= task
->sc
;
1325 * mgmt tasks do not have a scatterlist since they come
1326 * in from the iscsi interface.
1328 debug_scsi("mtask deq [cid %d itt 0x%x]\n", conn
->id
,
1331 /* Prepare PDU, optionally w/ immediate data */
1332 iscsi_tcp_send_hdr_prep(conn
, task
->hdr
, sizeof(*task
->hdr
));
1334 /* If we have immediate data, attach a payload */
1335 if (task
->data_count
)
1336 iscsi_tcp_send_linear_data_prepare(conn
, task
->data
,
1341 BUG_ON(__kfifo_len(tcp_task
->r2tqueue
));
1343 tcp_task
->exp_datasn
= 0;
1345 /* Prepare PDU, optionally w/ immediate data */
1346 debug_scsi("task deq [cid %d itt 0x%x imm %d unsol %d]\n",
1347 conn
->id
, task
->itt
, task
->imm_count
,
1349 iscsi_tcp_send_hdr_prep(conn
, task
->hdr
, task
->hdr_len
);
1351 if (!task
->imm_count
)
1354 /* If we have immediate data, attach a payload */
1355 err
= iscsi_tcp_send_data_prep(conn
, scsi_out(sc
)->table
.sgl
,
1356 scsi_out(sc
)->table
.nents
,
1357 0, task
->imm_count
);
1360 tcp_task
->sent
+= task
->imm_count
;
1361 task
->imm_count
= 0;
1366 * iscsi_tcp_task_xmit - xmit normal PDU task
1367 * @task: iscsi command task
1369 * We're expected to return 0 when everything was transmitted succesfully,
1370 * -EAGAIN if there's still data in the queue, or != 0 for any other kind
1374 iscsi_tcp_task_xmit(struct iscsi_task
*task
)
1376 struct iscsi_conn
*conn
= task
->conn
;
1377 struct iscsi_tcp_task
*tcp_task
= task
->dd_data
;
1378 struct scsi_cmnd
*sc
= task
->sc
;
1379 struct scsi_data_buffer
*sdb
;
1383 /* Flush any pending data first. */
1384 rc
= iscsi_tcp_flush(conn
);
1390 if (task
->hdr
->itt
== RESERVED_ITT
)
1391 iscsi_put_task(task
);
1395 /* Are we done already? */
1396 if (sc
->sc_data_direction
!= DMA_TO_DEVICE
)
1400 if (task
->unsol_count
!= 0) {
1401 struct iscsi_data
*hdr
= &tcp_task
->unsol_dtask
.hdr
;
1403 /* Prepare a header for the unsolicited PDU.
1404 * The amount of data we want to send will be
1405 * in task->data_count.
1406 * FIXME: return the data count instead.
1408 iscsi_prep_unsolicit_data_pdu(task
, hdr
);
1410 debug_tcp("unsol dout [itt 0x%x doff %d dlen %d]\n",
1411 task
->itt
, tcp_task
->sent
, task
->data_count
);
1413 iscsi_tcp_send_hdr_prep(conn
, hdr
, sizeof(*hdr
));
1414 rc
= iscsi_tcp_send_data_prep(conn
, sdb
->table
.sgl
,
1415 sdb
->table
.nents
, tcp_task
->sent
,
1419 tcp_task
->sent
+= task
->data_count
;
1420 task
->unsol_count
-= task
->data_count
;
1423 struct iscsi_session
*session
= conn
->session
;
1424 struct iscsi_r2t_info
*r2t
;
1426 /* All unsolicited PDUs sent. Check for solicited PDUs.
1428 spin_lock_bh(&session
->lock
);
1429 r2t
= tcp_task
->r2t
;
1431 /* Continue with this R2T? */
1432 if (!iscsi_solicit_data_cont(conn
, task
, r2t
)) {
1433 debug_scsi(" done with r2t %p\n", r2t
);
1435 __kfifo_put(tcp_task
->r2tpool
.queue
,
1436 (void*)&r2t
, sizeof(void*));
1437 tcp_task
->r2t
= r2t
= NULL
;
1442 __kfifo_get(tcp_task
->r2tqueue
, (void*)&tcp_task
->r2t
,
1444 r2t
= tcp_task
->r2t
;
1446 spin_unlock_bh(&session
->lock
);
1448 /* Waiting for more R2Ts to arrive. */
1450 debug_tcp("no R2Ts yet\n");
1454 debug_scsi("sol dout %p [dsn %d itt 0x%x doff %d dlen %d]\n",
1455 r2t
, r2t
->solicit_datasn
- 1, task
->itt
,
1456 r2t
->data_offset
+ r2t
->sent
, r2t
->data_count
);
1458 iscsi_tcp_send_hdr_prep(conn
, &r2t
->dtask
.hdr
,
1459 sizeof(struct iscsi_hdr
));
1461 rc
= iscsi_tcp_send_data_prep(conn
, sdb
->table
.sgl
,
1463 r2t
->data_offset
+ r2t
->sent
,
1467 tcp_task
->sent
+= r2t
->data_count
;
1468 r2t
->sent
+= r2t
->data_count
;
1473 iscsi_conn_failure(conn
, rc
);
1477 static struct iscsi_cls_conn
*
1478 iscsi_tcp_conn_create(struct iscsi_cls_session
*cls_session
, uint32_t conn_idx
)
1480 struct iscsi_conn
*conn
;
1481 struct iscsi_cls_conn
*cls_conn
;
1482 struct iscsi_tcp_conn
*tcp_conn
;
1484 cls_conn
= iscsi_conn_setup(cls_session
, sizeof(*tcp_conn
), conn_idx
);
1487 conn
= cls_conn
->dd_data
;
1489 * due to strange issues with iser these are not set
1490 * in iscsi_conn_setup
1492 conn
->max_recv_dlength
= ISCSI_DEF_MAX_RECV_SEG_LEN
;
1494 tcp_conn
= conn
->dd_data
;
1495 tcp_conn
->iscsi_conn
= conn
;
1497 tcp_conn
->tx_hash
.tfm
= crypto_alloc_hash("crc32c", 0,
1499 tcp_conn
->tx_hash
.flags
= 0;
1500 if (IS_ERR(tcp_conn
->tx_hash
.tfm
))
1503 tcp_conn
->rx_hash
.tfm
= crypto_alloc_hash("crc32c", 0,
1505 tcp_conn
->rx_hash
.flags
= 0;
1506 if (IS_ERR(tcp_conn
->rx_hash
.tfm
))
1512 crypto_free_hash(tcp_conn
->tx_hash
.tfm
);
1514 iscsi_conn_printk(KERN_ERR
, conn
,
1515 "Could not create connection due to crc32c "
1516 "loading error. Make sure the crc32c "
1517 "module is built as a module or into the "
1519 iscsi_conn_teardown(cls_conn
);
1524 iscsi_tcp_release_conn(struct iscsi_conn
*conn
)
1526 struct iscsi_session
*session
= conn
->session
;
1527 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1528 struct socket
*sock
= tcp_conn
->sock
;
1533 sock_hold(sock
->sk
);
1534 iscsi_conn_restore_callbacks(tcp_conn
);
1537 spin_lock_bh(&session
->lock
);
1538 tcp_conn
->sock
= NULL
;
1539 spin_unlock_bh(&session
->lock
);
1544 iscsi_tcp_conn_destroy(struct iscsi_cls_conn
*cls_conn
)
1546 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
1547 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1549 iscsi_tcp_release_conn(conn
);
1551 if (tcp_conn
->tx_hash
.tfm
)
1552 crypto_free_hash(tcp_conn
->tx_hash
.tfm
);
1553 if (tcp_conn
->rx_hash
.tfm
)
1554 crypto_free_hash(tcp_conn
->rx_hash
.tfm
);
1556 iscsi_conn_teardown(cls_conn
);
1560 iscsi_tcp_conn_stop(struct iscsi_cls_conn
*cls_conn
, int flag
)
1562 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
1563 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1565 /* userspace may have goofed up and not bound us */
1566 if (!tcp_conn
->sock
)
1569 * Make sure our recv side is stopped.
1570 * Older tools called conn stop before ep_disconnect
1571 * so IO could still be coming in.
1573 write_lock_bh(&tcp_conn
->sock
->sk
->sk_callback_lock
);
1574 set_bit(ISCSI_SUSPEND_BIT
, &conn
->suspend_rx
);
1575 write_unlock_bh(&tcp_conn
->sock
->sk
->sk_callback_lock
);
1577 iscsi_conn_stop(cls_conn
, flag
);
1578 iscsi_tcp_release_conn(conn
);
1581 static int iscsi_tcp_get_addr(struct iscsi_conn
*conn
, struct socket
*sock
,
1582 char *buf
, int *port
,
1583 int (*getname
)(struct socket
*, struct sockaddr
*,
1586 struct sockaddr_storage
*addr
;
1587 struct sockaddr_in6
*sin6
;
1588 struct sockaddr_in
*sin
;
1591 addr
= kmalloc(sizeof(*addr
), GFP_KERNEL
);
1595 if (getname(sock
, (struct sockaddr
*) addr
, &len
)) {
1600 switch (addr
->ss_family
) {
1602 sin
= (struct sockaddr_in
*)addr
;
1603 spin_lock_bh(&conn
->session
->lock
);
1604 sprintf(buf
, NIPQUAD_FMT
, NIPQUAD(sin
->sin_addr
.s_addr
));
1605 *port
= be16_to_cpu(sin
->sin_port
);
1606 spin_unlock_bh(&conn
->session
->lock
);
1609 sin6
= (struct sockaddr_in6
*)addr
;
1610 spin_lock_bh(&conn
->session
->lock
);
1611 sprintf(buf
, NIP6_FMT
, NIP6(sin6
->sin6_addr
));
1612 *port
= be16_to_cpu(sin6
->sin6_port
);
1613 spin_unlock_bh(&conn
->session
->lock
);
1622 iscsi_tcp_conn_bind(struct iscsi_cls_session
*cls_session
,
1623 struct iscsi_cls_conn
*cls_conn
, uint64_t transport_eph
,
1626 struct Scsi_Host
*shost
= iscsi_session_to_shost(cls_session
);
1627 struct iscsi_host
*ihost
= shost_priv(shost
);
1628 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
1629 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1631 struct socket
*sock
;
1634 /* lookup for existing socket */
1635 sock
= sockfd_lookup((int)transport_eph
, &err
);
1637 iscsi_conn_printk(KERN_ERR
, conn
,
1638 "sockfd_lookup failed %d\n", err
);
1642 * copy these values now because if we drop the session
1643 * userspace may still want to query the values since we will
1644 * be using them for the reconnect
1646 err
= iscsi_tcp_get_addr(conn
, sock
, conn
->portal_address
,
1647 &conn
->portal_port
, kernel_getpeername
);
1651 err
= iscsi_tcp_get_addr(conn
, sock
, ihost
->local_address
,
1652 &ihost
->local_port
, kernel_getsockname
);
1656 err
= iscsi_conn_bind(cls_session
, cls_conn
, is_leading
);
1660 /* bind iSCSI connection and socket */
1661 tcp_conn
->sock
= sock
;
1663 /* setup Socket parameters */
1666 sk
->sk_sndtimeo
= 15 * HZ
; /* FIXME: make it configurable */
1667 sk
->sk_allocation
= GFP_ATOMIC
;
1669 iscsi_conn_set_callbacks(conn
);
1670 tcp_conn
->sendpage
= tcp_conn
->sock
->ops
->sendpage
;
1672 * set receive state machine into initial state
1674 iscsi_tcp_hdr_recv_prep(tcp_conn
);
1683 iscsi_r2tpool_alloc(struct iscsi_session
*session
)
1689 * initialize per-task: R2T pool and xmit queue
1691 for (cmd_i
= 0; cmd_i
< session
->cmds_max
; cmd_i
++) {
1692 struct iscsi_task
*task
= session
->cmds
[cmd_i
];
1693 struct iscsi_tcp_task
*tcp_task
= task
->dd_data
;
1696 * pre-allocated x4 as much r2ts to handle race when
1697 * target acks DataOut faster than we data_xmit() queues
1698 * could replenish r2tqueue.
1702 if (iscsi_pool_init(&tcp_task
->r2tpool
, session
->max_r2t
* 4, NULL
,
1703 sizeof(struct iscsi_r2t_info
))) {
1704 goto r2t_alloc_fail
;
1707 /* R2T xmit queue */
1708 tcp_task
->r2tqueue
= kfifo_alloc(
1709 session
->max_r2t
* 4 * sizeof(void*), GFP_KERNEL
, NULL
);
1710 if (tcp_task
->r2tqueue
== ERR_PTR(-ENOMEM
)) {
1711 iscsi_pool_free(&tcp_task
->r2tpool
);
1712 goto r2t_alloc_fail
;
1719 for (i
= 0; i
< cmd_i
; i
++) {
1720 struct iscsi_task
*task
= session
->cmds
[i
];
1721 struct iscsi_tcp_task
*tcp_task
= task
->dd_data
;
1723 kfifo_free(tcp_task
->r2tqueue
);
1724 iscsi_pool_free(&tcp_task
->r2tpool
);
1730 iscsi_r2tpool_free(struct iscsi_session
*session
)
1734 for (i
= 0; i
< session
->cmds_max
; i
++) {
1735 struct iscsi_task
*task
= session
->cmds
[i
];
1736 struct iscsi_tcp_task
*tcp_task
= task
->dd_data
;
1738 kfifo_free(tcp_task
->r2tqueue
);
1739 iscsi_pool_free(&tcp_task
->r2tpool
);
1744 iscsi_conn_set_param(struct iscsi_cls_conn
*cls_conn
, enum iscsi_param param
,
1745 char *buf
, int buflen
)
1747 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
1748 struct iscsi_session
*session
= conn
->session
;
1749 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1753 case ISCSI_PARAM_HDRDGST_EN
:
1754 iscsi_set_param(cls_conn
, param
, buf
, buflen
);
1756 case ISCSI_PARAM_DATADGST_EN
:
1757 iscsi_set_param(cls_conn
, param
, buf
, buflen
);
1758 tcp_conn
->sendpage
= conn
->datadgst_en
?
1759 sock_no_sendpage
: tcp_conn
->sock
->ops
->sendpage
;
1761 case ISCSI_PARAM_MAX_R2T
:
1762 sscanf(buf
, "%d", &value
);
1763 if (value
<= 0 || !is_power_of_2(value
))
1765 if (session
->max_r2t
== value
)
1767 iscsi_r2tpool_free(session
);
1768 iscsi_set_param(cls_conn
, param
, buf
, buflen
);
1769 if (iscsi_r2tpool_alloc(session
))
1773 return iscsi_set_param(cls_conn
, param
, buf
, buflen
);
1780 iscsi_tcp_conn_get_param(struct iscsi_cls_conn
*cls_conn
,
1781 enum iscsi_param param
, char *buf
)
1783 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
1787 case ISCSI_PARAM_CONN_PORT
:
1788 spin_lock_bh(&conn
->session
->lock
);
1789 len
= sprintf(buf
, "%hu\n", conn
->portal_port
);
1790 spin_unlock_bh(&conn
->session
->lock
);
1792 case ISCSI_PARAM_CONN_ADDRESS
:
1793 spin_lock_bh(&conn
->session
->lock
);
1794 len
= sprintf(buf
, "%s\n", conn
->portal_address
);
1795 spin_unlock_bh(&conn
->session
->lock
);
1798 return iscsi_conn_get_param(cls_conn
, param
, buf
);
1805 iscsi_conn_get_stats(struct iscsi_cls_conn
*cls_conn
, struct iscsi_stats
*stats
)
1807 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
1808 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1810 stats
->txdata_octets
= conn
->txdata_octets
;
1811 stats
->rxdata_octets
= conn
->rxdata_octets
;
1812 stats
->scsicmd_pdus
= conn
->scsicmd_pdus_cnt
;
1813 stats
->dataout_pdus
= conn
->dataout_pdus_cnt
;
1814 stats
->scsirsp_pdus
= conn
->scsirsp_pdus_cnt
;
1815 stats
->datain_pdus
= conn
->datain_pdus_cnt
;
1816 stats
->r2t_pdus
= conn
->r2t_pdus_cnt
;
1817 stats
->tmfcmd_pdus
= conn
->tmfcmd_pdus_cnt
;
1818 stats
->tmfrsp_pdus
= conn
->tmfrsp_pdus_cnt
;
1819 stats
->custom_length
= 3;
1820 strcpy(stats
->custom
[0].desc
, "tx_sendpage_failures");
1821 stats
->custom
[0].value
= tcp_conn
->sendpage_failures_cnt
;
1822 strcpy(stats
->custom
[1].desc
, "rx_discontiguous_hdr");
1823 stats
->custom
[1].value
= tcp_conn
->discontiguous_hdr_cnt
;
1824 strcpy(stats
->custom
[2].desc
, "eh_abort_cnt");
1825 stats
->custom
[2].value
= conn
->eh_abort_cnt
;
1828 static struct iscsi_cls_session
*
1829 iscsi_tcp_session_create(struct iscsi_endpoint
*ep
, uint16_t cmds_max
,
1830 uint16_t qdepth
, uint32_t initial_cmdsn
,
1833 struct iscsi_cls_session
*cls_session
;
1834 struct iscsi_session
*session
;
1835 struct Scsi_Host
*shost
;
1839 printk(KERN_ERR
"iscsi_tcp: invalid ep %p.\n", ep
);
1843 shost
= iscsi_host_alloc(&iscsi_sht
, 0, qdepth
);
1846 shost
->transportt
= iscsi_tcp_scsi_transport
;
1847 shost
->max_lun
= iscsi_max_lun
;
1849 shost
->max_channel
= 0;
1850 shost
->max_cmd_len
= SCSI_MAX_VARLEN_CDB_SIZE
;
1852 if (iscsi_host_add(shost
, NULL
))
1854 *hostno
= shost
->host_no
;
1856 cls_session
= iscsi_session_setup(&iscsi_tcp_transport
, shost
, cmds_max
,
1857 sizeof(struct iscsi_tcp_task
),
1861 session
= cls_session
->dd_data
;
1863 shost
->can_queue
= session
->scsi_cmds_max
;
1864 for (cmd_i
= 0; cmd_i
< session
->cmds_max
; cmd_i
++) {
1865 struct iscsi_task
*task
= session
->cmds
[cmd_i
];
1866 struct iscsi_tcp_task
*tcp_task
= task
->dd_data
;
1868 task
->hdr
= &tcp_task
->hdr
.cmd_hdr
;
1869 task
->hdr_max
= sizeof(tcp_task
->hdr
) - ISCSI_DIGEST_SIZE
;
1872 if (iscsi_r2tpool_alloc(session
))
1873 goto remove_session
;
1877 iscsi_session_teardown(cls_session
);
1879 iscsi_host_remove(shost
);
1881 iscsi_host_free(shost
);
1885 static void iscsi_tcp_session_destroy(struct iscsi_cls_session
*cls_session
)
1887 struct Scsi_Host
*shost
= iscsi_session_to_shost(cls_session
);
1889 iscsi_r2tpool_free(cls_session
->dd_data
);
1890 iscsi_session_teardown(cls_session
);
1892 iscsi_host_remove(shost
);
1893 iscsi_host_free(shost
);
1896 static int iscsi_tcp_slave_configure(struct scsi_device
*sdev
)
1898 blk_queue_bounce_limit(sdev
->request_queue
, BLK_BOUNCE_ANY
);
1899 blk_queue_dma_alignment(sdev
->request_queue
, 0);
1903 static struct scsi_host_template iscsi_sht
= {
1904 .module
= THIS_MODULE
,
1905 .name
= "iSCSI Initiator over TCP/IP",
1906 .queuecommand
= iscsi_queuecommand
,
1907 .change_queue_depth
= iscsi_change_queue_depth
,
1908 .can_queue
= ISCSI_DEF_XMIT_CMDS_MAX
- 1,
1909 .sg_tablesize
= 4096,
1910 .max_sectors
= 0xFFFF,
1911 .cmd_per_lun
= ISCSI_DEF_CMD_PER_LUN
,
1912 .eh_abort_handler
= iscsi_eh_abort
,
1913 .eh_device_reset_handler
= iscsi_eh_device_reset
,
1914 .eh_target_reset_handler
= iscsi_eh_target_reset
,
1915 .use_clustering
= DISABLE_CLUSTERING
,
1916 .slave_configure
= iscsi_tcp_slave_configure
,
1917 .proc_name
= "iscsi_tcp",
1921 static struct iscsi_transport iscsi_tcp_transport
= {
1922 .owner
= THIS_MODULE
,
1924 .caps
= CAP_RECOVERY_L0
| CAP_MULTI_R2T
| CAP_HDRDGST
1926 .param_mask
= ISCSI_MAX_RECV_DLENGTH
|
1927 ISCSI_MAX_XMIT_DLENGTH
|
1930 ISCSI_INITIAL_R2T_EN
|
1935 ISCSI_PDU_INORDER_EN
|
1936 ISCSI_DATASEQ_INORDER_EN
|
1939 ISCSI_CONN_ADDRESS
|
1941 ISCSI_PERSISTENT_PORT
|
1942 ISCSI_PERSISTENT_ADDRESS
|
1943 ISCSI_TARGET_NAME
| ISCSI_TPGT
|
1944 ISCSI_USERNAME
| ISCSI_PASSWORD
|
1945 ISCSI_USERNAME_IN
| ISCSI_PASSWORD_IN
|
1946 ISCSI_FAST_ABORT
| ISCSI_ABORT_TMO
|
1947 ISCSI_LU_RESET_TMO
|
1948 ISCSI_PING_TMO
| ISCSI_RECV_TMO
|
1949 ISCSI_IFACE_NAME
| ISCSI_INITIATOR_NAME
,
1950 .host_param_mask
= ISCSI_HOST_HWADDRESS
| ISCSI_HOST_IPADDRESS
|
1951 ISCSI_HOST_INITIATOR_NAME
|
1952 ISCSI_HOST_NETDEV_NAME
,
1953 /* session management */
1954 .create_session
= iscsi_tcp_session_create
,
1955 .destroy_session
= iscsi_tcp_session_destroy
,
1956 /* connection management */
1957 .create_conn
= iscsi_tcp_conn_create
,
1958 .bind_conn
= iscsi_tcp_conn_bind
,
1959 .destroy_conn
= iscsi_tcp_conn_destroy
,
1960 .set_param
= iscsi_conn_set_param
,
1961 .get_conn_param
= iscsi_tcp_conn_get_param
,
1962 .get_session_param
= iscsi_session_get_param
,
1963 .start_conn
= iscsi_conn_start
,
1964 .stop_conn
= iscsi_tcp_conn_stop
,
1965 /* iscsi host params */
1966 .get_host_param
= iscsi_host_get_param
,
1967 .set_host_param
= iscsi_host_set_param
,
1969 .send_pdu
= iscsi_conn_send_pdu
,
1970 .get_stats
= iscsi_conn_get_stats
,
1971 .init_task
= iscsi_tcp_task_init
,
1972 .xmit_task
= iscsi_tcp_task_xmit
,
1973 .cleanup_task
= iscsi_tcp_cleanup_task
,
1975 .session_recovery_timedout
= iscsi_session_recovery_timedout
,
1979 iscsi_tcp_init(void)
1981 if (iscsi_max_lun
< 1) {
1982 printk(KERN_ERR
"iscsi_tcp: Invalid max_lun value of %u\n",
1987 iscsi_tcp_scsi_transport
= iscsi_register_transport(
1988 &iscsi_tcp_transport
);
1989 if (!iscsi_tcp_scsi_transport
)
1996 iscsi_tcp_exit(void)
1998 iscsi_unregister_transport(&iscsi_tcp_transport
);
2001 module_init(iscsi_tcp_init
);
2002 module_exit(iscsi_tcp_exit
);