2 * iSCSI Initiator over TCP/IP Data-Path
4 * Copyright (C) 2004 Dmitry Yusupov
5 * Copyright (C) 2004 Alex Aizman
6 * Copyright (C) 2005 - 2006 Mike Christie
7 * Copyright (C) 2006 Red Hat, Inc. All rights reserved.
8 * maintained by open-iscsi@googlegroups.com
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published
12 * by the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
15 * This program is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * General Public License for more details.
20 * See the file COPYING included with this distribution for more details.
29 #include <linux/types.h>
30 #include <linux/list.h>
31 #include <linux/inet.h>
32 #include <linux/file.h>
33 #include <linux/blkdev.h>
34 #include <linux/crypto.h>
35 #include <linux/delay.h>
36 #include <linux/kfifo.h>
37 #include <linux/scatterlist.h>
39 #include <scsi/scsi_cmnd.h>
40 #include <scsi/scsi_device.h>
41 #include <scsi/scsi_host.h>
42 #include <scsi/scsi.h>
43 #include <scsi/scsi_transport_iscsi.h>
45 #include "iscsi_tcp.h"
47 MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
48 "Alex Aizman <itn780@yahoo.com>");
49 MODULE_DESCRIPTION("iSCSI/TCP data-path");
50 MODULE_LICENSE("GPL");
55 #define debug_tcp(fmt...) printk(KERN_INFO "tcp: " fmt)
57 #define debug_tcp(fmt...)
67 static struct scsi_transport_template
*iscsi_tcp_scsi_transport
;
68 static struct scsi_host_template iscsi_sht
;
69 static struct iscsi_transport iscsi_tcp_transport
;
71 static unsigned int iscsi_max_lun
= 512;
72 module_param_named(max_lun
, iscsi_max_lun
, uint
, S_IRUGO
);
74 static int iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn
*tcp_conn
,
75 struct iscsi_segment
*segment
);
78 * Scatterlist handling: inside the iscsi_segment, we
79 * remember an index into the scatterlist, and set data/size
80 * to the current scatterlist entry. For highmem pages, we
83 * Note that the page is unmapped when we return from
84 * TCP's data_ready handler, so we may end up mapping and
85 * unmapping the same page repeatedly. The whole reason
86 * for this is that we shouldn't keep the page mapped
87 * outside the softirq.
91 * iscsi_tcp_segment_init_sg - init indicated scatterlist entry
92 * @segment: the buffer object
94 * @offset: byte offset into that sg entry
96 * This function sets up the segment so that subsequent
97 * data is copied to the indicated sg entry, at the given
101 iscsi_tcp_segment_init_sg(struct iscsi_segment
*segment
,
102 struct scatterlist
*sg
, unsigned int offset
)
105 segment
->sg_offset
= offset
;
106 segment
->size
= min(sg
->length
- offset
,
107 segment
->total_size
- segment
->total_copied
);
108 segment
->data
= NULL
;
112 * iscsi_tcp_segment_map - map the current S/G page
113 * @segment: iscsi_segment
114 * @recv: 1 if called from recv path
116 * We only need to possibly kmap data if scatter lists are being used,
117 * because the iscsi passthrough and internal IO paths will never use high
121 iscsi_tcp_segment_map(struct iscsi_segment
*segment
, int recv
)
123 struct scatterlist
*sg
;
125 if (segment
->data
!= NULL
|| !segment
->sg
)
129 BUG_ON(segment
->sg_mapped
);
130 BUG_ON(sg
->length
== 0);
133 * If the page count is greater than one it is ok to send
134 * to the network layer's zero copy send path. If not we
135 * have to go the slow sendmsg path. We always map for the
138 if (page_count(sg_page(sg
)) >= 1 && !recv
)
141 debug_tcp("iscsi_tcp_segment_map %s %p\n", recv
? "recv" : "xmit",
143 segment
->sg_mapped
= kmap_atomic(sg_page(sg
), KM_SOFTIRQ0
);
144 segment
->data
= segment
->sg_mapped
+ sg
->offset
+ segment
->sg_offset
;
148 iscsi_tcp_segment_unmap(struct iscsi_segment
*segment
)
150 debug_tcp("iscsi_tcp_segment_unmap %p\n", segment
);
152 if (segment
->sg_mapped
) {
153 debug_tcp("iscsi_tcp_segment_unmap valid\n");
154 kunmap_atomic(segment
->sg_mapped
, KM_SOFTIRQ0
);
155 segment
->sg_mapped
= NULL
;
156 segment
->data
= NULL
;
161 * Splice the digest buffer into the buffer
164 iscsi_tcp_segment_splice_digest(struct iscsi_segment
*segment
, void *digest
)
166 segment
->data
= digest
;
167 segment
->digest_len
= ISCSI_DIGEST_SIZE
;
168 segment
->total_size
+= ISCSI_DIGEST_SIZE
;
169 segment
->size
= ISCSI_DIGEST_SIZE
;
172 segment
->hash
= NULL
;
176 * iscsi_tcp_segment_done - check whether the segment is complete
177 * @segment: iscsi segment to check
178 * @recv: set to one of this is called from the recv path
179 * @copied: number of bytes copied
181 * Check if we're done receiving this segment. If the receive
182 * buffer is full but we expect more data, move on to the
183 * next entry in the scatterlist.
185 * If the amount of data we received isn't a multiple of 4,
186 * we will transparently receive the pad bytes, too.
188 * This function must be re-entrant.
191 iscsi_tcp_segment_done(struct iscsi_segment
*segment
, int recv
, unsigned copied
)
193 static unsigned char padbuf
[ISCSI_PAD_LEN
];
194 struct scatterlist sg
;
197 debug_tcp("copied %u %u size %u %s\n", segment
->copied
, copied
,
198 segment
->size
, recv
? "recv" : "xmit");
199 if (segment
->hash
&& copied
) {
201 * If a segment is kmapd we must unmap it before sending
202 * to the crypto layer since that will try to kmap it again.
204 iscsi_tcp_segment_unmap(segment
);
206 if (!segment
->data
) {
207 sg_init_table(&sg
, 1);
208 sg_set_page(&sg
, sg_page(segment
->sg
), copied
,
209 segment
->copied
+ segment
->sg_offset
+
210 segment
->sg
->offset
);
212 sg_init_one(&sg
, segment
->data
+ segment
->copied
,
214 crypto_hash_update(segment
->hash
, &sg
, copied
);
217 segment
->copied
+= copied
;
218 if (segment
->copied
< segment
->size
) {
219 iscsi_tcp_segment_map(segment
, recv
);
223 segment
->total_copied
+= segment
->copied
;
227 /* Unmap the current scatterlist page, if there is one. */
228 iscsi_tcp_segment_unmap(segment
);
230 /* Do we have more scatterlist entries? */
231 debug_tcp("total copied %u total size %u\n", segment
->total_copied
,
232 segment
->total_size
);
233 if (segment
->total_copied
< segment
->total_size
) {
234 /* Proceed to the next entry in the scatterlist. */
235 iscsi_tcp_segment_init_sg(segment
, sg_next(segment
->sg
),
237 iscsi_tcp_segment_map(segment
, recv
);
238 BUG_ON(segment
->size
== 0);
242 /* Do we need to handle padding? */
243 pad
= iscsi_padding(segment
->total_copied
);
245 debug_tcp("consume %d pad bytes\n", pad
);
246 segment
->total_size
+= pad
;
248 segment
->data
= padbuf
;
253 * Set us up for transferring the data digest. hdr digest
254 * is completely handled in hdr done function.
257 crypto_hash_final(segment
->hash
, segment
->digest
);
258 iscsi_tcp_segment_splice_digest(segment
,
259 recv
? segment
->recv_digest
: segment
->digest
);
267 * iscsi_tcp_xmit_segment - transmit segment
268 * @tcp_conn: the iSCSI TCP connection
269 * @segment: the buffer to transmnit
271 * This function transmits as much of the buffer as
272 * the network layer will accept, and returns the number of
275 * If CRC hashing is enabled, the function will compute the
276 * hash as it goes. When the entire segment has been transmitted,
277 * it will retrieve the hash value and send it as well.
280 iscsi_tcp_xmit_segment(struct iscsi_tcp_conn
*tcp_conn
,
281 struct iscsi_segment
*segment
)
283 struct socket
*sk
= tcp_conn
->sock
;
284 unsigned int copied
= 0;
287 while (!iscsi_tcp_segment_done(segment
, 0, r
)) {
288 struct scatterlist
*sg
;
289 unsigned int offset
, copy
;
293 offset
= segment
->copied
;
294 copy
= segment
->size
- offset
;
296 if (segment
->total_copied
+ segment
->size
< segment
->total_size
)
299 /* Use sendpage if we can; else fall back to sendmsg */
300 if (!segment
->data
) {
302 offset
+= segment
->sg_offset
+ sg
->offset
;
303 r
= tcp_conn
->sendpage(sk
, sg_page(sg
), offset
, copy
,
306 struct msghdr msg
= { .msg_flags
= flags
};
308 .iov_base
= segment
->data
+ offset
,
312 r
= kernel_sendmsg(sk
, &msg
, &iov
, 1, copy
);
316 iscsi_tcp_segment_unmap(segment
);
317 if (copied
|| r
== -EAGAIN
)
327 * iscsi_tcp_segment_recv - copy data to segment
328 * @tcp_conn: the iSCSI TCP connection
329 * @segment: the buffer to copy to
331 * @len: amount of data available
333 * This function copies up to @len bytes to the
334 * given buffer, and returns the number of bytes
335 * consumed, which can actually be less than @len.
337 * If hash digest is enabled, the function will update the
338 * hash while copying.
339 * Combining these two operations doesn't buy us a lot (yet),
340 * but in the future we could implement combined copy+crc,
341 * just way we do for network layer checksums.
344 iscsi_tcp_segment_recv(struct iscsi_tcp_conn
*tcp_conn
,
345 struct iscsi_segment
*segment
, const void *ptr
,
348 unsigned int copy
= 0, copied
= 0;
350 while (!iscsi_tcp_segment_done(segment
, 1, copy
)) {
352 debug_tcp("iscsi_tcp_segment_recv copied %d bytes\n",
357 copy
= min(len
- copied
, segment
->size
- segment
->copied
);
358 debug_tcp("iscsi_tcp_segment_recv copying %d\n", copy
);
359 memcpy(segment
->data
+ segment
->copied
, ptr
+ copied
, copy
);
366 iscsi_tcp_dgst_header(struct hash_desc
*hash
, const void *hdr
, size_t hdrlen
,
367 unsigned char digest
[ISCSI_DIGEST_SIZE
])
369 struct scatterlist sg
;
371 sg_init_one(&sg
, hdr
, hdrlen
);
372 crypto_hash_digest(hash
, &sg
, hdrlen
, digest
);
376 iscsi_tcp_dgst_verify(struct iscsi_tcp_conn
*tcp_conn
,
377 struct iscsi_segment
*segment
)
379 if (!segment
->digest_len
)
382 if (memcmp(segment
->recv_digest
, segment
->digest
,
383 segment
->digest_len
)) {
384 debug_scsi("digest mismatch\n");
392 * Helper function to set up segment buffer
395 __iscsi_segment_init(struct iscsi_segment
*segment
, size_t size
,
396 iscsi_segment_done_fn_t
*done
, struct hash_desc
*hash
)
398 memset(segment
, 0, sizeof(*segment
));
399 segment
->total_size
= size
;
400 segment
->done
= done
;
403 segment
->hash
= hash
;
404 crypto_hash_init(hash
);
409 iscsi_segment_init_linear(struct iscsi_segment
*segment
, void *data
,
410 size_t size
, iscsi_segment_done_fn_t
*done
,
411 struct hash_desc
*hash
)
413 __iscsi_segment_init(segment
, size
, done
, hash
);
414 segment
->data
= data
;
415 segment
->size
= size
;
419 iscsi_segment_seek_sg(struct iscsi_segment
*segment
,
420 struct scatterlist
*sg_list
, unsigned int sg_count
,
421 unsigned int offset
, size_t size
,
422 iscsi_segment_done_fn_t
*done
, struct hash_desc
*hash
)
424 struct scatterlist
*sg
;
427 debug_scsi("iscsi_segment_seek_sg offset %u size %llu\n",
429 __iscsi_segment_init(segment
, size
, done
, hash
);
430 for_each_sg(sg_list
, sg
, sg_count
, i
) {
431 debug_scsi("sg %d, len %u offset %u\n", i
, sg
->length
,
433 if (offset
< sg
->length
) {
434 iscsi_tcp_segment_init_sg(segment
, sg
, offset
);
437 offset
-= sg
->length
;
440 return ISCSI_ERR_DATA_OFFSET
;
444 * iscsi_tcp_hdr_recv_prep - prep segment for hdr reception
445 * @tcp_conn: iscsi connection to prep for
447 * This function always passes NULL for the hash argument, because when this
448 * function is called we do not yet know the final size of the header and want
449 * to delay the digest processing until we know that.
452 iscsi_tcp_hdr_recv_prep(struct iscsi_tcp_conn
*tcp_conn
)
454 debug_tcp("iscsi_tcp_hdr_recv_prep(%p%s)\n", tcp_conn
,
455 tcp_conn
->iscsi_conn
->hdrdgst_en
? ", digest enabled" : "");
456 iscsi_segment_init_linear(&tcp_conn
->in
.segment
,
457 tcp_conn
->in
.hdr_buf
, sizeof(struct iscsi_hdr
),
458 iscsi_tcp_hdr_recv_done
, NULL
);
462 * Handle incoming reply to any other type of command
465 iscsi_tcp_data_recv_done(struct iscsi_tcp_conn
*tcp_conn
,
466 struct iscsi_segment
*segment
)
468 struct iscsi_conn
*conn
= tcp_conn
->iscsi_conn
;
471 if (!iscsi_tcp_dgst_verify(tcp_conn
, segment
))
472 return ISCSI_ERR_DATA_DGST
;
474 rc
= iscsi_complete_pdu(conn
, tcp_conn
->in
.hdr
,
475 conn
->data
, tcp_conn
->in
.datalen
);
479 iscsi_tcp_hdr_recv_prep(tcp_conn
);
484 iscsi_tcp_data_recv_prep(struct iscsi_tcp_conn
*tcp_conn
)
486 struct iscsi_conn
*conn
= tcp_conn
->iscsi_conn
;
487 struct hash_desc
*rx_hash
= NULL
;
489 if (conn
->datadgst_en
)
490 rx_hash
= &tcp_conn
->rx_hash
;
492 iscsi_segment_init_linear(&tcp_conn
->in
.segment
,
493 conn
->data
, tcp_conn
->in
.datalen
,
494 iscsi_tcp_data_recv_done
, rx_hash
);
498 * must be called with session lock
501 iscsi_tcp_cleanup_ctask(struct iscsi_conn
*conn
, struct iscsi_cmd_task
*ctask
)
503 struct iscsi_tcp_cmd_task
*tcp_ctask
= ctask
->dd_data
;
504 struct iscsi_r2t_info
*r2t
;
506 /* flush ctask's r2t queues */
507 while (__kfifo_get(tcp_ctask
->r2tqueue
, (void*)&r2t
, sizeof(void*))) {
508 __kfifo_put(tcp_ctask
->r2tpool
.queue
, (void*)&r2t
,
510 debug_scsi("iscsi_tcp_cleanup_ctask pending r2t dropped\n");
513 r2t
= tcp_ctask
->r2t
;
515 __kfifo_put(tcp_ctask
->r2tpool
.queue
, (void*)&r2t
,
517 tcp_ctask
->r2t
= NULL
;
522 * iscsi_data_rsp - SCSI Data-In Response processing
523 * @conn: iscsi connection
524 * @ctask: scsi command task
527 iscsi_data_rsp(struct iscsi_conn
*conn
, struct iscsi_cmd_task
*ctask
)
529 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
530 struct iscsi_tcp_cmd_task
*tcp_ctask
= ctask
->dd_data
;
531 struct iscsi_data_rsp
*rhdr
= (struct iscsi_data_rsp
*)tcp_conn
->in
.hdr
;
532 struct iscsi_session
*session
= conn
->session
;
533 struct scsi_cmnd
*sc
= ctask
->sc
;
534 int datasn
= be32_to_cpu(rhdr
->datasn
);
535 unsigned total_in_length
= scsi_in(sc
)->length
;
537 iscsi_update_cmdsn(session
, (struct iscsi_nopin
*)rhdr
);
538 if (tcp_conn
->in
.datalen
== 0)
541 if (tcp_ctask
->exp_datasn
!= datasn
) {
542 debug_tcp("%s: ctask->exp_datasn(%d) != rhdr->datasn(%d)\n",
543 __FUNCTION__
, tcp_ctask
->exp_datasn
, datasn
);
544 return ISCSI_ERR_DATASN
;
547 tcp_ctask
->exp_datasn
++;
549 tcp_ctask
->data_offset
= be32_to_cpu(rhdr
->offset
);
550 if (tcp_ctask
->data_offset
+ tcp_conn
->in
.datalen
> total_in_length
) {
551 debug_tcp("%s: data_offset(%d) + data_len(%d) > total_length_in(%d)\n",
552 __FUNCTION__
, tcp_ctask
->data_offset
,
553 tcp_conn
->in
.datalen
, total_in_length
);
554 return ISCSI_ERR_DATA_OFFSET
;
557 if (rhdr
->flags
& ISCSI_FLAG_DATA_STATUS
) {
558 sc
->result
= (DID_OK
<< 16) | rhdr
->cmd_status
;
559 conn
->exp_statsn
= be32_to_cpu(rhdr
->statsn
) + 1;
560 if (rhdr
->flags
& (ISCSI_FLAG_DATA_UNDERFLOW
|
561 ISCSI_FLAG_DATA_OVERFLOW
)) {
562 int res_count
= be32_to_cpu(rhdr
->residual_count
);
565 (rhdr
->flags
& ISCSI_FLAG_CMD_OVERFLOW
||
566 res_count
<= total_in_length
))
567 scsi_in(sc
)->resid
= res_count
;
569 sc
->result
= (DID_BAD_TARGET
<< 16) |
574 conn
->datain_pdus_cnt
++;
579 * iscsi_solicit_data_init - initialize first Data-Out
580 * @conn: iscsi connection
581 * @ctask: scsi command task
585 * Initialize first Data-Out within this R2T sequence and finds
586 * proper data_offset within this SCSI command.
588 * This function is called with connection lock taken.
591 iscsi_solicit_data_init(struct iscsi_conn
*conn
, struct iscsi_cmd_task
*ctask
,
592 struct iscsi_r2t_info
*r2t
)
594 struct iscsi_data
*hdr
;
596 hdr
= &r2t
->dtask
.hdr
;
597 memset(hdr
, 0, sizeof(struct iscsi_data
));
599 hdr
->datasn
= cpu_to_be32(r2t
->solicit_datasn
);
600 r2t
->solicit_datasn
++;
601 hdr
->opcode
= ISCSI_OP_SCSI_DATA_OUT
;
602 memcpy(hdr
->lun
, ctask
->hdr
->lun
, sizeof(hdr
->lun
));
603 hdr
->itt
= ctask
->hdr
->itt
;
604 hdr
->exp_statsn
= r2t
->exp_statsn
;
605 hdr
->offset
= cpu_to_be32(r2t
->data_offset
);
606 if (r2t
->data_length
> conn
->max_xmit_dlength
) {
607 hton24(hdr
->dlength
, conn
->max_xmit_dlength
);
608 r2t
->data_count
= conn
->max_xmit_dlength
;
611 hton24(hdr
->dlength
, r2t
->data_length
);
612 r2t
->data_count
= r2t
->data_length
;
613 hdr
->flags
= ISCSI_FLAG_CMD_FINAL
;
615 conn
->dataout_pdus_cnt
++;
621 * iscsi_r2t_rsp - iSCSI R2T Response processing
622 * @conn: iscsi connection
623 * @ctask: scsi command task
626 iscsi_r2t_rsp(struct iscsi_conn
*conn
, struct iscsi_cmd_task
*ctask
)
628 struct iscsi_r2t_info
*r2t
;
629 struct iscsi_session
*session
= conn
->session
;
630 struct iscsi_tcp_cmd_task
*tcp_ctask
= ctask
->dd_data
;
631 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
632 struct iscsi_r2t_rsp
*rhdr
= (struct iscsi_r2t_rsp
*)tcp_conn
->in
.hdr
;
633 int r2tsn
= be32_to_cpu(rhdr
->r2tsn
);
636 if (tcp_conn
->in
.datalen
) {
637 iscsi_conn_printk(KERN_ERR
, conn
,
638 "invalid R2t with datalen %d\n",
639 tcp_conn
->in
.datalen
);
640 return ISCSI_ERR_DATALEN
;
643 if (tcp_ctask
->exp_datasn
!= r2tsn
){
644 debug_tcp("%s: ctask->exp_datasn(%d) != rhdr->r2tsn(%d)\n",
645 __FUNCTION__
, tcp_ctask
->exp_datasn
, r2tsn
);
646 return ISCSI_ERR_R2TSN
;
649 /* fill-in new R2T associated with the task */
650 iscsi_update_cmdsn(session
, (struct iscsi_nopin
*)rhdr
);
652 if (!ctask
->sc
|| session
->state
!= ISCSI_STATE_LOGGED_IN
) {
653 iscsi_conn_printk(KERN_INFO
, conn
,
654 "dropping R2T itt %d in recovery.\n",
659 rc
= __kfifo_get(tcp_ctask
->r2tpool
.queue
, (void*)&r2t
, sizeof(void*));
662 r2t
->exp_statsn
= rhdr
->statsn
;
663 r2t
->data_length
= be32_to_cpu(rhdr
->data_length
);
664 if (r2t
->data_length
== 0) {
665 iscsi_conn_printk(KERN_ERR
, conn
,
666 "invalid R2T with zero data len\n");
667 __kfifo_put(tcp_ctask
->r2tpool
.queue
, (void*)&r2t
,
669 return ISCSI_ERR_DATALEN
;
672 if (r2t
->data_length
> session
->max_burst
)
673 debug_scsi("invalid R2T with data len %u and max burst %u."
674 "Attempting to execute request.\n",
675 r2t
->data_length
, session
->max_burst
);
677 r2t
->data_offset
= be32_to_cpu(rhdr
->data_offset
);
678 if (r2t
->data_offset
+ r2t
->data_length
> scsi_out(ctask
->sc
)->length
) {
679 iscsi_conn_printk(KERN_ERR
, conn
,
680 "invalid R2T with data len %u at offset %u "
681 "and total length %d\n", r2t
->data_length
,
682 r2t
->data_offset
, scsi_out(ctask
->sc
)->length
);
683 __kfifo_put(tcp_ctask
->r2tpool
.queue
, (void*)&r2t
,
685 return ISCSI_ERR_DATALEN
;
688 r2t
->ttt
= rhdr
->ttt
; /* no flip */
689 r2t
->solicit_datasn
= 0;
691 iscsi_solicit_data_init(conn
, ctask
, r2t
);
693 tcp_ctask
->exp_datasn
= r2tsn
+ 1;
694 __kfifo_put(tcp_ctask
->r2tqueue
, (void*)&r2t
, sizeof(void*));
695 conn
->r2t_pdus_cnt
++;
697 iscsi_requeue_ctask(ctask
);
702 * Handle incoming reply to DataIn command
705 iscsi_tcp_process_data_in(struct iscsi_tcp_conn
*tcp_conn
,
706 struct iscsi_segment
*segment
)
708 struct iscsi_conn
*conn
= tcp_conn
->iscsi_conn
;
709 struct iscsi_hdr
*hdr
= tcp_conn
->in
.hdr
;
712 if (!iscsi_tcp_dgst_verify(tcp_conn
, segment
))
713 return ISCSI_ERR_DATA_DGST
;
715 /* check for non-exceptional status */
716 if (hdr
->flags
& ISCSI_FLAG_DATA_STATUS
) {
717 rc
= iscsi_complete_pdu(conn
, tcp_conn
->in
.hdr
, NULL
, 0);
722 iscsi_tcp_hdr_recv_prep(tcp_conn
);
727 * iscsi_tcp_hdr_dissect - process PDU header
728 * @conn: iSCSI connection
731 * This function analyzes the header of the PDU received,
732 * and performs several sanity checks. If the PDU is accompanied
733 * by data, the receive buffer is set up to copy the incoming data
734 * to the correct location.
737 iscsi_tcp_hdr_dissect(struct iscsi_conn
*conn
, struct iscsi_hdr
*hdr
)
739 int rc
= 0, opcode
, ahslen
;
740 struct iscsi_session
*session
= conn
->session
;
741 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
742 struct iscsi_cmd_task
*ctask
;
745 /* verify PDU length */
746 tcp_conn
->in
.datalen
= ntoh24(hdr
->dlength
);
747 if (tcp_conn
->in
.datalen
> conn
->max_recv_dlength
) {
748 iscsi_conn_printk(KERN_ERR
, conn
,
749 "iscsi_tcp: datalen %d > %d\n",
750 tcp_conn
->in
.datalen
, conn
->max_recv_dlength
);
751 return ISCSI_ERR_DATALEN
;
754 /* Additional header segments. So far, we don't
755 * process additional headers.
757 ahslen
= hdr
->hlength
<< 2;
759 opcode
= hdr
->opcode
& ISCSI_OPCODE_MASK
;
760 /* verify itt (itt encoding: age+cid+itt) */
761 rc
= iscsi_verify_itt(conn
, hdr
, &itt
);
765 debug_tcp("opcode 0x%x ahslen %d datalen %d\n",
766 opcode
, ahslen
, tcp_conn
->in
.datalen
);
769 case ISCSI_OP_SCSI_DATA_IN
:
770 ctask
= session
->cmds
[itt
];
771 spin_lock(&conn
->session
->lock
);
772 rc
= iscsi_data_rsp(conn
, ctask
);
773 spin_unlock(&conn
->session
->lock
);
776 if (tcp_conn
->in
.datalen
) {
777 struct iscsi_tcp_cmd_task
*tcp_ctask
= ctask
->dd_data
;
778 struct hash_desc
*rx_hash
= NULL
;
779 struct scsi_data_buffer
*sdb
= scsi_in(ctask
->sc
);
782 * Setup copy of Data-In into the Scsi_Cmnd
784 * We set up the iscsi_segment to point to the next
785 * scatterlist entry to copy to. As we go along,
786 * we move on to the next scatterlist entry and
787 * update the digest per-entry.
789 if (conn
->datadgst_en
)
790 rx_hash
= &tcp_conn
->rx_hash
;
792 debug_tcp("iscsi_tcp_begin_data_in(%p, offset=%d, "
793 "datalen=%d)\n", tcp_conn
,
794 tcp_ctask
->data_offset
,
795 tcp_conn
->in
.datalen
);
796 return iscsi_segment_seek_sg(&tcp_conn
->in
.segment
,
799 tcp_ctask
->data_offset
,
800 tcp_conn
->in
.datalen
,
801 iscsi_tcp_process_data_in
,
805 case ISCSI_OP_SCSI_CMD_RSP
:
806 if (tcp_conn
->in
.datalen
) {
807 iscsi_tcp_data_recv_prep(tcp_conn
);
810 rc
= iscsi_complete_pdu(conn
, hdr
, NULL
, 0);
813 ctask
= session
->cmds
[itt
];
815 rc
= ISCSI_ERR_AHSLEN
;
816 else if (ctask
->sc
->sc_data_direction
== DMA_TO_DEVICE
) {
817 spin_lock(&session
->lock
);
818 rc
= iscsi_r2t_rsp(conn
, ctask
);
819 spin_unlock(&session
->lock
);
821 rc
= ISCSI_ERR_PROTO
;
823 case ISCSI_OP_LOGIN_RSP
:
824 case ISCSI_OP_TEXT_RSP
:
825 case ISCSI_OP_REJECT
:
826 case ISCSI_OP_ASYNC_EVENT
:
828 * It is possible that we could get a PDU with a buffer larger
829 * than 8K, but there are no targets that currently do this.
830 * For now we fail until we find a vendor that needs it
832 if (ISCSI_DEF_MAX_RECV_SEG_LEN
< tcp_conn
->in
.datalen
) {
833 iscsi_conn_printk(KERN_ERR
, conn
,
834 "iscsi_tcp: received buffer of "
835 "len %u but conn buffer is only %u "
837 tcp_conn
->in
.datalen
,
838 ISCSI_DEF_MAX_RECV_SEG_LEN
, opcode
);
839 rc
= ISCSI_ERR_PROTO
;
843 /* If there's data coming in with the response,
844 * receive it to the connection's buffer.
846 if (tcp_conn
->in
.datalen
) {
847 iscsi_tcp_data_recv_prep(tcp_conn
);
851 case ISCSI_OP_LOGOUT_RSP
:
852 case ISCSI_OP_NOOP_IN
:
853 case ISCSI_OP_SCSI_TMFUNC_RSP
:
854 rc
= iscsi_complete_pdu(conn
, hdr
, NULL
, 0);
857 rc
= ISCSI_ERR_BAD_OPCODE
;
862 /* Anything that comes with data should have
863 * been handled above. */
864 if (tcp_conn
->in
.datalen
)
865 return ISCSI_ERR_PROTO
;
866 iscsi_tcp_hdr_recv_prep(tcp_conn
);
873 * iscsi_tcp_hdr_recv_done - process PDU header
875 * This is the callback invoked when the PDU header has
876 * been received. If the header is followed by additional
877 * header segments, we go back for more data.
880 iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn
*tcp_conn
,
881 struct iscsi_segment
*segment
)
883 struct iscsi_conn
*conn
= tcp_conn
->iscsi_conn
;
884 struct iscsi_hdr
*hdr
;
886 /* Check if there are additional header segments
887 * *prior* to computing the digest, because we
888 * may need to go back to the caller for more.
890 hdr
= (struct iscsi_hdr
*) tcp_conn
->in
.hdr_buf
;
891 if (segment
->copied
== sizeof(struct iscsi_hdr
) && hdr
->hlength
) {
892 /* Bump the header length - the caller will
893 * just loop around and get the AHS for us, and
895 unsigned int ahslen
= hdr
->hlength
<< 2;
897 /* Make sure we don't overflow */
898 if (sizeof(*hdr
) + ahslen
> sizeof(tcp_conn
->in
.hdr_buf
))
899 return ISCSI_ERR_AHSLEN
;
901 segment
->total_size
+= ahslen
;
902 segment
->size
+= ahslen
;
906 /* We're done processing the header. See if we're doing
907 * header digests; if so, set up the recv_digest buffer
908 * and go back for more. */
909 if (conn
->hdrdgst_en
) {
910 if (segment
->digest_len
== 0) {
911 iscsi_tcp_segment_splice_digest(segment
,
912 segment
->recv_digest
);
915 iscsi_tcp_dgst_header(&tcp_conn
->rx_hash
, hdr
,
916 segment
->total_copied
- ISCSI_DIGEST_SIZE
,
919 if (!iscsi_tcp_dgst_verify(tcp_conn
, segment
))
920 return ISCSI_ERR_HDR_DGST
;
923 tcp_conn
->in
.hdr
= hdr
;
924 return iscsi_tcp_hdr_dissect(conn
, hdr
);
928 * iscsi_tcp_recv - TCP receive in sendfile fashion
929 * @rd_desc: read descriptor
930 * @skb: socket buffer
931 * @offset: offset in skb
932 * @len: skb->len - offset
935 iscsi_tcp_recv(read_descriptor_t
*rd_desc
, struct sk_buff
*skb
,
936 unsigned int offset
, size_t len
)
938 struct iscsi_conn
*conn
= rd_desc
->arg
.data
;
939 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
940 struct iscsi_segment
*segment
= &tcp_conn
->in
.segment
;
941 struct skb_seq_state seq
;
942 unsigned int consumed
= 0;
945 debug_tcp("in %d bytes\n", skb
->len
- offset
);
947 if (unlikely(conn
->suspend_rx
)) {
948 debug_tcp("conn %d Rx suspended!\n", conn
->id
);
952 skb_prepare_seq_read(skb
, offset
, skb
->len
, &seq
);
957 avail
= skb_seq_read(consumed
, &ptr
, &seq
);
959 debug_tcp("no more data avail. Consumed %d\n",
963 BUG_ON(segment
->copied
>= segment
->size
);
965 debug_tcp("skb %p ptr=%p avail=%u\n", skb
, ptr
, avail
);
966 rc
= iscsi_tcp_segment_recv(tcp_conn
, segment
, ptr
, avail
);
970 if (segment
->total_copied
>= segment
->total_size
) {
971 debug_tcp("segment done\n");
972 rc
= segment
->done(tcp_conn
, segment
);
974 skb_abort_seq_read(&seq
);
978 /* The done() functions sets up the
982 skb_abort_seq_read(&seq
);
983 conn
->rxdata_octets
+= consumed
;
987 debug_tcp("Error receiving PDU, errno=%d\n", rc
);
988 iscsi_conn_failure(conn
, ISCSI_ERR_CONN_FAILED
);
993 iscsi_tcp_data_ready(struct sock
*sk
, int flag
)
995 struct iscsi_conn
*conn
= sk
->sk_user_data
;
996 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
997 read_descriptor_t rd_desc
;
999 read_lock(&sk
->sk_callback_lock
);
1002 * Use rd_desc to pass 'conn' to iscsi_tcp_recv.
1003 * We set count to 1 because we want the network layer to
1004 * hand us all the skbs that are available. iscsi_tcp_recv
1005 * handled pdus that cross buffers or pdus that still need data.
1007 rd_desc
.arg
.data
= conn
;
1009 tcp_read_sock(sk
, &rd_desc
, iscsi_tcp_recv
);
1011 read_unlock(&sk
->sk_callback_lock
);
1013 /* If we had to (atomically) map a highmem page,
1015 iscsi_tcp_segment_unmap(&tcp_conn
->in
.segment
);
1019 iscsi_tcp_state_change(struct sock
*sk
)
1021 struct iscsi_tcp_conn
*tcp_conn
;
1022 struct iscsi_conn
*conn
;
1023 struct iscsi_session
*session
;
1024 void (*old_state_change
)(struct sock
*);
1026 read_lock(&sk
->sk_callback_lock
);
1028 conn
= (struct iscsi_conn
*)sk
->sk_user_data
;
1029 session
= conn
->session
;
1031 if ((sk
->sk_state
== TCP_CLOSE_WAIT
||
1032 sk
->sk_state
== TCP_CLOSE
) &&
1033 !atomic_read(&sk
->sk_rmem_alloc
)) {
1034 debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n");
1035 iscsi_conn_failure(conn
, ISCSI_ERR_CONN_FAILED
);
1038 tcp_conn
= conn
->dd_data
;
1039 old_state_change
= tcp_conn
->old_state_change
;
1041 read_unlock(&sk
->sk_callback_lock
);
1043 old_state_change(sk
);
1047 * iscsi_write_space - Called when more output buffer space is available
1048 * @sk: socket space is available for
1051 iscsi_write_space(struct sock
*sk
)
1053 struct iscsi_conn
*conn
= (struct iscsi_conn
*)sk
->sk_user_data
;
1054 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1056 tcp_conn
->old_write_space(sk
);
1057 debug_tcp("iscsi_write_space: cid %d\n", conn
->id
);
1058 scsi_queue_work(conn
->session
->host
, &conn
->xmitwork
);
1062 iscsi_conn_set_callbacks(struct iscsi_conn
*conn
)
1064 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1065 struct sock
*sk
= tcp_conn
->sock
->sk
;
1067 /* assign new callbacks */
1068 write_lock_bh(&sk
->sk_callback_lock
);
1069 sk
->sk_user_data
= conn
;
1070 tcp_conn
->old_data_ready
= sk
->sk_data_ready
;
1071 tcp_conn
->old_state_change
= sk
->sk_state_change
;
1072 tcp_conn
->old_write_space
= sk
->sk_write_space
;
1073 sk
->sk_data_ready
= iscsi_tcp_data_ready
;
1074 sk
->sk_state_change
= iscsi_tcp_state_change
;
1075 sk
->sk_write_space
= iscsi_write_space
;
1076 write_unlock_bh(&sk
->sk_callback_lock
);
1080 iscsi_conn_restore_callbacks(struct iscsi_tcp_conn
*tcp_conn
)
1082 struct sock
*sk
= tcp_conn
->sock
->sk
;
1084 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
1085 write_lock_bh(&sk
->sk_callback_lock
);
1086 sk
->sk_user_data
= NULL
;
1087 sk
->sk_data_ready
= tcp_conn
->old_data_ready
;
1088 sk
->sk_state_change
= tcp_conn
->old_state_change
;
1089 sk
->sk_write_space
= tcp_conn
->old_write_space
;
1090 sk
->sk_no_check
= 0;
1091 write_unlock_bh(&sk
->sk_callback_lock
);
1095 * iscsi_xmit - TCP transmit
1098 iscsi_xmit(struct iscsi_conn
*conn
)
1100 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1101 struct iscsi_segment
*segment
= &tcp_conn
->out
.segment
;
1102 unsigned int consumed
= 0;
1106 rc
= iscsi_tcp_xmit_segment(tcp_conn
, segment
);
1114 if (segment
->total_copied
>= segment
->total_size
) {
1115 if (segment
->done
!= NULL
) {
1116 rc
= segment
->done(tcp_conn
, segment
);
1123 debug_tcp("xmit %d bytes\n", consumed
);
1125 conn
->txdata_octets
+= consumed
;
1129 /* Transmit error. We could initiate error recovery
1131 debug_tcp("Error sending PDU, errno=%d\n", rc
);
1132 iscsi_conn_failure(conn
, ISCSI_ERR_CONN_FAILED
);
1137 * iscsi_tcp_xmit_qlen - return the number of bytes queued for xmit
1140 iscsi_tcp_xmit_qlen(struct iscsi_conn
*conn
)
1142 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1143 struct iscsi_segment
*segment
= &tcp_conn
->out
.segment
;
1145 return segment
->total_copied
- segment
->total_size
;
1149 iscsi_tcp_flush(struct iscsi_conn
*conn
)
1153 while (iscsi_tcp_xmit_qlen(conn
)) {
1154 rc
= iscsi_xmit(conn
);
1165 * This is called when we're done sending the header.
1166 * Simply copy the data_segment to the send segment, and return.
1169 iscsi_tcp_send_hdr_done(struct iscsi_tcp_conn
*tcp_conn
,
1170 struct iscsi_segment
*segment
)
1172 tcp_conn
->out
.segment
= tcp_conn
->out
.data_segment
;
1173 debug_tcp("Header done. Next segment size %u total_size %u\n",
1174 tcp_conn
->out
.segment
.size
, tcp_conn
->out
.segment
.total_size
);
1179 iscsi_tcp_send_hdr_prep(struct iscsi_conn
*conn
, void *hdr
, size_t hdrlen
)
1181 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1183 debug_tcp("%s(%p%s)\n", __FUNCTION__
, tcp_conn
,
1184 conn
->hdrdgst_en
? ", digest enabled" : "");
1186 /* Clear the data segment - needs to be filled in by the
1187 * caller using iscsi_tcp_send_data_prep() */
1188 memset(&tcp_conn
->out
.data_segment
, 0, sizeof(struct iscsi_segment
));
1190 /* If header digest is enabled, compute the CRC and
1191 * place the digest into the same buffer. We make
1192 * sure that both iscsi_tcp_ctask and mtask have
1195 if (conn
->hdrdgst_en
) {
1196 iscsi_tcp_dgst_header(&tcp_conn
->tx_hash
, hdr
, hdrlen
,
1198 hdrlen
+= ISCSI_DIGEST_SIZE
;
1201 /* Remember header pointer for later, when we need
1202 * to decide whether there's a payload to go along
1203 * with the header. */
1204 tcp_conn
->out
.hdr
= hdr
;
1206 iscsi_segment_init_linear(&tcp_conn
->out
.segment
, hdr
, hdrlen
,
1207 iscsi_tcp_send_hdr_done
, NULL
);
1211 * Prepare the send buffer for the payload data.
1212 * Padding and checksumming will all be taken care
1213 * of by the iscsi_segment routines.
1216 iscsi_tcp_send_data_prep(struct iscsi_conn
*conn
, struct scatterlist
*sg
,
1217 unsigned int count
, unsigned int offset
,
1220 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1221 struct hash_desc
*tx_hash
= NULL
;
1222 unsigned int hdr_spec_len
;
1224 debug_tcp("%s(%p, offset=%d, datalen=%d%s)\n", __FUNCTION__
,
1225 tcp_conn
, offset
, len
,
1226 conn
->datadgst_en
? ", digest enabled" : "");
1228 /* Make sure the datalen matches what the caller
1229 said he would send. */
1230 hdr_spec_len
= ntoh24(tcp_conn
->out
.hdr
->dlength
);
1231 WARN_ON(iscsi_padded(len
) != iscsi_padded(hdr_spec_len
));
1233 if (conn
->datadgst_en
)
1234 tx_hash
= &tcp_conn
->tx_hash
;
1236 return iscsi_segment_seek_sg(&tcp_conn
->out
.data_segment
,
1237 sg
, count
, offset
, len
,
1242 iscsi_tcp_send_linear_data_prepare(struct iscsi_conn
*conn
, void *data
,
1245 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1246 struct hash_desc
*tx_hash
= NULL
;
1247 unsigned int hdr_spec_len
;
1249 debug_tcp("%s(%p, datalen=%d%s)\n", __FUNCTION__
, tcp_conn
, len
,
1250 conn
->datadgst_en
? ", digest enabled" : "");
1252 /* Make sure the datalen matches what the caller
1253 said he would send. */
1254 hdr_spec_len
= ntoh24(tcp_conn
->out
.hdr
->dlength
);
1255 WARN_ON(iscsi_padded(len
) != iscsi_padded(hdr_spec_len
));
1257 if (conn
->datadgst_en
)
1258 tx_hash
= &tcp_conn
->tx_hash
;
1260 iscsi_segment_init_linear(&tcp_conn
->out
.data_segment
,
1261 data
, len
, NULL
, tx_hash
);
1265 * iscsi_solicit_data_cont - initialize next Data-Out
1266 * @conn: iscsi connection
1267 * @ctask: scsi command task
1269 * @left: bytes left to transfer
1272 * Initialize next Data-Out within this R2T sequence and continue
1273 * to process next Scatter-Gather element(if any) of this SCSI command.
1275 * Called under connection lock.
1278 iscsi_solicit_data_cont(struct iscsi_conn
*conn
, struct iscsi_cmd_task
*ctask
,
1279 struct iscsi_r2t_info
*r2t
)
1281 struct iscsi_data
*hdr
;
1282 int new_offset
, left
;
1284 BUG_ON(r2t
->data_length
- r2t
->sent
< 0);
1285 left
= r2t
->data_length
- r2t
->sent
;
1289 hdr
= &r2t
->dtask
.hdr
;
1290 memset(hdr
, 0, sizeof(struct iscsi_data
));
1291 hdr
->ttt
= r2t
->ttt
;
1292 hdr
->datasn
= cpu_to_be32(r2t
->solicit_datasn
);
1293 r2t
->solicit_datasn
++;
1294 hdr
->opcode
= ISCSI_OP_SCSI_DATA_OUT
;
1295 memcpy(hdr
->lun
, ctask
->hdr
->lun
, sizeof(hdr
->lun
));
1296 hdr
->itt
= ctask
->hdr
->itt
;
1297 hdr
->exp_statsn
= r2t
->exp_statsn
;
1298 new_offset
= r2t
->data_offset
+ r2t
->sent
;
1299 hdr
->offset
= cpu_to_be32(new_offset
);
1300 if (left
> conn
->max_xmit_dlength
) {
1301 hton24(hdr
->dlength
, conn
->max_xmit_dlength
);
1302 r2t
->data_count
= conn
->max_xmit_dlength
;
1304 hton24(hdr
->dlength
, left
);
1305 r2t
->data_count
= left
;
1306 hdr
->flags
= ISCSI_FLAG_CMD_FINAL
;
1309 conn
->dataout_pdus_cnt
++;
1314 * iscsi_tcp_ctask - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
1315 * @conn: iscsi connection
1316 * @ctask: scsi command task
1320 iscsi_tcp_ctask_init(struct iscsi_cmd_task
*ctask
)
1322 struct iscsi_tcp_cmd_task
*tcp_ctask
= ctask
->dd_data
;
1323 struct iscsi_conn
*conn
= ctask
->conn
;
1324 struct scsi_cmnd
*sc
= ctask
->sc
;
1327 BUG_ON(__kfifo_len(tcp_ctask
->r2tqueue
));
1328 tcp_ctask
->sent
= 0;
1329 tcp_ctask
->exp_datasn
= 0;
1331 /* Prepare PDU, optionally w/ immediate data */
1332 debug_scsi("ctask deq [cid %d itt 0x%x imm %d unsol %d]\n",
1333 conn
->id
, ctask
->itt
, ctask
->imm_count
,
1334 ctask
->unsol_count
);
1335 iscsi_tcp_send_hdr_prep(conn
, ctask
->hdr
, ctask
->hdr_len
);
1337 if (!ctask
->imm_count
)
1340 /* If we have immediate data, attach a payload */
1341 err
= iscsi_tcp_send_data_prep(conn
, scsi_out(sc
)->table
.sgl
,
1342 scsi_out(sc
)->table
.nents
,
1343 0, ctask
->imm_count
);
1346 tcp_ctask
->sent
+= ctask
->imm_count
;
1347 ctask
->imm_count
= 0;
1352 * iscsi_tcp_mtask_xmit - xmit management(immediate) task
1353 * @conn: iscsi connection
1354 * @mtask: task management task
1357 * The function can return -EAGAIN in which case caller must
1358 * call it again later, or recover. '0' return code means successful
1362 iscsi_tcp_mtask_xmit(struct iscsi_conn
*conn
, struct iscsi_mgmt_task
*mtask
)
1366 /* Flush any pending data first. */
1367 rc
= iscsi_tcp_flush(conn
);
1371 if (mtask
->hdr
->itt
== RESERVED_ITT
) {
1372 struct iscsi_session
*session
= conn
->session
;
1374 spin_lock_bh(&session
->lock
);
1375 iscsi_free_mgmt_task(conn
, mtask
);
1376 spin_unlock_bh(&session
->lock
);
1383 * iscsi_tcp_ctask_xmit - xmit normal PDU task
1384 * @conn: iscsi connection
1385 * @ctask: iscsi command task
1387 * We're expected to return 0 when everything was transmitted succesfully,
1388 * -EAGAIN if there's still data in the queue, or != 0 for any other kind
1392 iscsi_tcp_ctask_xmit(struct iscsi_conn
*conn
, struct iscsi_cmd_task
*ctask
)
1394 struct iscsi_tcp_cmd_task
*tcp_ctask
= ctask
->dd_data
;
1395 struct scsi_cmnd
*sc
= ctask
->sc
;
1396 struct scsi_data_buffer
*sdb
= scsi_out(sc
);
1400 /* Flush any pending data first. */
1401 rc
= iscsi_tcp_flush(conn
);
1405 /* Are we done already? */
1406 if (sc
->sc_data_direction
!= DMA_TO_DEVICE
)
1409 if (ctask
->unsol_count
!= 0) {
1410 struct iscsi_data
*hdr
= &tcp_ctask
->unsol_dtask
.hdr
;
1412 /* Prepare a header for the unsolicited PDU.
1413 * The amount of data we want to send will be
1414 * in ctask->data_count.
1415 * FIXME: return the data count instead.
1417 iscsi_prep_unsolicit_data_pdu(ctask
, hdr
);
1419 debug_tcp("unsol dout [itt 0x%x doff %d dlen %d]\n",
1420 ctask
->itt
, tcp_ctask
->sent
, ctask
->data_count
);
1422 iscsi_tcp_send_hdr_prep(conn
, hdr
, sizeof(*hdr
));
1423 rc
= iscsi_tcp_send_data_prep(conn
, sdb
->table
.sgl
,
1424 sdb
->table
.nents
, tcp_ctask
->sent
,
1428 tcp_ctask
->sent
+= ctask
->data_count
;
1429 ctask
->unsol_count
-= ctask
->data_count
;
1432 struct iscsi_session
*session
= conn
->session
;
1433 struct iscsi_r2t_info
*r2t
;
1435 /* All unsolicited PDUs sent. Check for solicited PDUs.
1437 spin_lock_bh(&session
->lock
);
1438 r2t
= tcp_ctask
->r2t
;
1440 /* Continue with this R2T? */
1441 if (!iscsi_solicit_data_cont(conn
, ctask
, r2t
)) {
1442 debug_scsi(" done with r2t %p\n", r2t
);
1444 __kfifo_put(tcp_ctask
->r2tpool
.queue
,
1445 (void*)&r2t
, sizeof(void*));
1446 tcp_ctask
->r2t
= r2t
= NULL
;
1451 __kfifo_get(tcp_ctask
->r2tqueue
, (void*)&tcp_ctask
->r2t
,
1453 r2t
= tcp_ctask
->r2t
;
1455 spin_unlock_bh(&session
->lock
);
1457 /* Waiting for more R2Ts to arrive. */
1459 debug_tcp("no R2Ts yet\n");
1463 debug_scsi("sol dout %p [dsn %d itt 0x%x doff %d dlen %d]\n",
1464 r2t
, r2t
->solicit_datasn
- 1, ctask
->itt
,
1465 r2t
->data_offset
+ r2t
->sent
, r2t
->data_count
);
1467 iscsi_tcp_send_hdr_prep(conn
, &r2t
->dtask
.hdr
,
1468 sizeof(struct iscsi_hdr
));
1470 rc
= iscsi_tcp_send_data_prep(conn
, sdb
->table
.sgl
,
1472 r2t
->data_offset
+ r2t
->sent
,
1476 tcp_ctask
->sent
+= r2t
->data_count
;
1477 r2t
->sent
+= r2t
->data_count
;
1482 iscsi_conn_failure(conn
, rc
);
1486 static struct iscsi_cls_conn
*
1487 iscsi_tcp_conn_create(struct iscsi_cls_session
*cls_session
, uint32_t conn_idx
)
1489 struct iscsi_conn
*conn
;
1490 struct iscsi_cls_conn
*cls_conn
;
1491 struct iscsi_tcp_conn
*tcp_conn
;
1493 cls_conn
= iscsi_conn_setup(cls_session
, conn_idx
);
1496 conn
= cls_conn
->dd_data
;
1498 * due to strange issues with iser these are not set
1499 * in iscsi_conn_setup
1501 conn
->max_recv_dlength
= ISCSI_DEF_MAX_RECV_SEG_LEN
;
1503 tcp_conn
= kzalloc(sizeof(*tcp_conn
), GFP_KERNEL
);
1505 goto tcp_conn_alloc_fail
;
1507 conn
->dd_data
= tcp_conn
;
1508 tcp_conn
->iscsi_conn
= conn
;
1510 tcp_conn
->tx_hash
.tfm
= crypto_alloc_hash("crc32c", 0,
1512 tcp_conn
->tx_hash
.flags
= 0;
1513 if (IS_ERR(tcp_conn
->tx_hash
.tfm
))
1516 tcp_conn
->rx_hash
.tfm
= crypto_alloc_hash("crc32c", 0,
1518 tcp_conn
->rx_hash
.flags
= 0;
1519 if (IS_ERR(tcp_conn
->rx_hash
.tfm
))
1525 crypto_free_hash(tcp_conn
->tx_hash
.tfm
);
1527 iscsi_conn_printk(KERN_ERR
, conn
,
1528 "Could not create connection due to crc32c "
1529 "loading error. Make sure the crc32c "
1530 "module is built as a module or into the "
1533 tcp_conn_alloc_fail
:
1534 iscsi_conn_teardown(cls_conn
);
1539 iscsi_tcp_release_conn(struct iscsi_conn
*conn
)
1541 struct iscsi_session
*session
= conn
->session
;
1542 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1543 struct socket
*sock
= tcp_conn
->sock
;
1548 sock_hold(sock
->sk
);
1549 iscsi_conn_restore_callbacks(tcp_conn
);
1552 spin_lock_bh(&session
->lock
);
1553 tcp_conn
->sock
= NULL
;
1554 conn
->recv_lock
= NULL
;
1555 spin_unlock_bh(&session
->lock
);
1560 iscsi_tcp_conn_destroy(struct iscsi_cls_conn
*cls_conn
)
1562 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
1563 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1565 iscsi_tcp_release_conn(conn
);
1566 iscsi_conn_teardown(cls_conn
);
1568 if (tcp_conn
->tx_hash
.tfm
)
1569 crypto_free_hash(tcp_conn
->tx_hash
.tfm
);
1570 if (tcp_conn
->rx_hash
.tfm
)
1571 crypto_free_hash(tcp_conn
->rx_hash
.tfm
);
1577 iscsi_tcp_conn_stop(struct iscsi_cls_conn
*cls_conn
, int flag
)
1579 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
1581 iscsi_conn_stop(cls_conn
, flag
);
1582 iscsi_tcp_release_conn(conn
);
1585 static int iscsi_tcp_get_addr(struct iscsi_conn
*conn
, struct socket
*sock
,
1586 char *buf
, int *port
,
1587 int (*getname
)(struct socket
*, struct sockaddr
*,
1590 struct sockaddr_storage
*addr
;
1591 struct sockaddr_in6
*sin6
;
1592 struct sockaddr_in
*sin
;
1595 addr
= kmalloc(sizeof(*addr
), GFP_KERNEL
);
1599 if (getname(sock
, (struct sockaddr
*) addr
, &len
)) {
1604 switch (addr
->ss_family
) {
1606 sin
= (struct sockaddr_in
*)addr
;
1607 spin_lock_bh(&conn
->session
->lock
);
1608 sprintf(buf
, NIPQUAD_FMT
, NIPQUAD(sin
->sin_addr
.s_addr
));
1609 *port
= be16_to_cpu(sin
->sin_port
);
1610 spin_unlock_bh(&conn
->session
->lock
);
1613 sin6
= (struct sockaddr_in6
*)addr
;
1614 spin_lock_bh(&conn
->session
->lock
);
1615 sprintf(buf
, NIP6_FMT
, NIP6(sin6
->sin6_addr
));
1616 *port
= be16_to_cpu(sin6
->sin6_port
);
1617 spin_unlock_bh(&conn
->session
->lock
);
1626 iscsi_tcp_conn_bind(struct iscsi_cls_session
*cls_session
,
1627 struct iscsi_cls_conn
*cls_conn
, uint64_t transport_eph
,
1630 struct Scsi_Host
*shost
= iscsi_session_to_shost(cls_session
);
1631 struct iscsi_host
*ihost
= shost_priv(shost
);
1632 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
1633 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1635 struct socket
*sock
;
1638 /* lookup for existing socket */
1639 sock
= sockfd_lookup((int)transport_eph
, &err
);
1641 iscsi_conn_printk(KERN_ERR
, conn
,
1642 "sockfd_lookup failed %d\n", err
);
1646 * copy these values now because if we drop the session
1647 * userspace may still want to query the values since we will
1648 * be using them for the reconnect
1650 err
= iscsi_tcp_get_addr(conn
, sock
, conn
->portal_address
,
1651 &conn
->portal_port
, kernel_getpeername
);
1655 err
= iscsi_tcp_get_addr(conn
, sock
, ihost
->local_address
,
1656 &ihost
->local_port
, kernel_getsockname
);
1660 err
= iscsi_conn_bind(cls_session
, cls_conn
, is_leading
);
1664 /* bind iSCSI connection and socket */
1665 tcp_conn
->sock
= sock
;
1667 /* setup Socket parameters */
1670 sk
->sk_sndtimeo
= 15 * HZ
; /* FIXME: make it configurable */
1671 sk
->sk_allocation
= GFP_ATOMIC
;
1673 /* FIXME: disable Nagle's algorithm */
1676 * Intercept TCP callbacks for sendfile like receive
1679 conn
->recv_lock
= &sk
->sk_callback_lock
;
1680 iscsi_conn_set_callbacks(conn
);
1681 tcp_conn
->sendpage
= tcp_conn
->sock
->ops
->sendpage
;
1683 * set receive state machine into initial state
1685 iscsi_tcp_hdr_recv_prep(tcp_conn
);
1693 /* called with host lock */
1695 iscsi_tcp_mtask_init(struct iscsi_conn
*conn
, struct iscsi_mgmt_task
*mtask
)
1697 debug_scsi("mtask deq [cid %d itt 0x%x]\n", conn
->id
, mtask
->itt
);
1699 /* Prepare PDU, optionally w/ immediate data */
1700 iscsi_tcp_send_hdr_prep(conn
, mtask
->hdr
, sizeof(*mtask
->hdr
));
1702 /* If we have immediate data, attach a payload */
1703 if (mtask
->data_count
)
1704 iscsi_tcp_send_linear_data_prepare(conn
, mtask
->data
,
1709 iscsi_r2tpool_alloc(struct iscsi_session
*session
)
1715 * initialize per-task: R2T pool and xmit queue
1717 for (cmd_i
= 0; cmd_i
< session
->cmds_max
; cmd_i
++) {
1718 struct iscsi_cmd_task
*ctask
= session
->cmds
[cmd_i
];
1719 struct iscsi_tcp_cmd_task
*tcp_ctask
= ctask
->dd_data
;
1722 * pre-allocated x4 as much r2ts to handle race when
1723 * target acks DataOut faster than we data_xmit() queues
1724 * could replenish r2tqueue.
1728 if (iscsi_pool_init(&tcp_ctask
->r2tpool
, session
->max_r2t
* 4, NULL
,
1729 sizeof(struct iscsi_r2t_info
))) {
1730 goto r2t_alloc_fail
;
1733 /* R2T xmit queue */
1734 tcp_ctask
->r2tqueue
= kfifo_alloc(
1735 session
->max_r2t
* 4 * sizeof(void*), GFP_KERNEL
, NULL
);
1736 if (tcp_ctask
->r2tqueue
== ERR_PTR(-ENOMEM
)) {
1737 iscsi_pool_free(&tcp_ctask
->r2tpool
);
1738 goto r2t_alloc_fail
;
1745 for (i
= 0; i
< cmd_i
; i
++) {
1746 struct iscsi_cmd_task
*ctask
= session
->cmds
[i
];
1747 struct iscsi_tcp_cmd_task
*tcp_ctask
= ctask
->dd_data
;
1749 kfifo_free(tcp_ctask
->r2tqueue
);
1750 iscsi_pool_free(&tcp_ctask
->r2tpool
);
1756 iscsi_r2tpool_free(struct iscsi_session
*session
)
1760 for (i
= 0; i
< session
->cmds_max
; i
++) {
1761 struct iscsi_cmd_task
*ctask
= session
->cmds
[i
];
1762 struct iscsi_tcp_cmd_task
*tcp_ctask
= ctask
->dd_data
;
1764 kfifo_free(tcp_ctask
->r2tqueue
);
1765 iscsi_pool_free(&tcp_ctask
->r2tpool
);
1770 iscsi_conn_set_param(struct iscsi_cls_conn
*cls_conn
, enum iscsi_param param
,
1771 char *buf
, int buflen
)
1773 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
1774 struct iscsi_session
*session
= conn
->session
;
1775 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1779 case ISCSI_PARAM_HDRDGST_EN
:
1780 iscsi_set_param(cls_conn
, param
, buf
, buflen
);
1782 case ISCSI_PARAM_DATADGST_EN
:
1783 iscsi_set_param(cls_conn
, param
, buf
, buflen
);
1784 tcp_conn
->sendpage
= conn
->datadgst_en
?
1785 sock_no_sendpage
: tcp_conn
->sock
->ops
->sendpage
;
1787 case ISCSI_PARAM_MAX_R2T
:
1788 sscanf(buf
, "%d", &value
);
1789 if (value
<= 0 || !is_power_of_2(value
))
1791 if (session
->max_r2t
== value
)
1793 iscsi_r2tpool_free(session
);
1794 iscsi_set_param(cls_conn
, param
, buf
, buflen
);
1795 if (iscsi_r2tpool_alloc(session
))
1799 return iscsi_set_param(cls_conn
, param
, buf
, buflen
);
1806 iscsi_tcp_conn_get_param(struct iscsi_cls_conn
*cls_conn
,
1807 enum iscsi_param param
, char *buf
)
1809 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
1813 case ISCSI_PARAM_CONN_PORT
:
1814 spin_lock_bh(&conn
->session
->lock
);
1815 len
= sprintf(buf
, "%hu\n", conn
->portal_port
);
1816 spin_unlock_bh(&conn
->session
->lock
);
1818 case ISCSI_PARAM_CONN_ADDRESS
:
1819 spin_lock_bh(&conn
->session
->lock
);
1820 len
= sprintf(buf
, "%s\n", conn
->portal_address
);
1821 spin_unlock_bh(&conn
->session
->lock
);
1824 return iscsi_conn_get_param(cls_conn
, param
, buf
);
1831 iscsi_conn_get_stats(struct iscsi_cls_conn
*cls_conn
, struct iscsi_stats
*stats
)
1833 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
1834 struct iscsi_tcp_conn
*tcp_conn
= conn
->dd_data
;
1836 stats
->txdata_octets
= conn
->txdata_octets
;
1837 stats
->rxdata_octets
= conn
->rxdata_octets
;
1838 stats
->scsicmd_pdus
= conn
->scsicmd_pdus_cnt
;
1839 stats
->dataout_pdus
= conn
->dataout_pdus_cnt
;
1840 stats
->scsirsp_pdus
= conn
->scsirsp_pdus_cnt
;
1841 stats
->datain_pdus
= conn
->datain_pdus_cnt
;
1842 stats
->r2t_pdus
= conn
->r2t_pdus_cnt
;
1843 stats
->tmfcmd_pdus
= conn
->tmfcmd_pdus_cnt
;
1844 stats
->tmfrsp_pdus
= conn
->tmfrsp_pdus_cnt
;
1845 stats
->custom_length
= 3;
1846 strcpy(stats
->custom
[0].desc
, "tx_sendpage_failures");
1847 stats
->custom
[0].value
= tcp_conn
->sendpage_failures_cnt
;
1848 strcpy(stats
->custom
[1].desc
, "rx_discontiguous_hdr");
1849 stats
->custom
[1].value
= tcp_conn
->discontiguous_hdr_cnt
;
1850 strcpy(stats
->custom
[2].desc
, "eh_abort_cnt");
1851 stats
->custom
[2].value
= conn
->eh_abort_cnt
;
1854 static struct iscsi_cls_session
*
1855 iscsi_tcp_session_create(struct Scsi_Host
*shost
, uint16_t cmds_max
,
1856 uint16_t qdepth
, uint32_t initial_cmdsn
,
1859 struct iscsi_cls_session
*cls_session
;
1860 struct iscsi_session
*session
;
1864 printk(KERN_ERR
"iscsi_tcp: invalid shost %d.\n",
1869 shost
= iscsi_host_alloc(&iscsi_sht
, 0, qdepth
);
1872 shost
->transportt
= iscsi_tcp_scsi_transport
;
1873 shost
->max_lun
= iscsi_max_lun
;
1875 shost
->max_channel
= 0;
1876 shost
->max_cmd_len
= 16;
1877 shost
->can_queue
= cmds_max
;
1879 if (iscsi_host_add(shost
, NULL
))
1881 *hostno
= shost
->host_no
;
1883 cls_session
= iscsi_session_setup(&iscsi_tcp_transport
, shost
, cmds_max
,
1884 sizeof(struct iscsi_tcp_cmd_task
),
1885 sizeof(struct iscsi_tcp_mgmt_task
),
1889 session
= cls_session
->dd_data
;
1891 shost
->can_queue
= session
->cmds_max
;
1892 for (cmd_i
= 0; cmd_i
< session
->cmds_max
; cmd_i
++) {
1893 struct iscsi_cmd_task
*ctask
= session
->cmds
[cmd_i
];
1894 struct iscsi_tcp_cmd_task
*tcp_ctask
= ctask
->dd_data
;
1896 ctask
->hdr
= &tcp_ctask
->hdr
.cmd_hdr
;
1897 ctask
->hdr_max
= sizeof(tcp_ctask
->hdr
) - ISCSI_DIGEST_SIZE
;
1900 for (cmd_i
= 0; cmd_i
< session
->mgmtpool_max
; cmd_i
++) {
1901 struct iscsi_mgmt_task
*mtask
= session
->mgmt_cmds
[cmd_i
];
1902 struct iscsi_tcp_mgmt_task
*tcp_mtask
= mtask
->dd_data
;
1904 mtask
->hdr
= (struct iscsi_hdr
*) &tcp_mtask
->hdr
;
1907 if (iscsi_r2tpool_alloc(session
))
1908 goto remove_session
;
1912 iscsi_session_teardown(cls_session
);
1914 iscsi_host_remove(shost
);
1916 iscsi_host_free(shost
);
1920 static void iscsi_tcp_session_destroy(struct iscsi_cls_session
*cls_session
)
1922 struct Scsi_Host
*shost
= iscsi_session_to_shost(cls_session
);
1924 iscsi_r2tpool_free(cls_session
->dd_data
);
1926 iscsi_host_remove(shost
);
1927 iscsi_host_free(shost
);
1930 static int iscsi_tcp_slave_configure(struct scsi_device
*sdev
)
1932 blk_queue_bounce_limit(sdev
->request_queue
, BLK_BOUNCE_ANY
);
1933 blk_queue_dma_alignment(sdev
->request_queue
, 0);
1937 static struct scsi_host_template iscsi_sht
= {
1938 .module
= THIS_MODULE
,
1939 .name
= "iSCSI Initiator over TCP/IP",
1940 .queuecommand
= iscsi_queuecommand
,
1941 .change_queue_depth
= iscsi_change_queue_depth
,
1942 .can_queue
= ISCSI_DEF_XMIT_CMDS_MAX
- 1,
1943 .sg_tablesize
= 4096,
1944 .max_sectors
= 0xFFFF,
1945 .cmd_per_lun
= ISCSI_DEF_CMD_PER_LUN
,
1946 .eh_abort_handler
= iscsi_eh_abort
,
1947 .eh_device_reset_handler
= iscsi_eh_device_reset
,
1948 .eh_host_reset_handler
= iscsi_eh_host_reset
,
1949 .use_clustering
= DISABLE_CLUSTERING
,
1950 .slave_configure
= iscsi_tcp_slave_configure
,
1951 .proc_name
= "iscsi_tcp",
1955 static struct iscsi_transport iscsi_tcp_transport
= {
1956 .owner
= THIS_MODULE
,
1958 .caps
= CAP_RECOVERY_L0
| CAP_MULTI_R2T
| CAP_HDRDGST
1960 .param_mask
= ISCSI_MAX_RECV_DLENGTH
|
1961 ISCSI_MAX_XMIT_DLENGTH
|
1964 ISCSI_INITIAL_R2T_EN
|
1969 ISCSI_PDU_INORDER_EN
|
1970 ISCSI_DATASEQ_INORDER_EN
|
1973 ISCSI_CONN_ADDRESS
|
1975 ISCSI_PERSISTENT_PORT
|
1976 ISCSI_PERSISTENT_ADDRESS
|
1977 ISCSI_TARGET_NAME
| ISCSI_TPGT
|
1978 ISCSI_USERNAME
| ISCSI_PASSWORD
|
1979 ISCSI_USERNAME_IN
| ISCSI_PASSWORD_IN
|
1980 ISCSI_FAST_ABORT
| ISCSI_ABORT_TMO
|
1981 ISCSI_LU_RESET_TMO
|
1982 ISCSI_PING_TMO
| ISCSI_RECV_TMO
,
1983 .host_param_mask
= ISCSI_HOST_HWADDRESS
| ISCSI_HOST_IPADDRESS
|
1984 ISCSI_HOST_INITIATOR_NAME
|
1985 ISCSI_HOST_NETDEV_NAME
,
1986 .conndata_size
= sizeof(struct iscsi_conn
),
1987 .sessiondata_size
= sizeof(struct iscsi_session
),
1988 /* session management */
1989 .create_session
= iscsi_tcp_session_create
,
1990 .destroy_session
= iscsi_tcp_session_destroy
,
1991 /* connection management */
1992 .create_conn
= iscsi_tcp_conn_create
,
1993 .bind_conn
= iscsi_tcp_conn_bind
,
1994 .destroy_conn
= iscsi_tcp_conn_destroy
,
1995 .set_param
= iscsi_conn_set_param
,
1996 .get_conn_param
= iscsi_tcp_conn_get_param
,
1997 .get_session_param
= iscsi_session_get_param
,
1998 .start_conn
= iscsi_conn_start
,
1999 .stop_conn
= iscsi_tcp_conn_stop
,
2000 /* iscsi host params */
2001 .get_host_param
= iscsi_host_get_param
,
2002 .set_host_param
= iscsi_host_set_param
,
2004 .send_pdu
= iscsi_conn_send_pdu
,
2005 .get_stats
= iscsi_conn_get_stats
,
2006 .init_cmd_task
= iscsi_tcp_ctask_init
,
2007 .init_mgmt_task
= iscsi_tcp_mtask_init
,
2008 .xmit_cmd_task
= iscsi_tcp_ctask_xmit
,
2009 .xmit_mgmt_task
= iscsi_tcp_mtask_xmit
,
2010 .cleanup_cmd_task
= iscsi_tcp_cleanup_ctask
,
2012 .session_recovery_timedout
= iscsi_session_recovery_timedout
,
2016 iscsi_tcp_init(void)
2018 if (iscsi_max_lun
< 1) {
2019 printk(KERN_ERR
"iscsi_tcp: Invalid max_lun value of %u\n",
2024 iscsi_tcp_scsi_transport
= iscsi_register_transport(
2025 &iscsi_tcp_transport
);
2026 if (!iscsi_tcp_scsi_transport
)
2033 iscsi_tcp_exit(void)
2035 iscsi_unregister_transport(&iscsi_tcp_transport
);
2038 module_init(iscsi_tcp_init
);
2039 module_exit(iscsi_tcp_exit
);