?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[SCSI] iscsi: dont use sendpage for iscsi headers
[linux-2.6/kvm.git] / drivers / scsi / iscsi_tcp.c
blob7c2ed7bb7e94818904fa71dbb0232b172f3089b0
1 /*
2 * iSCSI Initiator over TCP/IP Data-Path
3 *
4 * Copyright (C) 2004 Dmitry Yusupov
5 * Copyright (C) 2004 Alex Aizman
6 * Copyright (C) 2005 - 2006 Mike Christie
7 * Copyright (C) 2006 Red Hat, Inc. All rights reserved.
8 * maintained by open-iscsi@googlegroups.com
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published
12 * by the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * General Public License for more details.
19 *
20 * See the file COPYING included with this distribution for more details.
21 *
22 * Credits:
23 * Christoph Hellwig
24 * FUJITA Tomonori
25 * Arne Redlich
26 * Zhenyu Wang
27 */
28
29 #include <linux/types.h>
30 #include <linux/list.h>
31 #include <linux/inet.h>
32 #include <linux/blkdev.h>
33 #include <linux/crypto.h>
34 #include <linux/delay.h>
35 #include <linux/kfifo.h>
36 #include <linux/scatterlist.h>
37 #include <linux/mutex.h>
38 #include <net/tcp.h>
39 #include <scsi/scsi_cmnd.h>
40 #include <scsi/scsi_host.h>
41 #include <scsi/scsi.h>
42 #include <scsi/scsi_transport_iscsi.h>
43
44 #include "iscsi_tcp.h"
45
46 MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
47 "Alex Aizman <itn780@yahoo.com>");
48 MODULE_DESCRIPTION("iSCSI/TCP data-path");
49 MODULE_LICENSE("GPL");
50 MODULE_VERSION("0:4.445");
51 /* #define DEBUG_TCP */
52 #define DEBUG_ASSERT
53
54 #ifdef DEBUG_TCP
55 #define debug_tcp(fmt...) printk(KERN_INFO "tcp: " fmt)
56 #else
57 #define debug_tcp(fmt...)
58 #endif
59
60 #ifndef DEBUG_ASSERT
61 #ifdef BUG_ON
62 #undef BUG_ON
63 #endif
64 #define BUG_ON(expr)
65 #endif
66
67 static unsigned int iscsi_max_lun = 512;
68 module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
69
70 /* global data */
71 static kmem_cache_t *taskcache;
72
73 static inline void
74 iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size)
75 {
76 ibuf->sg.page = virt_to_page(vbuf);
77 ibuf->sg.offset = offset_in_page(vbuf);
78 ibuf->sg.length = size;
79 ibuf->sent = 0;
80 ibuf->use_sendmsg = 1;
81 }
82
83 static inline void
84 iscsi_buf_init_sg(struct iscsi_buf *ibuf, struct scatterlist *sg)
85 {
86 ibuf->sg.page = sg->page;
87 ibuf->sg.offset = sg->offset;
88 ibuf->sg.length = sg->length;
89 /*
90 * Fastpath: sg element fits into single page
91 */
92 if (sg->length + sg->offset <= PAGE_SIZE && !PageSlab(sg->page))
93 ibuf->use_sendmsg = 0;
94 else
95 ibuf->use_sendmsg = 1;
96 ibuf->sent = 0;
97 }
98
99 static inline int
100 iscsi_buf_left(struct iscsi_buf *ibuf)
101 {
102 int rc;
103
104 rc = ibuf->sg.length - ibuf->sent;
105 BUG_ON(rc < 0);
106 return rc;
107 }
108
109 static inline void
110 iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf,
111 u8* crc)
112 {
113 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
114
115 crypto_digest_digest(tcp_conn->tx_tfm, &buf->sg, 1, crc);
116 buf->sg.length += sizeof(uint32_t);
117 }
118
119 static inline int
120 iscsi_hdr_extract(struct iscsi_tcp_conn *tcp_conn)
121 {
122 struct sk_buff *skb = tcp_conn->in.skb;
123
124 tcp_conn->in.zero_copy_hdr = 0;
125
126 if (tcp_conn->in.copy >= tcp_conn->hdr_size &&
127 tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
128 /*
129 * Zero-copy PDU Header: using connection context
130 * to store header pointer.
131 */
132 if (skb_shinfo(skb)->frag_list == NULL &&
133 !skb_shinfo(skb)->nr_frags) {
134 tcp_conn->in.hdr = (struct iscsi_hdr *)
135 ((char*)skb->data + tcp_conn->in.offset);
136 tcp_conn->in.zero_copy_hdr = 1;
137 } else {
138 /* ignoring return code since we checked
139 * in.copy before */
140 skb_copy_bits(skb, tcp_conn->in.offset,
141 &tcp_conn->hdr, tcp_conn->hdr_size);
142 tcp_conn->in.hdr = &tcp_conn->hdr;
143 }
144 tcp_conn->in.offset += tcp_conn->hdr_size;
145 tcp_conn->in.copy -= tcp_conn->hdr_size;
146 } else {
147 int hdr_remains;
148 int copylen;
149
150 /*
151 * PDU header scattered across SKB's,
152 * copying it... This'll happen quite rarely.
153 */
154
155 if (tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER)
156 tcp_conn->in.hdr_offset = 0;
157
158 hdr_remains = tcp_conn->hdr_size - tcp_conn->in.hdr_offset;
159 BUG_ON(hdr_remains <= 0);
160
161 copylen = min(tcp_conn->in.copy, hdr_remains);
162 skb_copy_bits(skb, tcp_conn->in.offset,
163 (char*)&tcp_conn->hdr + tcp_conn->in.hdr_offset,
164 copylen);
165
166 debug_tcp("PDU gather offset %d bytes %d in.offset %d "
167 "in.copy %d\n", tcp_conn->in.hdr_offset, copylen,
168 tcp_conn->in.offset, tcp_conn->in.copy);
169
170 tcp_conn->in.offset += copylen;
171 tcp_conn->in.copy -= copylen;
172 if (copylen < hdr_remains) {
173 tcp_conn->in_progress = IN_PROGRESS_HEADER_GATHER;
174 tcp_conn->in.hdr_offset += copylen;
175 return -EAGAIN;
176 }
177 tcp_conn->in.hdr = &tcp_conn->hdr;
178 tcp_conn->discontiguous_hdr_cnt++;
179 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
180 }
181
182 return 0;
183 }
184
185 /*
186 * must be called with session lock
187 */
188 static void
189 __iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
190 {
191 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
192 struct scsi_cmnd *sc;
193
194 sc = ctask->sc;
195 if (unlikely(!sc))
196 return;
197
198 if (sc->sc_data_direction == DMA_TO_DEVICE) {
199 struct iscsi_data_task *dtask, *n;
200
201 /* WRITE: cleanup Data-Out's if any */
202 list_for_each_entry_safe(dtask, n, &tcp_ctask->dataqueue,
203 item) {
204 list_del(&dtask->item);
205 mempool_free(dtask, tcp_ctask->datapool);
206 }
207 }
208 tcp_ctask->xmstate = XMSTATE_IDLE;
209 tcp_ctask->r2t = NULL;
210 }
211
212 /**
213 * iscsi_data_rsp - SCSI Data-In Response processing
214 * @conn: iscsi connection
215 * @ctask: scsi command task
216 **/
217 static int
218 iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
219 {
220 int rc;
221 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
222 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
223 struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr;
224 struct iscsi_session *session = conn->session;
225 int datasn = be32_to_cpu(rhdr->datasn);
226
227 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
228 if (rc)
229 return rc;
230 /*
231 * setup Data-In byte counter (gets decremented..)
232 */
233 ctask->data_count = tcp_conn->in.datalen;
234
235 if (tcp_conn->in.datalen == 0)
236 return 0;
237
238 if (ctask->datasn != datasn)
239 return ISCSI_ERR_DATASN;
240
241 ctask->datasn++;
242
243 tcp_ctask->data_offset = be32_to_cpu(rhdr->offset);
244 if (tcp_ctask->data_offset + tcp_conn->in.datalen > ctask->total_length)
245 return ISCSI_ERR_DATA_OFFSET;
246
247 if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) {
248 struct scsi_cmnd *sc = ctask->sc;
249
250 conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
251 if (rhdr->flags & ISCSI_FLAG_DATA_UNDERFLOW) {
252 int res_count = be32_to_cpu(rhdr->residual_count);
253
254 if (res_count > 0 &&
255 res_count <= sc->request_bufflen) {
256 sc->resid = res_count;
257 sc->result = (DID_OK << 16) | rhdr->cmd_status;
258 } else
259 sc->result = (DID_BAD_TARGET << 16) |
260 rhdr->cmd_status;
261 } else if (rhdr->flags & ISCSI_FLAG_DATA_OVERFLOW) {
262 sc->resid = be32_to_cpu(rhdr->residual_count);
263 sc->result = (DID_OK << 16) | rhdr->cmd_status;
264 } else
265 sc->result = (DID_OK << 16) | rhdr->cmd_status;
266 }
267
268 conn->datain_pdus_cnt++;
269 return 0;
270 }
271
272 /**
273 * iscsi_solicit_data_init - initialize first Data-Out
274 * @conn: iscsi connection
275 * @ctask: scsi command task
276 * @r2t: R2T info
277 *
278 * Notes:
279 * Initialize first Data-Out within this R2T sequence and finds
280 * proper data_offset within this SCSI command.
281 *
282 * This function is called with connection lock taken.
283 **/
284 static void
285 iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
286 struct iscsi_r2t_info *r2t)
287 {
288 struct iscsi_data *hdr;
289 struct iscsi_data_task *dtask;
290 struct scsi_cmnd *sc = ctask->sc;
291 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
292
293 dtask = mempool_alloc(tcp_ctask->datapool, GFP_ATOMIC);
294 BUG_ON(!dtask);
295 INIT_LIST_HEAD(&dtask->item);
296 hdr = &dtask->hdr;
297 memset(hdr, 0, sizeof(struct iscsi_data));
298 hdr->ttt = r2t->ttt;
299 hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
300 r2t->solicit_datasn++;
301 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
302 memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun));
303 hdr->itt = ctask->hdr->itt;
304 hdr->exp_statsn = r2t->exp_statsn;
305 hdr->offset = cpu_to_be32(r2t->data_offset);
306 if (r2t->data_length > conn->max_xmit_dlength) {
307 hton24(hdr->dlength, conn->max_xmit_dlength);
308 r2t->data_count = conn->max_xmit_dlength;
309 hdr->flags = 0;
310 } else {
311 hton24(hdr->dlength, r2t->data_length);
312 r2t->data_count = r2t->data_length;
313 hdr->flags = ISCSI_FLAG_CMD_FINAL;
314 }
315 conn->dataout_pdus_cnt++;
316
317 r2t->sent = 0;
318
319 iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr,
320 sizeof(struct iscsi_hdr));
321
322 r2t->dtask = dtask;
323
324 if (sc->use_sg) {
325 int i, sg_count = 0;
326 struct scatterlist *sg = sc->request_buffer;
327
328 r2t->sg = NULL;
329 for (i = 0; i < sc->use_sg; i++, sg += 1) {
330 /* FIXME: prefetch ? */
331 if (sg_count + sg->length > r2t->data_offset) {
332 int page_offset;
333
334 /* sg page found! */
335
336 /* offset within this page */
337 page_offset = r2t->data_offset - sg_count;
338
339 /* fill in this buffer */
340 iscsi_buf_init_sg(&r2t->sendbuf, sg);
341 r2t->sendbuf.sg.offset += page_offset;
342 r2t->sendbuf.sg.length -= page_offset;
343
344 /* xmit logic will continue with next one */
345 r2t->sg = sg + 1;
346 break;
347 }
348 sg_count += sg->length;
349 }
350 BUG_ON(r2t->sg == NULL);
351 } else
352 iscsi_buf_init_iov(&tcp_ctask->sendbuf,
353 (char*)sc->request_buffer + r2t->data_offset,
354 r2t->data_count);
355
356 list_add(&dtask->item, &tcp_ctask->dataqueue);
357 }
358
359 /**
360 * iscsi_r2t_rsp - iSCSI R2T Response processing
361 * @conn: iscsi connection
362 * @ctask: scsi command task
363 **/
364 static int
365 iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
366 {
367 struct iscsi_r2t_info *r2t;
368 struct iscsi_session *session = conn->session;
369 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
370 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
371 struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr;
372 int r2tsn = be32_to_cpu(rhdr->r2tsn);
373 int rc;
374
375 if (tcp_conn->in.datalen)
376 return ISCSI_ERR_DATALEN;
377
378 if (tcp_ctask->exp_r2tsn && tcp_ctask->exp_r2tsn != r2tsn)
379 return ISCSI_ERR_R2TSN;
380
381 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
382 if (rc)
383 return rc;
384
385 /* FIXME: use R2TSN to detect missing R2T */
386
387 /* fill-in new R2T associated with the task */
388 spin_lock(&session->lock);
389 if (!ctask->sc || ctask->mtask ||
390 session->state != ISCSI_STATE_LOGGED_IN) {
391 printk(KERN_INFO "iscsi_tcp: dropping R2T itt %d in "
392 "recovery...\n", ctask->itt);
393 spin_unlock(&session->lock);
394 return 0;
395 }
396 rc = __kfifo_get(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
397 BUG_ON(!rc);
398
399 r2t->exp_statsn = rhdr->statsn;
400 r2t->data_length = be32_to_cpu(rhdr->data_length);
401 if (r2t->data_length == 0 ||
402 r2t->data_length > session->max_burst) {
403 spin_unlock(&session->lock);
404 return ISCSI_ERR_DATALEN;
405 }
406
407 r2t->data_offset = be32_to_cpu(rhdr->data_offset);
408 if (r2t->data_offset + r2t->data_length > ctask->total_length) {
409 spin_unlock(&session->lock);
410 return ISCSI_ERR_DATALEN;
411 }
412
413 r2t->ttt = rhdr->ttt; /* no flip */
414 r2t->solicit_datasn = 0;
415
416 iscsi_solicit_data_init(conn, ctask, r2t);
417
418 tcp_ctask->exp_r2tsn = r2tsn + 1;
419 tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
420 __kfifo_put(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*));
421 __kfifo_put(conn->xmitqueue, (void*)&ctask, sizeof(void*));
422
423 scsi_queue_work(session->host, &conn->xmitwork);
424 conn->r2t_pdus_cnt++;
425 spin_unlock(&session->lock);
426
427 return 0;
428 }
429
430 static int
431 iscsi_tcp_hdr_recv(struct iscsi_conn *conn)
432 {
433 int rc = 0, opcode, ahslen;
434 struct iscsi_hdr *hdr;
435 struct iscsi_session *session = conn->session;
436 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
437 uint32_t cdgst, rdgst = 0, itt;
438
439 hdr = tcp_conn->in.hdr;
440
441 /* verify PDU length */
442 tcp_conn->in.datalen = ntoh24(hdr->dlength);
443 if (tcp_conn->in.datalen > conn->max_recv_dlength) {
444 printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n",
445 tcp_conn->in.datalen, conn->max_recv_dlength);
446 return ISCSI_ERR_DATALEN;
447 }
448 tcp_conn->data_copied = 0;
449
450 /* read AHS */
451 ahslen = hdr->hlength << 2;
452 tcp_conn->in.offset += ahslen;
453 tcp_conn->in.copy -= ahslen;
454 if (tcp_conn->in.copy < 0) {
455 printk(KERN_ERR "iscsi_tcp: can't handle AHS with length "
456 "%d bytes\n", ahslen);
457 return ISCSI_ERR_AHSLEN;
458 }
459
460 /* calculate read padding */
461 tcp_conn->in.padding = tcp_conn->in.datalen & (ISCSI_PAD_LEN-1);
462 if (tcp_conn->in.padding) {
463 tcp_conn->in.padding = ISCSI_PAD_LEN - tcp_conn->in.padding;
464 debug_scsi("read padding %d bytes\n", tcp_conn->in.padding);
465 }
466
467 if (conn->hdrdgst_en) {
468 struct scatterlist sg;
469
470 sg_init_one(&sg, (u8 *)hdr,
471 sizeof(struct iscsi_hdr) + ahslen);
472 crypto_digest_digest(tcp_conn->rx_tfm, &sg, 1, (u8 *)&cdgst);
473 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
474 ahslen);
475 if (cdgst != rdgst) {
476 printk(KERN_ERR "iscsi_tcp: hdrdgst error "
477 "recv 0x%x calc 0x%x\n", rdgst, cdgst);
478 return ISCSI_ERR_HDR_DGST;
479 }
480 }
481
482 opcode = hdr->opcode & ISCSI_OPCODE_MASK;
483 /* verify itt (itt encoding: age+cid+itt) */
484 rc = iscsi_verify_itt(conn, hdr, &itt);
485 if (rc == ISCSI_ERR_NO_SCSI_CMD) {
486 tcp_conn->in.datalen = 0; /* force drop */
487 return 0;
488 } else if (rc)
489 return rc;
490
491 debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n",
492 opcode, tcp_conn->in.offset, tcp_conn->in.copy,
493 ahslen, tcp_conn->in.datalen);
494
495 switch(opcode) {
496 case ISCSI_OP_SCSI_DATA_IN:
497 tcp_conn->in.ctask = session->cmds[itt];
498 rc = iscsi_data_rsp(conn, tcp_conn->in.ctask);
499 /* fall through */
500 case ISCSI_OP_SCSI_CMD_RSP:
501 tcp_conn->in.ctask = session->cmds[itt];
502 if (tcp_conn->in.datalen)
503 goto copy_hdr;
504
505 spin_lock(&session->lock);
506 __iscsi_ctask_cleanup(conn, tcp_conn->in.ctask);
507 rc = __iscsi_complete_pdu(conn, hdr, NULL, 0);
508 spin_unlock(&session->lock);
509 break;
510 case ISCSI_OP_R2T:
511 tcp_conn->in.ctask = session->cmds[itt];
512 if (ahslen)
513 rc = ISCSI_ERR_AHSLEN;
514 else if (tcp_conn->in.ctask->sc->sc_data_direction ==
515 DMA_TO_DEVICE)
516 rc = iscsi_r2t_rsp(conn, tcp_conn->in.ctask);
517 else
518 rc = ISCSI_ERR_PROTO;
519 break;
520 case ISCSI_OP_LOGIN_RSP:
521 case ISCSI_OP_TEXT_RSP:
522 case ISCSI_OP_LOGOUT_RSP:
523 case ISCSI_OP_NOOP_IN:
524 case ISCSI_OP_REJECT:
525 case ISCSI_OP_ASYNC_EVENT:
526 if (tcp_conn->in.datalen)
527 goto copy_hdr;
528 /* fall through */
529 case ISCSI_OP_SCSI_TMFUNC_RSP:
530 rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
531 break;
532 default:
533 rc = ISCSI_ERR_BAD_OPCODE;
534 break;
535 }
536
537 return rc;
538
539 copy_hdr:
540 /*
541 * if we did zero copy for the header but we will need multiple
542 * skbs to complete the command then we have to copy the header
543 * for later use
544 */
545 if (tcp_conn->in.zero_copy_hdr && tcp_conn->in.copy <
546 (tcp_conn->in.datalen + tcp_conn->in.padding +
547 (conn->datadgst_en ? 4 : 0))) {
548 debug_tcp("Copying header for later use. in.copy %d in.datalen"
549 " %d\n", tcp_conn->in.copy, tcp_conn->in.datalen);
550 memcpy(&tcp_conn->hdr, tcp_conn->in.hdr,
551 sizeof(struct iscsi_hdr));
552 tcp_conn->in.hdr = &tcp_conn->hdr;
553 tcp_conn->in.zero_copy_hdr = 0;
554 }
555 return 0;
556 }
557
558 /**
559 * iscsi_ctask_copy - copy skb bits to the destanation cmd task
560 * @conn: iscsi tcp connection
561 * @ctask: scsi command task
562 * @buf: buffer to copy to
563 * @buf_size: size of buffer
564 * @offset: offset within the buffer
565 *
566 * Notes:
567 * The function calls skb_copy_bits() and updates per-connection and
568 * per-cmd byte counters.
569 *
570 * Read counters (in bytes):
571 *
572 * conn->in.offset offset within in progress SKB
573 * conn->in.copy left to copy from in progress SKB
574 * including padding
575 * conn->in.copied copied already from in progress SKB
576 * conn->data_copied copied already from in progress buffer
577 * ctask->sent total bytes sent up to the MidLayer
578 * ctask->data_count left to copy from in progress Data-In
579 * buf_left left to copy from in progress buffer
580 **/
581 static inline int
582 iscsi_ctask_copy(struct iscsi_tcp_conn *tcp_conn, struct iscsi_cmd_task *ctask,
583 void *buf, int buf_size, int offset)
584 {
585 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
586 int buf_left = buf_size - (tcp_conn->data_copied + offset);
587 int size = min(tcp_conn->in.copy, buf_left);
588 int rc;
589
590 size = min(size, ctask->data_count);
591
592 debug_tcp("ctask_copy %d bytes at offset %d copied %d\n",
593 size, tcp_conn->in.offset, tcp_conn->in.copied);
594
595 BUG_ON(size <= 0);
596 BUG_ON(tcp_ctask->sent + size > ctask->total_length);
597
598 rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
599 (char*)buf + (offset + tcp_conn->data_copied), size);
600 /* must fit into skb->len */
601 BUG_ON(rc);
602
603 tcp_conn->in.offset += size;
604 tcp_conn->in.copy -= size;
605 tcp_conn->in.copied += size;
606 tcp_conn->data_copied += size;
607 tcp_ctask->sent += size;
608 ctask->data_count -= size;
609
610 BUG_ON(tcp_conn->in.copy < 0);
611 BUG_ON(ctask->data_count < 0);
612
613 if (buf_size != (tcp_conn->data_copied + offset)) {
614 if (!ctask->data_count) {
615 BUG_ON(buf_size - tcp_conn->data_copied < 0);
616 /* done with this PDU */
617 return buf_size - tcp_conn->data_copied;
618 }
619 return -EAGAIN;
620 }
621
622 /* done with this buffer or with both - PDU and buffer */
623 tcp_conn->data_copied = 0;
624 return 0;
625 }
626
627 /**
628 * iscsi_tcp_copy - copy skb bits to the destanation buffer
629 * @conn: iscsi tcp connection
630 *
631 * Notes:
632 * The function calls skb_copy_bits() and updates per-connection
633 * byte counters.
634 **/
635 static inline int
636 iscsi_tcp_copy(struct iscsi_tcp_conn *tcp_conn)
637 {
638 void *buf = tcp_conn->data;
639 int buf_size = tcp_conn->in.datalen;
640 int buf_left = buf_size - tcp_conn->data_copied;
641 int size = min(tcp_conn->in.copy, buf_left);
642 int rc;
643
644 debug_tcp("tcp_copy %d bytes at offset %d copied %d\n",
645 size, tcp_conn->in.offset, tcp_conn->data_copied);
646 BUG_ON(size <= 0);
647
648 rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
649 (char*)buf + tcp_conn->data_copied, size);
650 BUG_ON(rc);
651
652 tcp_conn->in.offset += size;
653 tcp_conn->in.copy -= size;
654 tcp_conn->in.copied += size;
655 tcp_conn->data_copied += size;
656
657 if (buf_size != tcp_conn->data_copied)
658 return -EAGAIN;
659
660 return 0;
661 }
662
663 static inline void
664 partial_sg_digest_update(struct iscsi_tcp_conn *tcp_conn,
665 struct scatterlist *sg, int offset, int length)
666 {
667 struct scatterlist temp;
668
669 memcpy(&temp, sg, sizeof(struct scatterlist));
670 temp.offset = offset;
671 temp.length = length;
672 crypto_digest_update(tcp_conn->data_rx_tfm, &temp, 1);
673 }
674
675 static void
676 iscsi_recv_digest_update(struct iscsi_tcp_conn *tcp_conn, char* buf, int len)
677 {
678 struct scatterlist tmp;
679
680 sg_init_one(&tmp, buf, len);
681 crypto_digest_update(tcp_conn->data_rx_tfm, &tmp, 1);
682 }
683
684 static int iscsi_scsi_data_in(struct iscsi_conn *conn)
685 {
686 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
687 struct iscsi_cmd_task *ctask = tcp_conn->in.ctask;
688 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
689 struct scsi_cmnd *sc = ctask->sc;
690 struct scatterlist *sg;
691 int i, offset, rc = 0;
692
693 BUG_ON((void*)ctask != sc->SCp.ptr);
694
695 /*
696 * copying Data-In into the Scsi_Cmnd
697 */
698 if (!sc->use_sg) {
699 i = ctask->data_count;
700 rc = iscsi_ctask_copy(tcp_conn, ctask, sc->request_buffer,
701 sc->request_bufflen,
702 tcp_ctask->data_offset);
703 if (rc == -EAGAIN)
704 return rc;
705 if (conn->datadgst_en)
706 iscsi_recv_digest_update(tcp_conn, sc->request_buffer,
707 i);
708 rc = 0;
709 goto done;
710 }
711
712 offset = tcp_ctask->data_offset;
713 sg = sc->request_buffer;
714
715 if (tcp_ctask->data_offset)
716 for (i = 0; i < tcp_ctask->sg_count; i++)
717 offset -= sg[i].length;
718 /* we've passed through partial sg*/
719 if (offset < 0)
720 offset = 0;
721
722 for (i = tcp_ctask->sg_count; i < sc->use_sg; i++) {
723 char *dest;
724
725 dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0);
726 rc = iscsi_ctask_copy(tcp_conn, ctask, dest + sg[i].offset,
727 sg[i].length, offset);
728 kunmap_atomic(dest, KM_SOFTIRQ0);
729 if (rc == -EAGAIN)
730 /* continue with the next SKB/PDU */
731 return rc;
732 if (!rc) {
733 if (conn->datadgst_en) {
734 if (!offset)
735 crypto_digest_update(
736 tcp_conn->data_rx_tfm,
737 &sg[i], 1);
738 else
739 partial_sg_digest_update(tcp_conn,
740 &sg[i],
741 sg[i].offset + offset,
742 sg[i].length - offset);
743 }
744 offset = 0;
745 tcp_ctask->sg_count++;
746 }
747
748 if (!ctask->data_count) {
749 if (rc && conn->datadgst_en)
750 /*
751 * data-in is complete, but buffer not...
752 */
753 partial_sg_digest_update(tcp_conn, &sg[i],
754 sg[i].offset, sg[i].length-rc);
755 rc = 0;
756 break;
757 }
758
759 if (!tcp_conn->in.copy)
760 return -EAGAIN;
761 }
762 BUG_ON(ctask->data_count);
763
764 done:
765 /* check for non-exceptional status */
766 if (tcp_conn->in.hdr->flags & ISCSI_FLAG_DATA_STATUS) {
767 debug_scsi("done [sc %lx res %d itt 0x%x]\n",
768 (long)sc, sc->result, ctask->itt);
769 spin_lock(&conn->session->lock);
770 __iscsi_ctask_cleanup(conn, ctask);
771 __iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0);
772 spin_unlock(&conn->session->lock);
773 }
774
775 return rc;
776 }
777
778 static int
779 iscsi_data_recv(struct iscsi_conn *conn)
780 {
781 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
782 int rc = 0, opcode;
783
784 opcode = tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK;
785 switch (opcode) {
786 case ISCSI_OP_SCSI_DATA_IN:
787 rc = iscsi_scsi_data_in(conn);
788 break;
789 case ISCSI_OP_SCSI_CMD_RSP:
790 spin_lock(&conn->session->lock);
791 __iscsi_ctask_cleanup(conn, tcp_conn->in.ctask);
792 spin_unlock(&conn->session->lock);
793 case ISCSI_OP_TEXT_RSP:
794 case ISCSI_OP_LOGIN_RSP:
795 case ISCSI_OP_NOOP_IN:
796 case ISCSI_OP_ASYNC_EVENT:
797 case ISCSI_OP_REJECT:
798 /*
799 * Collect data segment to the connection's data
800 * placeholder
801 */
802 if (iscsi_tcp_copy(tcp_conn)) {
803 rc = -EAGAIN;
804 goto exit;
805 }
806
807 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, tcp_conn->data,
808 tcp_conn->in.datalen);
809 if (!rc && conn->datadgst_en && opcode != ISCSI_OP_LOGIN_RSP)
810 iscsi_recv_digest_update(tcp_conn, tcp_conn->data,
811 tcp_conn->in.datalen);
812 break;
813 default:
814 BUG_ON(1);
815 }
816 exit:
817 return rc;
818 }
819
820 /**
821 * iscsi_tcp_data_recv - TCP receive in sendfile fashion
822 * @rd_desc: read descriptor
823 * @skb: socket buffer
824 * @offset: offset in skb
825 * @len: skb->len - offset
826 **/
827 static int
828 iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
829 unsigned int offset, size_t len)
830 {
831 int rc;
832 struct iscsi_conn *conn = rd_desc->arg.data;
833 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
834 int processed;
835 char pad[ISCSI_PAD_LEN];
836 struct scatterlist sg;
837
838 /*
839 * Save current SKB and its offset in the corresponding
840 * connection context.
841 */
842 tcp_conn->in.copy = skb->len - offset;
843 tcp_conn->in.offset = offset;
844 tcp_conn->in.skb = skb;
845 tcp_conn->in.len = tcp_conn->in.copy;
846 BUG_ON(tcp_conn->in.copy <= 0);
847 debug_tcp("in %d bytes\n", tcp_conn->in.copy);
848
849 more:
850 tcp_conn->in.copied = 0;
851 rc = 0;
852
853 if (unlikely(conn->suspend_rx)) {
854 debug_tcp("conn %d Rx suspended!\n", conn->id);
855 return 0;
856 }
857
858 if (tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER ||
859 tcp_conn->in_progress == IN_PROGRESS_HEADER_GATHER) {
860 rc = iscsi_hdr_extract(tcp_conn);
861 if (rc) {
862 if (rc == -EAGAIN)
863 goto nomore;
864 else {
865 iscsi_conn_failure(conn, rc);
866 return 0;
867 }
868 }
869
870 /*
871 * Verify and process incoming PDU header.
872 */
873 rc = iscsi_tcp_hdr_recv(conn);
874 if (!rc && tcp_conn->in.datalen) {
875 if (conn->datadgst_en) {
876 BUG_ON(!tcp_conn->data_rx_tfm);
877 crypto_digest_init(tcp_conn->data_rx_tfm);
878 }
879 tcp_conn->in_progress = IN_PROGRESS_DATA_RECV;
880 } else if (rc) {
881 iscsi_conn_failure(conn, rc);
882 return 0;
883 }
884 }
885
886 if (tcp_conn->in_progress == IN_PROGRESS_DDIGEST_RECV) {
887 uint32_t recv_digest;
888
889 debug_tcp("extra data_recv offset %d copy %d\n",
890 tcp_conn->in.offset, tcp_conn->in.copy);
891 skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
892 &recv_digest, 4);
893 tcp_conn->in.offset += 4;
894 tcp_conn->in.copy -= 4;
895 if (recv_digest != tcp_conn->in.datadgst) {
896 debug_tcp("iscsi_tcp: data digest error!"
897 "0x%x != 0x%x\n", recv_digest,
898 tcp_conn->in.datadgst);
899 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
900 return 0;
901 } else {
902 debug_tcp("iscsi_tcp: data digest match!"
903 "0x%x == 0x%x\n", recv_digest,
904 tcp_conn->in.datadgst);
905 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
906 }
907 }
908
909 if (tcp_conn->in_progress == IN_PROGRESS_DATA_RECV &&
910 tcp_conn->in.copy) {
911
912 debug_tcp("data_recv offset %d copy %d\n",
913 tcp_conn->in.offset, tcp_conn->in.copy);
914
915 rc = iscsi_data_recv(conn);
916 if (rc) {
917 if (rc == -EAGAIN)
918 goto again;
919 iscsi_conn_failure(conn, rc);
920 return 0;
921 }
922 tcp_conn->in.copy -= tcp_conn->in.padding;
923 tcp_conn->in.offset += tcp_conn->in.padding;
924 if (conn->datadgst_en) {
925 if (tcp_conn->in.padding) {
926 debug_tcp("padding -> %d\n",
927 tcp_conn->in.padding);
928 memset(pad, 0, tcp_conn->in.padding);
929 sg_init_one(&sg, pad, tcp_conn->in.padding);
930 crypto_digest_update(tcp_conn->data_rx_tfm,
931 &sg, 1);
932 }
933 crypto_digest_final(tcp_conn->data_rx_tfm,
934 (u8 *) & tcp_conn->in.datadgst);
935 debug_tcp("rx digest 0x%x\n", tcp_conn->in.datadgst);
936 tcp_conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
937 } else
938 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
939 }
940
941 debug_tcp("f, processed %d from out of %d padding %d\n",
942 tcp_conn->in.offset - offset, (int)len, tcp_conn->in.padding);
943 BUG_ON(tcp_conn->in.offset - offset > len);
944
945 if (tcp_conn->in.offset - offset != len) {
946 debug_tcp("continue to process %d bytes\n",
947 (int)len - (tcp_conn->in.offset - offset));
948 goto more;
949 }
950
951 nomore:
952 processed = tcp_conn->in.offset - offset;
953 BUG_ON(processed == 0);
954 return processed;
955
956 again:
957 processed = tcp_conn->in.offset - offset;
958 debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n",
959 processed, (int)len, (int)rd_desc->count);
960 BUG_ON(processed == 0);
961 BUG_ON(processed > len);
962
963 conn->rxdata_octets += processed;
964 return processed;
965 }
966
967 static void
968 iscsi_tcp_data_ready(struct sock *sk, int flag)
969 {
970 struct iscsi_conn *conn = sk->sk_user_data;
971 read_descriptor_t rd_desc;
972
973 read_lock(&sk->sk_callback_lock);
974
975 /*
976 * Use rd_desc to pass 'conn' to iscsi_tcp_data_recv.
977 * We set count to 1 because we want the network layer to
978 * hand us all the skbs that are available. iscsi_tcp_data_recv
979 * handled pdus that cross buffers or pdus that still need data.
980 */
981 rd_desc.arg.data = conn;
982 rd_desc.count = 1;
983 tcp_read_sock(sk, &rd_desc, iscsi_tcp_data_recv);
984
985 read_unlock(&sk->sk_callback_lock);
986 }
987
988 static void
989 iscsi_tcp_state_change(struct sock *sk)
990 {
991 struct iscsi_tcp_conn *tcp_conn;
992 struct iscsi_conn *conn;
993 struct iscsi_session *session;
994 void (*old_state_change)(struct sock *);
995
996 read_lock(&sk->sk_callback_lock);
997
998 conn = (struct iscsi_conn*)sk->sk_user_data;
999 session = conn->session;
1000
1001 if ((sk->sk_state == TCP_CLOSE_WAIT ||
1002 sk->sk_state == TCP_CLOSE) &&
1003 !atomic_read(&sk->sk_rmem_alloc)) {
1004 debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n");
1005 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1006 }
1007
1008 tcp_conn = conn->dd_data;
1009 old_state_change = tcp_conn->old_state_change;
1010
1011 read_unlock(&sk->sk_callback_lock);
1012
1013 old_state_change(sk);
1014 }
1015
1016 /**
1017 * iscsi_write_space - Called when more output buffer space is available
1018 * @sk: socket space is available for
1019 **/
1020 static void
1021 iscsi_write_space(struct sock *sk)
1022 {
1023 struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
1024 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1025
1026 tcp_conn->old_write_space(sk);
1027 debug_tcp("iscsi_write_space: cid %d\n", conn->id);
1028 clear_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
1029 scsi_queue_work(conn->session->host, &conn->xmitwork);
1030 }
1031
1032 static void
1033 iscsi_conn_set_callbacks(struct iscsi_conn *conn)
1034 {
1035 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1036 struct sock *sk = tcp_conn->sock->sk;
1037
1038 /* assign new callbacks */
1039 write_lock_bh(&sk->sk_callback_lock);
1040 sk->sk_user_data = conn;
1041 tcp_conn->old_data_ready = sk->sk_data_ready;
1042 tcp_conn->old_state_change = sk->sk_state_change;
1043 tcp_conn->old_write_space = sk->sk_write_space;
1044 sk->sk_data_ready = iscsi_tcp_data_ready;
1045 sk->sk_state_change = iscsi_tcp_state_change;
1046 sk->sk_write_space = iscsi_write_space;
1047 write_unlock_bh(&sk->sk_callback_lock);
1048 }
1049
1050 static void
1051 iscsi_conn_restore_callbacks(struct iscsi_conn *conn)
1052 {
1053 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1054 struct sock *sk = tcp_conn->sock->sk;
1055
1056 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
1057 write_lock_bh(&sk->sk_callback_lock);
1058 sk->sk_user_data = NULL;
1059 sk->sk_data_ready = tcp_conn->old_data_ready;
1060 sk->sk_state_change = tcp_conn->old_state_change;
1061 sk->sk_write_space = tcp_conn->old_write_space;
1062 sk->sk_no_check = 0;
1063 write_unlock_bh(&sk->sk_callback_lock);
1064 }
1065
1066 /**
1067 * iscsi_send - generic send routine
1068 * @sk: kernel's socket
1069 * @buf: buffer to write from
1070 * @size: actual size to write
1071 * @flags: socket's flags
1072 */
1073 static inline int
1074 iscsi_send(struct iscsi_conn *conn, struct iscsi_buf *buf, int size, int flags)
1075 {
1076 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1077 struct socket *sk = tcp_conn->sock;
1078 int offset = buf->sg.offset + buf->sent;
1079
1080 /*
1081 * if we got use_sg=0 or are sending something we kmallocd
1082 * then we did not have to do kmap (kmap returns page_address)
1083 *
1084 * if we got use_sg > 0, but had to drop down, we do not
1085 * set clustering so this should only happen for that
1086 * slab case.
1087 */
1088 if (buf->use_sendmsg)
1089 return sock_no_sendpage(sk, buf->sg.page, offset, size, flags);
1090 else
1091 return tcp_conn->sendpage(sk, buf->sg.page, offset, size,
1092 flags);
1093 }
1094
1095 /**
1096 * iscsi_sendhdr - send PDU Header via tcp_sendpage()
1097 * @conn: iscsi connection
1098 * @buf: buffer to write from
1099 * @datalen: lenght of data to be sent after the header
1100 *
1101 * Notes:
1102 * (Tx, Fast Path)
1103 **/
1104 static inline int
1105 iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen)
1106 {
1107 struct iscsi_tcp_conn *tcp_conn;
1108 int flags = 0; /* MSG_DONTWAIT; */
1109 int res, size;
1110
1111 size = buf->sg.length - buf->sent;
1112 BUG_ON(buf->sent + size > buf->sg.length);
1113 if (buf->sent + size != buf->sg.length || datalen)
1114 flags |= MSG_MORE;
1115
1116 res = iscsi_send(conn, buf, size, flags);
1117 debug_tcp("sendhdr %d bytes, sent %d res %d\n", size, buf->sent, res);
1118 if (res >= 0) {
1119 conn->txdata_octets += res;
1120 buf->sent += res;
1121 if (size != res)
1122 return -EAGAIN;
1123 return 0;
1124 } else if (res == -EAGAIN) {
1125 tcp_conn = conn->dd_data;
1126 tcp_conn->sendpage_failures_cnt++;
1127 set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
1128 } else if (res == -EPIPE)
1129 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1130
1131 return res;
1132 }
1133
1134 /**
1135 * iscsi_sendpage - send one page of iSCSI Data-Out.
1136 * @conn: iscsi connection
1137 * @buf: buffer to write from
1138 * @count: remaining data
1139 * @sent: number of bytes sent
1140 *
1141 * Notes:
1142 * (Tx, Fast Path)
1143 **/
1144 static inline int
1145 iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
1146 int *count, int *sent)
1147 {
1148 struct iscsi_tcp_conn *tcp_conn;
1149 int flags = 0; /* MSG_DONTWAIT; */
1150 int res, size;
1151
1152 size = buf->sg.length - buf->sent;
1153 BUG_ON(buf->sent + size > buf->sg.length);
1154 if (size > *count)
1155 size = *count;
1156 if (buf->sent + size != buf->sg.length || *count != size)
1157 flags |= MSG_MORE;
1158
1159 res = iscsi_send(conn, buf, size, flags);
1160 debug_tcp("sendpage: %d bytes, sent %d left %d sent %d res %d\n",
1161 size, buf->sent, *count, *sent, res);
1162 if (res >= 0) {
1163 conn->txdata_octets += res;
1164 buf->sent += res;
1165 *count -= res;
1166 *sent += res;
1167 if (size != res)
1168 return -EAGAIN;
1169 return 0;
1170 } else if (res == -EAGAIN) {
1171 tcp_conn = conn->dd_data;
1172 tcp_conn->sendpage_failures_cnt++;
1173 set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
1174 } else if (res == -EPIPE)
1175 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1176
1177 return res;
1178 }
1179
1180 static inline void
1181 iscsi_data_digest_init(struct iscsi_tcp_conn *tcp_conn,
1182 struct iscsi_cmd_task *ctask)
1183 {
1184 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1185
1186 BUG_ON(!tcp_conn->data_tx_tfm);
1187 crypto_digest_init(tcp_conn->data_tx_tfm);
1188 tcp_ctask->digest_count = 4;
1189 }
1190
1191 static int
1192 iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1193 struct iscsi_buf *buf, uint32_t *digest, int final)
1194 {
1195 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1196 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1197 int rc = 0;
1198 int sent = 0;
1199
1200 if (final)
1201 crypto_digest_final(tcp_conn->data_tx_tfm, (u8*)digest);
1202
1203 iscsi_buf_init_iov(buf, (char*)digest, 4);
1204 rc = iscsi_sendpage(conn, buf, &tcp_ctask->digest_count, &sent);
1205 if (rc) {
1206 tcp_ctask->datadigest = *digest;
1207 tcp_ctask->xmstate |= XMSTATE_DATA_DIGEST;
1208 } else
1209 tcp_ctask->digest_count = 4;
1210 return rc;
1211 }
1212
1213 /**
1214 * iscsi_solicit_data_cont - initialize next Data-Out
1215 * @conn: iscsi connection
1216 * @ctask: scsi command task
1217 * @r2t: R2T info
1218 * @left: bytes left to transfer
1219 *
1220 * Notes:
1221 * Initialize next Data-Out within this R2T sequence and continue
1222 * to process next Scatter-Gather element(if any) of this SCSI command.
1223 *
1224 * Called under connection lock.
1225 **/
1226 static void
1227 iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1228 struct iscsi_r2t_info *r2t, int left)
1229 {
1230 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1231 struct iscsi_data *hdr;
1232 struct iscsi_data_task *dtask;
1233 struct scsi_cmnd *sc = ctask->sc;
1234 int new_offset;
1235
1236 dtask = mempool_alloc(tcp_ctask->datapool, GFP_ATOMIC);
1237 BUG_ON(!dtask);
1238 INIT_LIST_HEAD(&dtask->item);
1239 hdr = &dtask->hdr;
1240 memset(hdr, 0, sizeof(struct iscsi_data));
1241 hdr->ttt = r2t->ttt;
1242 hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
1243 r2t->solicit_datasn++;
1244 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1245 memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun));
1246 hdr->itt = ctask->hdr->itt;
1247 hdr->exp_statsn = r2t->exp_statsn;
1248 new_offset = r2t->data_offset + r2t->sent;
1249 hdr->offset = cpu_to_be32(new_offset);
1250 if (left > conn->max_xmit_dlength) {
1251 hton24(hdr->dlength, conn->max_xmit_dlength);
1252 r2t->data_count = conn->max_xmit_dlength;
1253 } else {
1254 hton24(hdr->dlength, left);
1255 r2t->data_count = left;
1256 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1257 }
1258 conn->dataout_pdus_cnt++;
1259
1260 iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr,
1261 sizeof(struct iscsi_hdr));
1262
1263 r2t->dtask = dtask;
1264
1265 if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) {
1266 BUG_ON(tcp_ctask->bad_sg == r2t->sg);
1267 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1268 r2t->sg += 1;
1269 } else
1270 iscsi_buf_init_iov(&tcp_ctask->sendbuf,
1271 (char*)sc->request_buffer + new_offset,
1272 r2t->data_count);
1273
1274 list_add(&dtask->item, &tcp_ctask->dataqueue);
1275 }
1276
1277 static void
1278 iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1279 {
1280 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1281 struct iscsi_data_task *dtask;
1282
1283 dtask = mempool_alloc(tcp_ctask->datapool, GFP_ATOMIC);
1284 BUG_ON(!dtask);
1285 INIT_LIST_HEAD(&dtask->item);
1286
1287 iscsi_prep_unsolicit_data_pdu(ctask, &dtask->hdr,
1288 tcp_ctask->r2t_data_count);
1289 iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)&dtask->hdr,
1290 sizeof(struct iscsi_hdr));
1291
1292 list_add(&dtask->item, &tcp_ctask->dataqueue);
1293 tcp_ctask->dtask = dtask;
1294 }
1295
1296 /**
1297 * iscsi_tcp_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
1298 * @conn: iscsi connection
1299 * @ctask: scsi command task
1300 * @sc: scsi command
1301 **/
1302 static void
1303 iscsi_tcp_cmd_init(struct iscsi_cmd_task *ctask)
1304 {
1305 struct scsi_cmnd *sc = ctask->sc;
1306 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1307
1308 BUG_ON(__kfifo_len(tcp_ctask->r2tqueue));
1309
1310 tcp_ctask->sent = 0;
1311 tcp_ctask->sg_count = 0;
1312
1313 if (sc->sc_data_direction == DMA_TO_DEVICE) {
1314 tcp_ctask->xmstate = XMSTATE_W_HDR;
1315 tcp_ctask->exp_r2tsn = 0;
1316 BUG_ON(ctask->total_length == 0);
1317
1318 if (sc->use_sg) {
1319 struct scatterlist *sg = sc->request_buffer;
1320
1321 iscsi_buf_init_sg(&tcp_ctask->sendbuf,
1322 &sg[tcp_ctask->sg_count++]);
1323 tcp_ctask->sg = sg;
1324 tcp_ctask->bad_sg = sg + sc->use_sg;
1325 } else
1326 iscsi_buf_init_iov(&tcp_ctask->sendbuf,
1327 sc->request_buffer,
1328 sc->request_bufflen);
1329
1330 if (ctask->imm_count)
1331 tcp_ctask->xmstate |= XMSTATE_IMM_DATA;
1332
1333 tcp_ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1);
1334 if (tcp_ctask->pad_count) {
1335 tcp_ctask->pad_count = ISCSI_PAD_LEN -
1336 tcp_ctask->pad_count;
1337 debug_scsi("write padding %d bytes\n",
1338 tcp_ctask->pad_count);
1339 tcp_ctask->xmstate |= XMSTATE_W_PAD;
1340 }
1341
1342 if (ctask->unsol_count)
1343 tcp_ctask->xmstate |= XMSTATE_UNS_HDR |
1344 XMSTATE_UNS_INIT;
1345 tcp_ctask->r2t_data_count = ctask->total_length -
1346 ctask->imm_count -
1347 ctask->unsol_count;
1348
1349 debug_scsi("cmd [itt %x total %d imm %d imm_data %d "
1350 "r2t_data %d]\n",
1351 ctask->itt, ctask->total_length, ctask->imm_count,
1352 ctask->unsol_count, tcp_ctask->r2t_data_count);
1353 } else
1354 tcp_ctask->xmstate = XMSTATE_R_HDR;
1355
1356 iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)ctask->hdr,
1357 sizeof(struct iscsi_hdr));
1358 }
1359
1360 /**
1361 * iscsi_tcp_mtask_xmit - xmit management(immediate) task
1362 * @conn: iscsi connection
1363 * @mtask: task management task
1364 *
1365 * Notes:
1366 * The function can return -EAGAIN in which case caller must
1367 * call it again later, or recover. '0' return code means successful
1368 * xmit.
1369 *
1370 * Management xmit state machine consists of two states:
1371 * IN_PROGRESS_IMM_HEAD - PDU Header xmit in progress
1372 * IN_PROGRESS_IMM_DATA - PDU Data xmit in progress
1373 **/
1374 static int
1375 iscsi_tcp_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
1376 {
1377 struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
1378
1379 debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n",
1380 conn->id, tcp_mtask->xmstate, mtask->itt);
1381
1382 if (tcp_mtask->xmstate & XMSTATE_IMM_HDR) {
1383 tcp_mtask->xmstate &= ~XMSTATE_IMM_HDR;
1384 if (mtask->data_count)
1385 tcp_mtask->xmstate |= XMSTATE_IMM_DATA;
1386 if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE &&
1387 conn->stop_stage != STOP_CONN_RECOVER &&
1388 conn->hdrdgst_en)
1389 iscsi_hdr_digest(conn, &tcp_mtask->headbuf,
1390 (u8*)tcp_mtask->hdrext);
1391 if (iscsi_sendhdr(conn, &tcp_mtask->headbuf,
1392 mtask->data_count)) {
1393 tcp_mtask->xmstate |= XMSTATE_IMM_HDR;
1394 if (mtask->data_count)
1395 tcp_mtask->xmstate &= ~XMSTATE_IMM_DATA;
1396 return -EAGAIN;
1397 }
1398 }
1399
1400 if (tcp_mtask->xmstate & XMSTATE_IMM_DATA) {
1401 BUG_ON(!mtask->data_count);
1402 tcp_mtask->xmstate &= ~XMSTATE_IMM_DATA;
1403 /* FIXME: implement.
1404 * Virtual buffer could be spreaded across multiple pages...
1405 */
1406 do {
1407 if (iscsi_sendpage(conn, &tcp_mtask->sendbuf,
1408 &mtask->data_count, &tcp_mtask->sent)) {
1409 tcp_mtask->xmstate |= XMSTATE_IMM_DATA;
1410 return -EAGAIN;
1411 }
1412 } while (mtask->data_count);
1413 }
1414
1415 BUG_ON(tcp_mtask->xmstate != XMSTATE_IDLE);
1416 if (mtask->hdr->itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1417 struct iscsi_session *session = conn->session;
1418
1419 spin_lock_bh(&session->lock);
1420 list_del(&conn->mtask->running);
1421 __kfifo_put(session->mgmtpool.queue, (void*)&conn->mtask,
1422 sizeof(void*));
1423 spin_unlock_bh(&session->lock);
1424 }
1425 return 0;
1426 }
1427
1428 static inline int
1429 handle_xmstate_r_hdr(struct iscsi_conn *conn,
1430 struct iscsi_tcp_cmd_task *tcp_ctask)
1431 {
1432 tcp_ctask->xmstate &= ~XMSTATE_R_HDR;
1433 if (conn->hdrdgst_en)
1434 iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
1435 (u8*)tcp_ctask->hdrext);
1436 if (!iscsi_sendhdr(conn, &tcp_ctask->headbuf, 0)) {
1437 BUG_ON(tcp_ctask->xmstate != XMSTATE_IDLE);
1438 return 0; /* wait for Data-In */
1439 }
1440 tcp_ctask->xmstate |= XMSTATE_R_HDR;
1441 return -EAGAIN;
1442 }
1443
1444 static inline int
1445 handle_xmstate_w_hdr(struct iscsi_conn *conn,
1446 struct iscsi_cmd_task *ctask)
1447 {
1448 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1449
1450 tcp_ctask->xmstate &= ~XMSTATE_W_HDR;
1451 if (conn->hdrdgst_en)
1452 iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
1453 (u8*)tcp_ctask->hdrext);
1454 if (iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->imm_count)) {
1455 tcp_ctask->xmstate |= XMSTATE_W_HDR;
1456 return -EAGAIN;
1457 }
1458 return 0;
1459 }
1460
1461 static inline int
1462 handle_xmstate_data_digest(struct iscsi_conn *conn,
1463 struct iscsi_cmd_task *ctask)
1464 {
1465 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1466
1467 tcp_ctask->xmstate &= ~XMSTATE_DATA_DIGEST;
1468 debug_tcp("resent data digest 0x%x\n", tcp_ctask->datadigest);
1469 if (iscsi_digest_final_send(conn, ctask, &tcp_ctask->immbuf,
1470 &tcp_ctask->datadigest, 0)) {
1471 tcp_ctask->xmstate |= XMSTATE_DATA_DIGEST;
1472 debug_tcp("resent data digest 0x%x fail!\n",
1473 tcp_ctask->datadigest);
1474 return -EAGAIN;
1475 }
1476 return 0;
1477 }
1478
1479 static inline int
1480 handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1481 {
1482 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1483 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1484
1485 BUG_ON(!ctask->imm_count);
1486 tcp_ctask->xmstate &= ~XMSTATE_IMM_DATA;
1487
1488 if (conn->datadgst_en) {
1489 iscsi_data_digest_init(tcp_conn, ctask);
1490 tcp_ctask->immdigest = 0;
1491 }
1492
1493 for (;;) {
1494 if (iscsi_sendpage(conn, &tcp_ctask->sendbuf, &ctask->imm_count,
1495 &tcp_ctask->sent)) {
1496 tcp_ctask->xmstate |= XMSTATE_IMM_DATA;
1497 if (conn->datadgst_en) {
1498 crypto_digest_final(tcp_conn->data_tx_tfm,
1499 (u8*)&tcp_ctask->immdigest);
1500 debug_tcp("tx imm sendpage fail 0x%x\n",
1501 tcp_ctask->datadigest);
1502 }
1503 return -EAGAIN;
1504 }
1505 if (conn->datadgst_en)
1506 crypto_digest_update(tcp_conn->data_tx_tfm,
1507 &tcp_ctask->sendbuf.sg, 1);
1508
1509 if (!ctask->imm_count)
1510 break;
1511 iscsi_buf_init_sg(&tcp_ctask->sendbuf,
1512 &tcp_ctask->sg[tcp_ctask->sg_count++]);
1513 }
1514
1515 if (conn->datadgst_en && !(tcp_ctask->xmstate & XMSTATE_W_PAD)) {
1516 if (iscsi_digest_final_send(conn, ctask, &tcp_ctask->immbuf,
1517 &tcp_ctask->immdigest, 1)) {
1518 debug_tcp("sending imm digest 0x%x fail!\n",
1519 tcp_ctask->immdigest);
1520 return -EAGAIN;
1521 }
1522 debug_tcp("sending imm digest 0x%x\n", tcp_ctask->immdigest);
1523 }
1524
1525 return 0;
1526 }
1527
1528 static inline int
1529 handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1530 {
1531 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1532 struct iscsi_data_task *dtask;
1533
1534 tcp_ctask->xmstate |= XMSTATE_UNS_DATA;
1535 if (tcp_ctask->xmstate & XMSTATE_UNS_INIT) {
1536 iscsi_unsolicit_data_init(conn, ctask);
1537 BUG_ON(!tcp_ctask->dtask);
1538 dtask = tcp_ctask->dtask;
1539 if (conn->hdrdgst_en)
1540 iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
1541 (u8*)dtask->hdrext);
1542 tcp_ctask->xmstate &= ~XMSTATE_UNS_INIT;
1543 }
1544 if (iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->data_count)) {
1545 tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA;
1546 tcp_ctask->xmstate |= XMSTATE_UNS_HDR;
1547 return -EAGAIN;
1548 }
1549
1550 debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",
1551 ctask->itt, ctask->unsol_count, tcp_ctask->sent);
1552 return 0;
1553 }
1554
1555 static inline int
1556 handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1557 {
1558 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1559 struct iscsi_data_task *dtask = tcp_ctask->dtask;
1560 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1561
1562 BUG_ON(!ctask->data_count);
1563 tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA;
1564
1565 if (conn->datadgst_en) {
1566 iscsi_data_digest_init(tcp_conn, ctask);
1567 dtask->digest = 0;
1568 }
1569
1570 for (;;) {
1571 int start = tcp_ctask->sent;
1572
1573 if (iscsi_sendpage(conn, &tcp_ctask->sendbuf,
1574 &ctask->data_count, &tcp_ctask->sent)) {
1575 ctask->unsol_count -= tcp_ctask->sent - start;
1576 tcp_ctask->xmstate |= XMSTATE_UNS_DATA;
1577 /* will continue with this ctask later.. */
1578 if (conn->datadgst_en) {
1579 crypto_digest_final(tcp_conn->data_tx_tfm,
1580 (u8 *)&dtask->digest);
1581 debug_tcp("tx uns data fail 0x%x\n",
1582 dtask->digest);
1583 }
1584 return -EAGAIN;
1585 }
1586
1587 BUG_ON(tcp_ctask->sent > ctask->total_length);
1588 ctask->unsol_count -= tcp_ctask->sent - start;
1589
1590 /*
1591 * XXX:we may run here with un-initial sendbuf.
1592 * so pass it
1593 */
1594 if (conn->datadgst_en && tcp_ctask->sent - start > 0)
1595 crypto_digest_update(tcp_conn->data_tx_tfm,
1596 &tcp_ctask->sendbuf.sg, 1);
1597
1598 if (!ctask->data_count)
1599 break;
1600 iscsi_buf_init_sg(&tcp_ctask->sendbuf,
1601 &tcp_ctask->sg[tcp_ctask->sg_count++]);
1602 }
1603 BUG_ON(ctask->unsol_count < 0);
1604
1605 /*
1606 * Done with the Data-Out. Next, check if we need
1607 * to send another unsolicited Data-Out.
1608 */
1609 if (ctask->unsol_count) {
1610 if (conn->datadgst_en) {
1611 if (iscsi_digest_final_send(conn, ctask,
1612 &dtask->digestbuf,
1613 &dtask->digest, 1)) {
1614 debug_tcp("send uns digest 0x%x fail\n",
1615 dtask->digest);
1616 return -EAGAIN;
1617 }
1618 debug_tcp("sending uns digest 0x%x, more uns\n",
1619 dtask->digest);
1620 }
1621 tcp_ctask->xmstate |= XMSTATE_UNS_INIT;
1622 return 1;
1623 }
1624
1625 if (conn->datadgst_en && !(tcp_ctask->xmstate & XMSTATE_W_PAD)) {
1626 if (iscsi_digest_final_send(conn, ctask,
1627 &dtask->digestbuf,
1628 &dtask->digest, 1)) {
1629 debug_tcp("send last uns digest 0x%x fail\n",
1630 dtask->digest);
1631 return -EAGAIN;
1632 }
1633 debug_tcp("sending uns digest 0x%x\n",dtask->digest);
1634 }
1635
1636 return 0;
1637 }
1638
1639 static inline int
1640 handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1641 {
1642 struct iscsi_session *session = conn->session;
1643 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1644 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1645 struct iscsi_r2t_info *r2t = tcp_ctask->r2t;
1646 struct iscsi_data_task *dtask = r2t->dtask;
1647 int left;
1648
1649 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
1650 tcp_ctask->dtask = dtask;
1651
1652 if (conn->datadgst_en) {
1653 iscsi_data_digest_init(tcp_conn, ctask);
1654 dtask->digest = 0;
1655 }
1656 solicit_again:
1657 /*
1658 * send Data-Out whitnin this R2T sequence.
1659 */
1660 if (!r2t->data_count)
1661 goto data_out_done;
1662
1663 if (iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent)) {
1664 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1665 /* will continue with this ctask later.. */
1666 if (conn->datadgst_en) {
1667 crypto_digest_final(tcp_conn->data_tx_tfm,
1668 (u8 *)&dtask->digest);
1669 debug_tcp("r2t data send fail 0x%x\n", dtask->digest);
1670 }
1671 return -EAGAIN;
1672 }
1673
1674 BUG_ON(r2t->data_count < 0);
1675 if (conn->datadgst_en)
1676 crypto_digest_update(tcp_conn->data_tx_tfm, &r2t->sendbuf.sg,
1677 1);
1678
1679 if (r2t->data_count) {
1680 BUG_ON(ctask->sc->use_sg == 0);
1681 if (!iscsi_buf_left(&r2t->sendbuf)) {
1682 BUG_ON(tcp_ctask->bad_sg == r2t->sg);
1683 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1684 r2t->sg += 1;
1685 }
1686 goto solicit_again;
1687 }
1688
1689 data_out_done:
1690 /*
1691 * Done with this Data-Out. Next, check if we have
1692 * to send another Data-Out for this R2T.
1693 */
1694 BUG_ON(r2t->data_length - r2t->sent < 0);
1695 left = r2t->data_length - r2t->sent;
1696 if (left) {
1697 if (conn->datadgst_en) {
1698 if (iscsi_digest_final_send(conn, ctask,
1699 &dtask->digestbuf,
1700 &dtask->digest, 1)) {
1701 debug_tcp("send r2t data digest 0x%x"
1702 "fail\n", dtask->digest);
1703 return -EAGAIN;
1704 }
1705 debug_tcp("r2t data send digest 0x%x\n",
1706 dtask->digest);
1707 }
1708 iscsi_solicit_data_cont(conn, ctask, r2t, left);
1709 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1710 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1711 return 1;
1712 }
1713
1714 /*
1715 * Done with this R2T. Check if there are more
1716 * outstanding R2Ts ready to be processed.
1717 */
1718 BUG_ON(tcp_ctask->r2t_data_count - r2t->data_length < 0);
1719 if (conn->datadgst_en) {
1720 if (iscsi_digest_final_send(conn, ctask, &dtask->digestbuf,
1721 &dtask->digest, 1)) {
1722 debug_tcp("send last r2t data digest 0x%x"
1723 "fail\n", dtask->digest);
1724 return -EAGAIN;
1725 }
1726 debug_tcp("r2t done dout digest 0x%x\n", dtask->digest);
1727 }
1728
1729 tcp_ctask->r2t_data_count -= r2t->data_length;
1730 tcp_ctask->r2t = NULL;
1731 spin_lock_bh(&session->lock);
1732 __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
1733 spin_unlock_bh(&session->lock);
1734 if (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
1735 tcp_ctask->r2t = r2t;
1736 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1737 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1738 return 1;
1739 }
1740
1741 return 0;
1742 }
1743
1744 static inline int
1745 handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1746 {
1747 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1748 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1749 struct iscsi_data_task *dtask = tcp_ctask->dtask;
1750 int sent;
1751
1752 tcp_ctask->xmstate &= ~XMSTATE_W_PAD;
1753 iscsi_buf_init_iov(&tcp_ctask->sendbuf, (char*)&tcp_ctask->pad,
1754 tcp_ctask->pad_count);
1755 if (iscsi_sendpage(conn, &tcp_ctask->sendbuf, &tcp_ctask->pad_count,
1756 &sent)) {
1757 tcp_ctask->xmstate |= XMSTATE_W_PAD;
1758 return -EAGAIN;
1759 }
1760
1761 if (conn->datadgst_en) {
1762 crypto_digest_update(tcp_conn->data_tx_tfm,
1763 &tcp_ctask->sendbuf.sg, 1);
1764 /* imm data? */
1765 if (!dtask) {
1766 if (iscsi_digest_final_send(conn, ctask,
1767 &tcp_ctask->immbuf,
1768 &tcp_ctask->immdigest, 1)) {
1769 debug_tcp("send padding digest 0x%x"
1770 "fail!\n", tcp_ctask->immdigest);
1771 return -EAGAIN;
1772 }
1773 debug_tcp("done with padding, digest 0x%x\n",
1774 tcp_ctask->datadigest);
1775 } else {
1776 if (iscsi_digest_final_send(conn, ctask,
1777 &dtask->digestbuf,
1778 &dtask->digest, 1)) {
1779 debug_tcp("send padding digest 0x%x"
1780 "fail\n", dtask->digest);
1781 return -EAGAIN;
1782 }
1783 debug_tcp("done with padding, digest 0x%x\n",
1784 dtask->digest);
1785 }
1786 }
1787
1788 return 0;
1789 }
1790
1791 static int
1792 iscsi_tcp_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1793 {
1794 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1795 int rc = 0;
1796
1797 debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n",
1798 conn->id, tcp_ctask->xmstate, ctask->itt);
1799
1800 /*
1801 * serialize with TMF AbortTask
1802 */
1803 if (ctask->mtask)
1804 return rc;
1805
1806 if (tcp_ctask->xmstate & XMSTATE_R_HDR) {
1807 rc = handle_xmstate_r_hdr(conn, tcp_ctask);
1808 return rc;
1809 }
1810
1811 if (tcp_ctask->xmstate & XMSTATE_W_HDR) {
1812 rc = handle_xmstate_w_hdr(conn, ctask);
1813 if (rc)
1814 return rc;
1815 }
1816
1817 /* XXX: for data digest xmit recover */
1818 if (tcp_ctask->xmstate & XMSTATE_DATA_DIGEST) {
1819 rc = handle_xmstate_data_digest(conn, ctask);
1820 if (rc)
1821 return rc;
1822 }
1823
1824 if (tcp_ctask->xmstate & XMSTATE_IMM_DATA) {
1825 rc = handle_xmstate_imm_data(conn, ctask);
1826 if (rc)
1827 return rc;
1828 }
1829
1830 if (tcp_ctask->xmstate & XMSTATE_UNS_HDR) {
1831 BUG_ON(!ctask->unsol_count);
1832 tcp_ctask->xmstate &= ~XMSTATE_UNS_HDR;
1833 unsolicit_head_again:
1834 rc = handle_xmstate_uns_hdr(conn, ctask);
1835 if (rc)
1836 return rc;
1837 }
1838
1839 if (tcp_ctask->xmstate & XMSTATE_UNS_DATA) {
1840 rc = handle_xmstate_uns_data(conn, ctask);
1841 if (rc == 1)
1842 goto unsolicit_head_again;
1843 else if (rc)
1844 return rc;
1845 goto done;
1846 }
1847
1848 if (tcp_ctask->xmstate & XMSTATE_SOL_HDR) {
1849 struct iscsi_r2t_info *r2t;
1850
1851 tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1852 tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1853 if (!tcp_ctask->r2t)
1854 __kfifo_get(tcp_ctask->r2tqueue, (void*)&tcp_ctask->r2t,
1855 sizeof(void*));
1856 solicit_head_again:
1857 r2t = tcp_ctask->r2t;
1858 if (conn->hdrdgst_en)
1859 iscsi_hdr_digest(conn, &r2t->headbuf,
1860 (u8*)r2t->dtask->hdrext);
1861 if (iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count)) {
1862 tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
1863 tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
1864 return -EAGAIN;
1865 }
1866
1867 debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
1868 r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,
1869 r2t->sent);
1870 }
1871
1872 if (tcp_ctask->xmstate & XMSTATE_SOL_DATA) {
1873 rc = handle_xmstate_sol_data(conn, ctask);
1874 if (rc == 1)
1875 goto solicit_head_again;
1876 if (rc)
1877 return rc;
1878 }
1879
1880 done:
1881 /*
1882 * Last thing to check is whether we need to send write
1883 * padding. Note that we check for xmstate equality, not just the bit.
1884 */
1885 if (tcp_ctask->xmstate == XMSTATE_W_PAD)
1886 rc = handle_xmstate_w_pad(conn, ctask);
1887
1888 return rc;
1889 }
1890
1891 static struct iscsi_cls_conn *
1892 iscsi_tcp_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx)
1893 {
1894 struct iscsi_conn *conn;
1895 struct iscsi_cls_conn *cls_conn;
1896 struct iscsi_tcp_conn *tcp_conn;
1897
1898 cls_conn = iscsi_conn_setup(cls_session, conn_idx);
1899 if (!cls_conn)
1900 return NULL;
1901 conn = cls_conn->dd_data;
1902 /*
1903 * due to strange issues with iser these are not set
1904 * in iscsi_conn_setup
1905 */
1906 conn->max_recv_dlength = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
1907
1908 tcp_conn = kzalloc(sizeof(*tcp_conn), GFP_KERNEL);
1909 if (!tcp_conn)
1910 goto tcp_conn_alloc_fail;
1911
1912 conn->dd_data = tcp_conn;
1913 tcp_conn->iscsi_conn = conn;
1914 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1915 /* initial operational parameters */
1916 tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
1917 tcp_conn->data_size = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
1918
1919 /* allocate initial PDU receive place holder */
1920 if (tcp_conn->data_size <= PAGE_SIZE)
1921 tcp_conn->data = kmalloc(tcp_conn->data_size, GFP_KERNEL);
1922 else
1923 tcp_conn->data = (void*)__get_free_pages(GFP_KERNEL,
1924 get_order(tcp_conn->data_size));
1925 if (!tcp_conn->data)
1926 goto max_recv_dlenght_alloc_fail;
1927
1928 return cls_conn;
1929
1930 max_recv_dlenght_alloc_fail:
1931 kfree(tcp_conn);
1932 tcp_conn_alloc_fail:
1933 iscsi_conn_teardown(cls_conn);
1934 return NULL;
1935 }
1936
1937 static void
1938 iscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
1939 {
1940 struct iscsi_conn *conn = cls_conn->dd_data;
1941 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1942 int digest = 0;
1943
1944 if (conn->hdrdgst_en || conn->datadgst_en)
1945 digest = 1;
1946
1947 iscsi_conn_teardown(cls_conn);
1948
1949 /* now free tcp_conn */
1950 if (digest) {
1951 if (tcp_conn->tx_tfm)
1952 crypto_free_tfm(tcp_conn->tx_tfm);
1953 if (tcp_conn->rx_tfm)
1954 crypto_free_tfm(tcp_conn->rx_tfm);
1955 if (tcp_conn->data_tx_tfm)
1956 crypto_free_tfm(tcp_conn->data_tx_tfm);
1957 if (tcp_conn->data_rx_tfm)
1958 crypto_free_tfm(tcp_conn->data_rx_tfm);
1959 }
1960
1961 /* free conn->data, size = MaxRecvDataSegmentLength */
1962 if (tcp_conn->data_size <= PAGE_SIZE)
1963 kfree(tcp_conn->data);
1964 else
1965 free_pages((unsigned long)tcp_conn->data,
1966 get_order(tcp_conn->data_size));
1967 kfree(tcp_conn);
1968 }
1969
1970 static int
1971 iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session,
1972 struct iscsi_cls_conn *cls_conn, uint64_t transport_eph,
1973 int is_leading)
1974 {
1975 struct iscsi_conn *conn = cls_conn->dd_data;
1976 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1977 struct sock *sk;
1978 struct socket *sock;
1979 int err;
1980
1981 /* lookup for existing socket */
1982 sock = sockfd_lookup((int)transport_eph, &err);
1983 if (!sock) {
1984 printk(KERN_ERR "iscsi_tcp: sockfd_lookup failed %d\n", err);
1985 return -EEXIST;
1986 }
1987
1988 err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
1989 if (err)
1990 return err;
1991
1992 if (conn->stop_stage != STOP_CONN_SUSPEND) {
1993 /* bind iSCSI connection and socket */
1994 tcp_conn->sock = sock;
1995
1996 /* setup Socket parameters */
1997 sk = sock->sk;
1998 sk->sk_reuse = 1;
1999 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
2000 sk->sk_allocation = GFP_ATOMIC;
2001
2002 /* FIXME: disable Nagle's algorithm */
2003
2004 /*
2005 * Intercept TCP callbacks for sendfile like receive
2006 * processing.
2007 */
2008 conn->recv_lock = &sk->sk_callback_lock;
2009 iscsi_conn_set_callbacks(conn);
2010 tcp_conn->sendpage = tcp_conn->sock->ops->sendpage;
2011 /*
2012 * set receive state machine into initial state
2013 */
2014 tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2015 }
2016
2017 return 0;
2018 }
2019
2020 static void
2021 iscsi_tcp_cleanup_ctask(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2022 {
2023 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2024 struct iscsi_r2t_info *r2t;
2025
2026 /* flush ctask's r2t queues */
2027 while (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*)))
2028 __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t,
2029 sizeof(void*));
2030
2031 __iscsi_ctask_cleanup(conn, ctask);
2032 }
2033
2034 static void
2035 iscsi_tcp_suspend_conn_rx(struct iscsi_conn *conn)
2036 {
2037 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2038 struct sock *sk;
2039
2040 if (!tcp_conn->sock)
2041 return;
2042
2043 sk = tcp_conn->sock->sk;
2044 write_lock_bh(&sk->sk_callback_lock);
2045 set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx);
2046 write_unlock_bh(&sk->sk_callback_lock);
2047 }
2048
2049 static void
2050 iscsi_tcp_terminate_conn(struct iscsi_conn *conn)
2051 {
2052 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2053
2054 if (!tcp_conn->sock)
2055 return;
2056
2057 sock_hold(tcp_conn->sock->sk);
2058 iscsi_conn_restore_callbacks(conn);
2059 sock_put(tcp_conn->sock->sk);
2060
2061 sock_release(tcp_conn->sock);
2062 tcp_conn->sock = NULL;
2063 conn->recv_lock = NULL;
2064 }
2065
2066 /* called with host lock */
2067 static void
2068 iscsi_tcp_mgmt_init(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask,
2069 char *data, uint32_t data_size)
2070 {
2071 struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
2072
2073 iscsi_buf_init_iov(&tcp_mtask->headbuf, (char*)mtask->hdr,
2074 sizeof(struct iscsi_hdr));
2075 tcp_mtask->xmstate = XMSTATE_IMM_HDR;
2076
2077 if (mtask->data_count)
2078 iscsi_buf_init_iov(&tcp_mtask->sendbuf, (char*)mtask->data,
2079 mtask->data_count);
2080 }
2081
2082 static int
2083 iscsi_r2tpool_alloc(struct iscsi_session *session)
2084 {
2085 int i;
2086 int cmd_i;
2087
2088 /*
2089 * initialize per-task: R2T pool and xmit queue
2090 */
2091 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
2092 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
2093 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2094
2095 /*
2096 * pre-allocated x4 as much r2ts to handle race when
2097 * target acks DataOut faster than we data_xmit() queues
2098 * could replenish r2tqueue.
2099 */
2100
2101 /* R2T pool */
2102 if (iscsi_pool_init(&tcp_ctask->r2tpool, session->max_r2t * 4,
2103 (void***)&tcp_ctask->r2ts,
2104 sizeof(struct iscsi_r2t_info))) {
2105 goto r2t_alloc_fail;
2106 }
2107
2108 /* R2T xmit queue */
2109 tcp_ctask->r2tqueue = kfifo_alloc(
2110 session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL);
2111 if (tcp_ctask->r2tqueue == ERR_PTR(-ENOMEM)) {
2112 iscsi_pool_free(&tcp_ctask->r2tpool,
2113 (void**)tcp_ctask->r2ts);
2114 goto r2t_alloc_fail;
2115 }
2116
2117 /*
2118 * number of
2119 * Data-Out PDU's within R2T-sequence can be quite big;
2120 * using mempool
2121 */
2122 tcp_ctask->datapool = mempool_create_slab_pool(ISCSI_DTASK_DEFAULT_MAX,
2123 taskcache);
2124 if (tcp_ctask->datapool == NULL) {
2125 kfifo_free(tcp_ctask->r2tqueue);
2126 iscsi_pool_free(&tcp_ctask->r2tpool,
2127 (void**)tcp_ctask->r2ts);
2128 goto r2t_alloc_fail;
2129 }
2130 INIT_LIST_HEAD(&tcp_ctask->dataqueue);
2131 }
2132
2133 return 0;
2134
2135 r2t_alloc_fail:
2136 for (i = 0; i < cmd_i; i++) {
2137 struct iscsi_cmd_task *ctask = session->cmds[i];
2138 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2139
2140 mempool_destroy(tcp_ctask->datapool);
2141 kfifo_free(tcp_ctask->r2tqueue);
2142 iscsi_pool_free(&tcp_ctask->r2tpool,
2143 (void**)tcp_ctask->r2ts);
2144 }
2145 return -ENOMEM;
2146 }
2147
2148 static void
2149 iscsi_r2tpool_free(struct iscsi_session *session)
2150 {
2151 int i;
2152
2153 for (i = 0; i < session->cmds_max; i++) {
2154 struct iscsi_cmd_task *ctask = session->cmds[i];
2155 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2156
2157 mempool_destroy(tcp_ctask->datapool);
2158 kfifo_free(tcp_ctask->r2tqueue);
2159 iscsi_pool_free(&tcp_ctask->r2tpool,
2160 (void**)tcp_ctask->r2ts);
2161 }
2162 }
2163
2164 static int
2165 iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
2166 uint32_t value)
2167 {
2168 struct iscsi_conn *conn = cls_conn->dd_data;
2169 struct iscsi_session *session = conn->session;
2170 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2171
2172 switch(param) {
2173 case ISCSI_PARAM_MAX_RECV_DLENGTH: {
2174 char *saveptr = tcp_conn->data;
2175 gfp_t flags = GFP_KERNEL;
2176
2177 if (tcp_conn->data_size >= value) {
2178 conn->max_recv_dlength = value;
2179 break;
2180 }
2181
2182 spin_lock_bh(&session->lock);
2183 if (conn->stop_stage == STOP_CONN_RECOVER)
2184 flags = GFP_ATOMIC;
2185 spin_unlock_bh(&session->lock);
2186
2187 if (value <= PAGE_SIZE)
2188 tcp_conn->data = kmalloc(value, flags);
2189 else
2190 tcp_conn->data = (void*)__get_free_pages(flags,
2191 get_order(value));
2192 if (tcp_conn->data == NULL) {
2193 tcp_conn->data = saveptr;
2194 return -ENOMEM;
2195 }
2196 if (tcp_conn->data_size <= PAGE_SIZE)
2197 kfree(saveptr);
2198 else
2199 free_pages((unsigned long)saveptr,
2200 get_order(tcp_conn->data_size));
2201 conn->max_recv_dlength = value;
2202 tcp_conn->data_size = value;
2203 }
2204 break;
2205 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
2206 conn->max_xmit_dlength = value;
2207 break;
2208 case ISCSI_PARAM_HDRDGST_EN:
2209 conn->hdrdgst_en = value;
2210 tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
2211 if (conn->hdrdgst_en) {
2212 tcp_conn->hdr_size += sizeof(__u32);
2213 if (!tcp_conn->tx_tfm)
2214 tcp_conn->tx_tfm = crypto_alloc_tfm("crc32c",
2215 0);
2216 if (!tcp_conn->tx_tfm)
2217 return -ENOMEM;
2218 if (!tcp_conn->rx_tfm)
2219 tcp_conn->rx_tfm = crypto_alloc_tfm("crc32c",
2220 0);
2221 if (!tcp_conn->rx_tfm) {
2222 crypto_free_tfm(tcp_conn->tx_tfm);
2223 return -ENOMEM;
2224 }
2225 } else {
2226 if (tcp_conn->tx_tfm)
2227 crypto_free_tfm(tcp_conn->tx_tfm);
2228 if (tcp_conn->rx_tfm)
2229 crypto_free_tfm(tcp_conn->rx_tfm);
2230 }
2231 break;
2232 case ISCSI_PARAM_DATADGST_EN:
2233 conn->datadgst_en = value;
2234 if (conn->datadgst_en) {
2235 if (!tcp_conn->data_tx_tfm)
2236 tcp_conn->data_tx_tfm =
2237 crypto_alloc_tfm("crc32c", 0);
2238 if (!tcp_conn->data_tx_tfm)
2239 return -ENOMEM;
2240 if (!tcp_conn->data_rx_tfm)
2241 tcp_conn->data_rx_tfm =
2242 crypto_alloc_tfm("crc32c", 0);
2243 if (!tcp_conn->data_rx_tfm) {
2244 crypto_free_tfm(tcp_conn->data_tx_tfm);
2245 return -ENOMEM;
2246 }
2247 } else {
2248 if (tcp_conn->data_tx_tfm)
2249 crypto_free_tfm(tcp_conn->data_tx_tfm);
2250 if (tcp_conn->data_rx_tfm)
2251 crypto_free_tfm(tcp_conn->data_rx_tfm);
2252 }
2253 tcp_conn->sendpage = conn->datadgst_en ?
2254 sock_no_sendpage : tcp_conn->sock->ops->sendpage;
2255 break;
2256 case ISCSI_PARAM_INITIAL_R2T_EN:
2257 session->initial_r2t_en = value;
2258 break;
2259 case ISCSI_PARAM_MAX_R2T:
2260 if (session->max_r2t == roundup_pow_of_two(value))
2261 break;
2262 iscsi_r2tpool_free(session);
2263 session->max_r2t = value;
2264 if (session->max_r2t & (session->max_r2t - 1))
2265 session->max_r2t = roundup_pow_of_two(session->max_r2t);
2266 if (iscsi_r2tpool_alloc(session))
2267 return -ENOMEM;
2268 break;
2269 case ISCSI_PARAM_IMM_DATA_EN:
2270 session->imm_data_en = value;
2271 break;
2272 case ISCSI_PARAM_FIRST_BURST:
2273 session->first_burst = value;
2274 break;
2275 case ISCSI_PARAM_MAX_BURST:
2276 session->max_burst = value;
2277 break;
2278 case ISCSI_PARAM_PDU_INORDER_EN:
2279 session->pdu_inorder_en = value;
2280 break;
2281 case ISCSI_PARAM_DATASEQ_INORDER_EN:
2282 session->dataseq_inorder_en = value;
2283 break;
2284 case ISCSI_PARAM_ERL:
2285 session->erl = value;
2286 break;
2287 case ISCSI_PARAM_IFMARKER_EN:
2288 BUG_ON(value);
2289 session->ifmarker_en = value;
2290 break;
2291 case ISCSI_PARAM_OFMARKER_EN:
2292 BUG_ON(value);
2293 session->ofmarker_en = value;
2294 break;
2295 case ISCSI_PARAM_EXP_STATSN:
2296 conn->exp_statsn = value;
2297 break;
2298 default:
2299 break;
2300 }
2301
2302 return 0;
2303 }
2304
2305 static int
2306 iscsi_session_get_param(struct iscsi_cls_session *cls_session,
2307 enum iscsi_param param, uint32_t *value)
2308 {
2309 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
2310 struct iscsi_session *session = iscsi_hostdata(shost->hostdata);
2311
2312 switch(param) {
2313 case ISCSI_PARAM_INITIAL_R2T_EN:
2314 *value = session->initial_r2t_en;
2315 break;
2316 case ISCSI_PARAM_MAX_R2T:
2317 *value = session->max_r2t;
2318 break;
2319 case ISCSI_PARAM_IMM_DATA_EN:
2320 *value = session->imm_data_en;
2321 break;
2322 case ISCSI_PARAM_FIRST_BURST:
2323 *value = session->first_burst;
2324 break;
2325 case ISCSI_PARAM_MAX_BURST:
2326 *value = session->max_burst;
2327 break;
2328 case ISCSI_PARAM_PDU_INORDER_EN:
2329 *value = session->pdu_inorder_en;
2330 break;
2331 case ISCSI_PARAM_DATASEQ_INORDER_EN:
2332 *value = session->dataseq_inorder_en;
2333 break;
2334 case ISCSI_PARAM_ERL:
2335 *value = session->erl;
2336 break;
2337 case ISCSI_PARAM_IFMARKER_EN:
2338 *value = session->ifmarker_en;
2339 break;
2340 case ISCSI_PARAM_OFMARKER_EN:
2341 *value = session->ofmarker_en;
2342 break;
2343 default:
2344 return -EINVAL;
2345 }
2346
2347 return 0;
2348 }
2349
2350 static int
2351 iscsi_conn_get_param(struct iscsi_cls_conn *cls_conn,
2352 enum iscsi_param param, uint32_t *value)
2353 {
2354 struct iscsi_conn *conn = cls_conn->dd_data;
2355 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2356 struct inet_sock *inet;
2357
2358 switch(param) {
2359 case ISCSI_PARAM_MAX_RECV_DLENGTH:
2360 *value = conn->max_recv_dlength;
2361 break;
2362 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
2363 *value = conn->max_xmit_dlength;
2364 break;
2365 case ISCSI_PARAM_HDRDGST_EN:
2366 *value = conn->hdrdgst_en;
2367 break;
2368 case ISCSI_PARAM_DATADGST_EN:
2369 *value = conn->datadgst_en;
2370 break;
2371 case ISCSI_PARAM_CONN_PORT:
2372 mutex_lock(&conn->xmitmutex);
2373 if (!tcp_conn->sock) {
2374 mutex_unlock(&conn->xmitmutex);
2375 return -EINVAL;
2376 }
2377
2378 inet = inet_sk(tcp_conn->sock->sk);
2379 *value = be16_to_cpu(inet->dport);
2380 mutex_unlock(&conn->xmitmutex);
2381 case ISCSI_PARAM_EXP_STATSN:
2382 *value = conn->exp_statsn;
2383 break;
2384 default:
2385 return -EINVAL;
2386 }
2387
2388 return 0;
2389 }
2390
2391 static int
2392 iscsi_conn_get_str_param(struct iscsi_cls_conn *cls_conn,
2393 enum iscsi_param param, char *buf)
2394 {
2395 struct iscsi_conn *conn = cls_conn->dd_data;
2396 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2397 struct sock *sk;
2398 struct inet_sock *inet;
2399 struct ipv6_pinfo *np;
2400 int len = 0;
2401
2402 switch (param) {
2403 case ISCSI_PARAM_CONN_ADDRESS:
2404 mutex_lock(&conn->xmitmutex);
2405 if (!tcp_conn->sock) {
2406 mutex_unlock(&conn->xmitmutex);
2407 return -EINVAL;
2408 }
2409
2410 sk = tcp_conn->sock->sk;
2411 if (sk->sk_family == PF_INET) {
2412 inet = inet_sk(sk);
2413 len = sprintf(buf, "%u.%u.%u.%u\n",
2414 NIPQUAD(inet->daddr));
2415 } else {
2416 np = inet6_sk(sk);
2417 len = sprintf(buf,
2418 "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n",
2419 NIP6(np->daddr));
2420 }
2421 mutex_unlock(&conn->xmitmutex);
2422 break;
2423 default:
2424 return -EINVAL;
2425 }
2426
2427 return len;
2428 }
2429
2430 static void
2431 iscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats)
2432 {
2433 struct iscsi_conn *conn = cls_conn->dd_data;
2434 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2435
2436 stats->txdata_octets = conn->txdata_octets;
2437 stats->rxdata_octets = conn->rxdata_octets;
2438 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
2439 stats->dataout_pdus = conn->dataout_pdus_cnt;
2440 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
2441 stats->datain_pdus = conn->datain_pdus_cnt;
2442 stats->r2t_pdus = conn->r2t_pdus_cnt;
2443 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
2444 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
2445 stats->custom_length = 3;
2446 strcpy(stats->custom[0].desc, "tx_sendpage_failures");
2447 stats->custom[0].value = tcp_conn->sendpage_failures_cnt;
2448 strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
2449 stats->custom[1].value = tcp_conn->discontiguous_hdr_cnt;
2450 strcpy(stats->custom[2].desc, "eh_abort_cnt");
2451 stats->custom[2].value = conn->eh_abort_cnt;
2452 }
2453
2454 static struct iscsi_cls_session *
2455 iscsi_tcp_session_create(struct iscsi_transport *iscsit,
2456 struct scsi_transport_template *scsit,
2457 uint32_t initial_cmdsn, uint32_t *hostno)
2458 {
2459 struct iscsi_cls_session *cls_session;
2460 struct iscsi_session *session;
2461 uint32_t hn;
2462 int cmd_i;
2463
2464 cls_session = iscsi_session_setup(iscsit, scsit,
2465 sizeof(struct iscsi_tcp_cmd_task),
2466 sizeof(struct iscsi_tcp_mgmt_task),
2467 initial_cmdsn, &hn);
2468 if (!cls_session)
2469 return NULL;
2470 *hostno = hn;
2471
2472 session = class_to_transport_session(cls_session);
2473 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
2474 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
2475 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2476
2477 ctask->hdr = &tcp_ctask->hdr;
2478 }
2479
2480 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) {
2481 struct iscsi_mgmt_task *mtask = session->mgmt_cmds[cmd_i];
2482 struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
2483
2484 mtask->hdr = &tcp_mtask->hdr;
2485 }
2486
2487 if (iscsi_r2tpool_alloc(class_to_transport_session(cls_session)))
2488 goto r2tpool_alloc_fail;
2489
2490 return cls_session;
2491
2492 r2tpool_alloc_fail:
2493 iscsi_session_teardown(cls_session);
2494 return NULL;
2495 }
2496
2497 static void iscsi_tcp_session_destroy(struct iscsi_cls_session *cls_session)
2498 {
2499 struct iscsi_session *session = class_to_transport_session(cls_session);
2500 struct iscsi_data_task *dtask, *n;
2501 int cmd_i;
2502
2503 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
2504 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
2505 struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
2506
2507 list_for_each_entry_safe(dtask, n, &tcp_ctask->dataqueue,
2508 item) {
2509 list_del(&dtask->item);
2510 mempool_free(dtask, tcp_ctask->datapool);
2511 }
2512 }
2513
2514 iscsi_r2tpool_free(class_to_transport_session(cls_session));
2515 iscsi_session_teardown(cls_session);
2516 }
2517
2518 static struct scsi_host_template iscsi_sht = {
2519 .name = "iSCSI Initiator over TCP/IP, v."
2520 ISCSI_VERSION_STR,
2521 .queuecommand = iscsi_queuecommand,
2522 .change_queue_depth = iscsi_change_queue_depth,
2523 .can_queue = ISCSI_XMIT_CMDS_MAX - 1,
2524 .sg_tablesize = ISCSI_SG_TABLESIZE,
2525 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
2526 .eh_abort_handler = iscsi_eh_abort,
2527 .eh_host_reset_handler = iscsi_eh_host_reset,
2528 .use_clustering = DISABLE_CLUSTERING,
2529 .proc_name = "iscsi_tcp",
2530 .this_id = -1,
2531 };
2532
2533 static struct iscsi_transport iscsi_tcp_transport = {
2534 .owner = THIS_MODULE,
2535 .name = "tcp",
2536 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
2537 | CAP_DATADGST,
2538 .param_mask = ISCSI_MAX_RECV_DLENGTH |
2539 ISCSI_MAX_XMIT_DLENGTH |
2540 ISCSI_HDRDGST_EN |
2541 ISCSI_DATADGST_EN |
2542 ISCSI_INITIAL_R2T_EN |
2543 ISCSI_MAX_R2T |
2544 ISCSI_IMM_DATA_EN |
2545 ISCSI_FIRST_BURST |
2546 ISCSI_MAX_BURST |
2547 ISCSI_PDU_INORDER_EN |
2548 ISCSI_DATASEQ_INORDER_EN |
2549 ISCSI_ERL |
2550 ISCSI_CONN_PORT |
2551 ISCSI_CONN_ADDRESS |
2552 ISCSI_EXP_STATSN,
2553 .host_template = &iscsi_sht,
2554 .conndata_size = sizeof(struct iscsi_conn),
2555 .max_conn = 1,
2556 .max_cmd_len = ISCSI_TCP_MAX_CMD_LEN,
2557 /* session management */
2558 .create_session = iscsi_tcp_session_create,
2559 .destroy_session = iscsi_tcp_session_destroy,
2560 /* connection management */
2561 .create_conn = iscsi_tcp_conn_create,
2562 .bind_conn = iscsi_tcp_conn_bind,
2563 .destroy_conn = iscsi_tcp_conn_destroy,
2564 .set_param = iscsi_conn_set_param,
2565 .get_conn_param = iscsi_conn_get_param,
2566 .get_conn_str_param = iscsi_conn_get_str_param,
2567 .get_session_param = iscsi_session_get_param,
2568 .start_conn = iscsi_conn_start,
2569 .stop_conn = iscsi_conn_stop,
2570 /* these are called as part of conn recovery */
2571 .suspend_conn_recv = iscsi_tcp_suspend_conn_rx,
2572 .terminate_conn = iscsi_tcp_terminate_conn,
2573 /* IO */
2574 .send_pdu = iscsi_conn_send_pdu,
2575 .get_stats = iscsi_conn_get_stats,
2576 .init_cmd_task = iscsi_tcp_cmd_init,
2577 .init_mgmt_task = iscsi_tcp_mgmt_init,
2578 .xmit_cmd_task = iscsi_tcp_ctask_xmit,
2579 .xmit_mgmt_task = iscsi_tcp_mtask_xmit,
2580 .cleanup_cmd_task = iscsi_tcp_cleanup_ctask,
2581 /* recovery */
2582 .session_recovery_timedout = iscsi_session_recovery_timedout,
2583 };
2584
2585 static int __init
2586 iscsi_tcp_init(void)
2587 {
2588 if (iscsi_max_lun < 1) {
2589 printk(KERN_ERR "iscsi_tcp: Invalid max_lun value of %u\n",
2590 iscsi_max_lun);
2591 return -EINVAL;
2592 }
2593 iscsi_tcp_transport.max_lun = iscsi_max_lun;
2594
2595 taskcache = kmem_cache_create("iscsi_taskcache",
2596 sizeof(struct iscsi_data_task), 0,
2597 SLAB_HWCACHE_ALIGN, NULL, NULL);
2598 if (!taskcache)
2599 return -ENOMEM;
2600
2601 if (!iscsi_register_transport(&iscsi_tcp_transport))
2602 kmem_cache_destroy(taskcache);
2603
2604 return 0;
2605 }
2606
2607 static void __exit
2608 iscsi_tcp_exit(void)
2609 {
2610 iscsi_unregister_transport(&iscsi_tcp_transport);
2611 kmem_cache_destroy(taskcache);
2612 }
2613
2614 module_init(iscsi_tcp_init);
2615 module_exit(iscsi_tcp_exit);
kernel-based virtual machine