2 * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management
4 * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
6 * This program is distributed in the hope that it will be useful, but WITHOUT
7 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
8 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
9 * release for licensing terms and conditions.
11 * Written by: Dimitris Michailidis (dm@chelsio.com)
12 * Karen Xie (kxie@chelsio.com)
15 #include <linux/if_vlan.h>
16 #include <linux/slab.h>
17 #include <linux/version.h>
19 #include "cxgb3_defs.h"
20 #include "cxgb3_ctl_defs.h"
21 #include "firmware_exports.h"
22 #include "cxgb3i_offload.h"
23 #include "cxgb3i_pdu.h"
24 #include "cxgb3i_ddp.h"
26 #ifdef __DEBUG_C3CN_CONN__
27 #define c3cn_conn_debug cxgb3i_log_debug
29 #define c3cn_conn_debug(fmt...)
32 #ifdef __DEBUG_C3CN_TX__
33 #define c3cn_tx_debug cxgb3i_log_debug
35 #define c3cn_tx_debug(fmt...)
38 #ifdef __DEBUG_C3CN_RX__
39 #define c3cn_rx_debug cxgb3i_log_debug
41 #define c3cn_rx_debug(fmt...)
45 * module parameters releated to offloaded iscsi connection
47 static int cxgb3_rcv_win
= 256 * 1024;
48 module_param(cxgb3_rcv_win
, int, 0644);
49 MODULE_PARM_DESC(cxgb3_rcv_win
, "TCP receive window in bytes (default=256KB)");
51 static int cxgb3_snd_win
= 128 * 1024;
52 module_param(cxgb3_snd_win
, int, 0644);
53 MODULE_PARM_DESC(cxgb3_snd_win
, "TCP send window in bytes (default=128KB)");
55 static int cxgb3_rx_credit_thres
= 10 * 1024;
56 module_param(cxgb3_rx_credit_thres
, int, 0644);
57 MODULE_PARM_DESC(rx_credit_thres
,
58 "RX credits return threshold in bytes (default=10KB)");
60 static unsigned int cxgb3_max_connect
= 8 * 1024;
61 module_param(cxgb3_max_connect
, uint
, 0644);
62 MODULE_PARM_DESC(cxgb3_max_connect
, "Max. # of connections (default=8092)");
64 static unsigned int cxgb3_sport_base
= 20000;
65 module_param(cxgb3_sport_base
, uint
, 0644);
66 MODULE_PARM_DESC(cxgb3_sport_base
, "starting port number (default=20000)");
69 * cxgb3i tcp connection data(per adapter) list
71 static LIST_HEAD(cdata_list
);
72 static DEFINE_RWLOCK(cdata_rwlock
);
74 static int c3cn_push_tx_frames(struct s3_conn
*c3cn
, int req_completion
);
75 static void c3cn_release_offload_resources(struct s3_conn
*c3cn
);
78 * iscsi source port management
80 * Find a free source port in the port allocation map. We use a very simple
81 * rotor scheme to look for the next free port.
83 * If a source port has been specified make sure that it doesn't collide with
84 * our normal source port allocation map. If it's outside the range of our
85 * allocation/deallocation scheme just let them use it.
87 * If the source port is outside our allocation range, the caller is
88 * responsible for keeping track of their port usage.
90 static int c3cn_get_port(struct s3_conn
*c3cn
, struct cxgb3i_sdev_data
*cdata
)
98 if (c3cn
->saddr
.sin_port
) {
99 cxgb3i_log_error("connect, sin_port NON-ZERO %u.\n",
100 c3cn
->saddr
.sin_port
);
104 spin_lock_bh(&cdata
->lock
);
105 start
= idx
= cdata
->sport_next
;
107 if (++idx
>= cxgb3_max_connect
)
109 if (!cdata
->sport_conn
[idx
]) {
110 c3cn
->saddr
.sin_port
= htons(cxgb3_sport_base
+ idx
);
111 cdata
->sport_next
= idx
;
112 cdata
->sport_conn
[idx
] = c3cn
;
113 spin_unlock_bh(&cdata
->lock
);
115 c3cn_conn_debug("%s reserve port %u.\n",
117 cxgb3_sport_base
+ idx
);
120 } while (idx
!= start
);
121 spin_unlock_bh(&cdata
->lock
);
124 return -EADDRNOTAVAIL
;
127 static void c3cn_put_port(struct s3_conn
*c3cn
)
132 if (c3cn
->saddr
.sin_port
) {
133 struct cxgb3i_sdev_data
*cdata
= CXGB3_SDEV_DATA(c3cn
->cdev
);
134 int idx
= ntohs(c3cn
->saddr
.sin_port
) - cxgb3_sport_base
;
136 c3cn
->saddr
.sin_port
= 0;
137 if (idx
< 0 || idx
>= cxgb3_max_connect
)
139 spin_lock_bh(&cdata
->lock
);
140 cdata
->sport_conn
[idx
] = NULL
;
141 spin_unlock_bh(&cdata
->lock
);
142 c3cn_conn_debug("%s, release port %u.\n",
143 cdata
->cdev
->name
, cxgb3_sport_base
+ idx
);
147 static inline void c3cn_set_flag(struct s3_conn
*c3cn
, enum c3cn_flags flag
)
149 __set_bit(flag
, &c3cn
->flags
);
150 c3cn_conn_debug("c3cn 0x%p, set %d, s %u, f 0x%lx.\n",
151 c3cn
, flag
, c3cn
->state
, c3cn
->flags
);
154 static inline void c3cn_clear_flag(struct s3_conn
*c3cn
, enum c3cn_flags flag
)
156 __clear_bit(flag
, &c3cn
->flags
);
157 c3cn_conn_debug("c3cn 0x%p, clear %d, s %u, f 0x%lx.\n",
158 c3cn
, flag
, c3cn
->state
, c3cn
->flags
);
161 static inline int c3cn_flag(struct s3_conn
*c3cn
, enum c3cn_flags flag
)
165 return test_bit(flag
, &c3cn
->flags
);
168 static void c3cn_set_state(struct s3_conn
*c3cn
, int state
)
170 c3cn_conn_debug("c3cn 0x%p state -> %u.\n", c3cn
, state
);
174 static inline void c3cn_hold(struct s3_conn
*c3cn
)
176 atomic_inc(&c3cn
->refcnt
);
179 static inline void c3cn_put(struct s3_conn
*c3cn
)
181 if (atomic_dec_and_test(&c3cn
->refcnt
)) {
182 c3cn_conn_debug("free c3cn 0x%p, s %u, f 0x%lx.\n",
183 c3cn
, c3cn
->state
, c3cn
->flags
);
188 static void c3cn_closed(struct s3_conn
*c3cn
)
190 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
191 c3cn
, c3cn
->state
, c3cn
->flags
);
194 c3cn_release_offload_resources(c3cn
);
195 c3cn_set_state(c3cn
, C3CN_STATE_CLOSED
);
196 cxgb3i_conn_closing(c3cn
);
200 * CPL (Chelsio Protocol Language) defines a message passing interface between
201 * the host driver and T3 asic.
202 * The section below implments CPLs that related to iscsi tcp connection
203 * open/close/abort and data send/receive.
207 * CPL connection active open request: host ->
209 static unsigned int find_best_mtu(const struct t3c_data
*d
, unsigned short mtu
)
213 while (i
< d
->nmtus
- 1 && d
->mtus
[i
+ 1] <= mtu
)
218 static unsigned int select_mss(struct s3_conn
*c3cn
, unsigned int pmtu
)
221 struct dst_entry
*dst
= c3cn
->dst_cache
;
222 struct t3cdev
*cdev
= c3cn
->cdev
;
223 const struct t3c_data
*td
= T3C_DATA(cdev
);
224 u16 advmss
= dst_metric(dst
, RTAX_ADVMSS
);
226 if (advmss
> pmtu
- 40)
228 if (advmss
< td
->mtus
[0] - 40)
229 advmss
= td
->mtus
[0] - 40;
230 idx
= find_best_mtu(td
, advmss
+ 40);
234 static inline int compute_wscale(int win
)
237 while (wscale
< 14 && (65535<<wscale
) < win
)
242 static inline unsigned int calc_opt0h(struct s3_conn
*c3cn
)
244 int wscale
= compute_wscale(cxgb3_rcv_win
);
245 return V_KEEP_ALIVE(1) |
247 V_WND_SCALE(wscale
) |
248 V_MSS_IDX(c3cn
->mss_idx
);
251 static inline unsigned int calc_opt0l(struct s3_conn
*c3cn
)
253 return V_ULP_MODE(ULP_MODE_ISCSI
) |
254 V_RCV_BUFSIZ(cxgb3_rcv_win
>>10);
257 static void make_act_open_req(struct s3_conn
*c3cn
, struct sk_buff
*skb
,
258 unsigned int atid
, const struct l2t_entry
*e
)
260 struct cpl_act_open_req
*req
;
262 c3cn_conn_debug("c3cn 0x%p, atid 0x%x.\n", c3cn
, atid
);
264 skb
->priority
= CPL_PRIORITY_SETUP
;
265 req
= (struct cpl_act_open_req
*)__skb_put(skb
, sizeof(*req
));
266 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_FORWARD
));
268 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ
, atid
));
269 req
->local_port
= c3cn
->saddr
.sin_port
;
270 req
->peer_port
= c3cn
->daddr
.sin_port
;
271 req
->local_ip
= c3cn
->saddr
.sin_addr
.s_addr
;
272 req
->peer_ip
= c3cn
->daddr
.sin_addr
.s_addr
;
273 req
->opt0h
= htonl(calc_opt0h(c3cn
) | V_L2T_IDX(e
->idx
) |
274 V_TX_CHANNEL(e
->smt_idx
));
275 req
->opt0l
= htonl(calc_opt0l(c3cn
));
280 static void fail_act_open(struct s3_conn
*c3cn
, int errno
)
282 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
283 c3cn
, c3cn
->state
, c3cn
->flags
);
288 static void act_open_req_arp_failure(struct t3cdev
*dev
, struct sk_buff
*skb
)
290 struct s3_conn
*c3cn
= (struct s3_conn
*)skb
->sk
;
292 c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn
, c3cn
->state
);
295 spin_lock_bh(&c3cn
->lock
);
296 if (c3cn
->state
== C3CN_STATE_CONNECTING
)
297 fail_act_open(c3cn
, -EHOSTUNREACH
);
298 spin_unlock_bh(&c3cn
->lock
);
304 * CPL connection close request: host ->
306 * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to
307 * the write queue (i.e., after any unsent txt data).
309 static void skb_entail(struct s3_conn
*c3cn
, struct sk_buff
*skb
,
312 skb_tcp_seq(skb
) = c3cn
->write_seq
;
313 skb_flags(skb
) = flags
;
314 __skb_queue_tail(&c3cn
->write_queue
, skb
);
317 static void send_close_req(struct s3_conn
*c3cn
)
319 struct sk_buff
*skb
= c3cn
->cpl_close
;
320 struct cpl_close_con_req
*req
= (struct cpl_close_con_req
*)skb
->head
;
321 unsigned int tid
= c3cn
->tid
;
323 c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
324 c3cn
, c3cn
->state
, c3cn
->flags
);
326 c3cn
->cpl_close
= NULL
;
328 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON
));
329 req
->wr
.wr_lo
= htonl(V_WR_TID(tid
));
330 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ
, tid
));
331 req
->rsvd
= htonl(c3cn
->write_seq
);
333 skb_entail(c3cn
, skb
, C3CB_FLAG_NO_APPEND
);
334 if (c3cn
->state
!= C3CN_STATE_CONNECTING
)
335 c3cn_push_tx_frames(c3cn
, 1);
339 * CPL connection abort request: host ->
341 * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs
342 * for the same connection and also that we do not try to send a message
343 * after the connection has closed.
345 static void abort_arp_failure(struct t3cdev
*cdev
, struct sk_buff
*skb
)
347 struct cpl_abort_req
*req
= cplhdr(skb
);
349 c3cn_conn_debug("tdev 0x%p.\n", cdev
);
351 req
->cmd
= CPL_ABORT_NO_RST
;
352 cxgb3_ofld_send(cdev
, skb
);
355 static inline void c3cn_purge_write_queue(struct s3_conn
*c3cn
)
359 while ((skb
= __skb_dequeue(&c3cn
->write_queue
)))
363 static void send_abort_req(struct s3_conn
*c3cn
)
365 struct sk_buff
*skb
= c3cn
->cpl_abort_req
;
366 struct cpl_abort_req
*req
;
367 unsigned int tid
= c3cn
->tid
;
369 if (unlikely(c3cn
->state
== C3CN_STATE_ABORTING
) || !skb
||
373 c3cn_set_state(c3cn
, C3CN_STATE_ABORTING
);
375 c3cn_conn_debug("c3cn 0x%p, flag ABORT_RPL + ABORT_SHUT.\n", c3cn
);
377 c3cn_set_flag(c3cn
, C3CN_ABORT_RPL_PENDING
);
379 /* Purge the send queue so we don't send anything after an abort. */
380 c3cn_purge_write_queue(c3cn
);
382 c3cn
->cpl_abort_req
= NULL
;
383 req
= (struct cpl_abort_req
*)skb
->head
;
384 memset(req
, 0, sizeof(*req
));
386 skb
->priority
= CPL_PRIORITY_DATA
;
387 set_arp_failure_handler(skb
, abort_arp_failure
);
389 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ
));
390 req
->wr
.wr_lo
= htonl(V_WR_TID(tid
));
391 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ
, tid
));
392 req
->rsvd0
= htonl(c3cn
->snd_nxt
);
393 req
->rsvd1
= !c3cn_flag(c3cn
, C3CN_TX_DATA_SENT
);
394 req
->cmd
= CPL_ABORT_SEND_RST
;
396 l2t_send(c3cn
->cdev
, skb
, c3cn
->l2t
);
400 * CPL connection abort reply: host ->
402 * Send an ABORT_RPL message in response of the ABORT_REQ received.
404 static void send_abort_rpl(struct s3_conn
*c3cn
, int rst_status
)
406 struct sk_buff
*skb
= c3cn
->cpl_abort_rpl
;
407 struct cpl_abort_rpl
*rpl
= (struct cpl_abort_rpl
*)skb
->head
;
409 c3cn
->cpl_abort_rpl
= NULL
;
411 skb
->priority
= CPL_PRIORITY_DATA
;
412 memset(rpl
, 0, sizeof(*rpl
));
413 rpl
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL
));
414 rpl
->wr
.wr_lo
= htonl(V_WR_TID(c3cn
->tid
));
415 OPCODE_TID(rpl
) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL
, c3cn
->tid
));
416 rpl
->cmd
= rst_status
;
418 cxgb3_ofld_send(c3cn
->cdev
, skb
);
422 * CPL connection rx data ack: host ->
423 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
426 static u32
send_rx_credits(struct s3_conn
*c3cn
, u32 credits
, u32 dack
)
429 struct cpl_rx_data_ack
*req
;
431 skb
= alloc_skb(sizeof(*req
), GFP_ATOMIC
);
435 req
= (struct cpl_rx_data_ack
*)__skb_put(skb
, sizeof(*req
));
436 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_FORWARD
));
438 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK
, c3cn
->tid
));
439 req
->credit_dack
= htonl(dack
| V_RX_CREDITS(credits
));
440 skb
->priority
= CPL_PRIORITY_ACK
;
441 cxgb3_ofld_send(c3cn
->cdev
, skb
);
446 * CPL connection tx data: host ->
448 * Send iscsi PDU via TX_DATA CPL message. Returns the number of
450 * Each TX_DATA consumes work request credit (wrs), so we need to keep track of
451 * how many we've used so far and how many are pending (i.e., yet ack'ed by T3).
455 * For ULP connections HW may inserts digest bytes into the pdu. Those digest
456 * bytes are not sent by the host but are part of the TCP payload and therefore
457 * consume TCP sequence space.
459 static const unsigned int cxgb3_ulp_extra_len
[] = { 0, 4, 4, 8 };
460 static inline unsigned int ulp_extra_len(const struct sk_buff
*skb
)
462 return cxgb3_ulp_extra_len
[skb_ulp_mode(skb
) & 3];
465 static unsigned int wrlen __read_mostly
;
468 * The number of WRs needed for an skb depends on the number of fragments
469 * in the skb and whether it has any payload in its main body. This maps the
470 * length of the gather list represented by an skb into the # of necessary WRs.
471 * The extra two fragments are for iscsi bhs and payload padding.
473 #define SKB_WR_LIST_SIZE (MAX_SKB_FRAGS + 2)
474 static unsigned int skb_wrs
[SKB_WR_LIST_SIZE
] __read_mostly
;
476 static void s3_init_wr_tab(unsigned int wr_len
)
480 if (skb_wrs
[1]) /* already initialized */
483 for (i
= 1; i
< SKB_WR_LIST_SIZE
; i
++) {
484 int sgl_len
= (3 * i
) / 2 + (i
& 1);
487 skb_wrs
[i
] = (sgl_len
<= wr_len
488 ? 1 : 1 + (sgl_len
- 2) / (wr_len
- 1));
494 static inline void reset_wr_list(struct s3_conn
*c3cn
)
496 c3cn
->wr_pending_head
= c3cn
->wr_pending_tail
= NULL
;
500 * Add a WR to a connections's list of pending WRs. This is a singly-linked
501 * list of sk_buffs operating as a FIFO. The head is kept in wr_pending_head
502 * and the tail in wr_pending_tail.
504 static inline void enqueue_wr(struct s3_conn
*c3cn
,
507 skb_tx_wr_next(skb
) = NULL
;
510 * We want to take an extra reference since both us and the driver
511 * need to free the packet before it's really freed. We know there's
512 * just one user currently so we use atomic_set rather than skb_get
513 * to avoid the atomic op.
515 atomic_set(&skb
->users
, 2);
517 if (!c3cn
->wr_pending_head
)
518 c3cn
->wr_pending_head
= skb
;
520 skb_tx_wr_next(c3cn
->wr_pending_tail
) = skb
;
521 c3cn
->wr_pending_tail
= skb
;
524 static int count_pending_wrs(struct s3_conn
*c3cn
)
527 const struct sk_buff
*skb
= c3cn
->wr_pending_head
;
531 skb
= skb_tx_wr_next(skb
);
536 static inline struct sk_buff
*peek_wr(const struct s3_conn
*c3cn
)
538 return c3cn
->wr_pending_head
;
541 static inline void free_wr_skb(struct sk_buff
*skb
)
546 static inline struct sk_buff
*dequeue_wr(struct s3_conn
*c3cn
)
548 struct sk_buff
*skb
= c3cn
->wr_pending_head
;
551 /* Don't bother clearing the tail */
552 c3cn
->wr_pending_head
= skb_tx_wr_next(skb
);
553 skb_tx_wr_next(skb
) = NULL
;
558 static void purge_wr_queue(struct s3_conn
*c3cn
)
561 while ((skb
= dequeue_wr(c3cn
)) != NULL
)
565 static inline void make_tx_data_wr(struct s3_conn
*c3cn
, struct sk_buff
*skb
,
566 int len
, int req_completion
)
568 struct tx_data_wr
*req
;
570 skb_reset_transport_header(skb
);
571 req
= (struct tx_data_wr
*)__skb_push(skb
, sizeof(*req
));
572 req
->wr_hi
= htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA
) |
573 (req_completion
? F_WR_COMPL
: 0));
574 req
->wr_lo
= htonl(V_WR_TID(c3cn
->tid
));
575 req
->sndseq
= htonl(c3cn
->snd_nxt
);
576 /* len includes the length of any HW ULP additions */
577 req
->len
= htonl(len
);
578 req
->param
= htonl(V_TX_PORT(c3cn
->l2t
->smt_idx
));
579 /* V_TX_ULP_SUBMODE sets both the mode and submode */
580 req
->flags
= htonl(V_TX_ULP_SUBMODE(skb_ulp_mode(skb
)) |
581 V_TX_SHOVE((skb_peek(&c3cn
->write_queue
) ? 0 : 1)));
583 if (!c3cn_flag(c3cn
, C3CN_TX_DATA_SENT
)) {
584 req
->flags
|= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT
|
585 V_TX_CPU_IDX(c3cn
->qset
));
586 /* Sendbuffer is in units of 32KB. */
587 req
->param
|= htonl(V_TX_SNDBUF(cxgb3_snd_win
>> 15));
588 c3cn_set_flag(c3cn
, C3CN_TX_DATA_SENT
);
593 * c3cn_push_tx_frames -- start transmit
594 * @c3cn: the offloaded connection
595 * @req_completion: request wr_ack or not
597 * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
598 * connection's send queue and sends them on to T3. Must be called with the
599 * connection's lock held. Returns the amount of send buffer space that was
600 * freed as a result of sending queued data to T3.
602 static void arp_failure_discard(struct t3cdev
*cdev
, struct sk_buff
*skb
)
607 static int c3cn_push_tx_frames(struct s3_conn
*c3cn
, int req_completion
)
612 struct cxgb3i_sdev_data
*cdata
;
614 if (unlikely(c3cn
->state
== C3CN_STATE_CONNECTING
||
615 c3cn
->state
== C3CN_STATE_CLOSE_WAIT_1
||
616 c3cn
->state
>= C3CN_STATE_ABORTING
)) {
617 c3cn_tx_debug("c3cn 0x%p, in closing state %u.\n",
623 cdata
= CXGB3_SDEV_DATA(cdev
);
625 while (c3cn
->wr_avail
626 && (skb
= skb_peek(&c3cn
->write_queue
)) != NULL
) {
627 int len
= skb
->len
; /* length before skb_push */
628 int frags
= skb_shinfo(skb
)->nr_frags
+ (len
!= skb
->data_len
);
629 int wrs_needed
= skb_wrs
[frags
];
631 if (wrs_needed
> 1 && len
+ sizeof(struct tx_data_wr
) <= wrlen
)
634 WARN_ON(frags
>= SKB_WR_LIST_SIZE
|| wrs_needed
< 1);
636 if (c3cn
->wr_avail
< wrs_needed
) {
637 c3cn_tx_debug("c3cn 0x%p, skb len %u/%u, frag %u, "
639 c3cn
, skb
->len
, skb
->data_len
, frags
,
640 wrs_needed
, c3cn
->wr_avail
);
644 __skb_unlink(skb
, &c3cn
->write_queue
);
645 skb
->priority
= CPL_PRIORITY_DATA
;
646 skb
->csum
= wrs_needed
; /* remember this until the WR_ACK */
647 c3cn
->wr_avail
-= wrs_needed
;
648 c3cn
->wr_unacked
+= wrs_needed
;
649 enqueue_wr(c3cn
, skb
);
651 c3cn_tx_debug("c3cn 0x%p, enqueue, skb len %u/%u, frag %u, "
652 "wr %d, left %u, unack %u.\n",
653 c3cn
, skb
->len
, skb
->data_len
, frags
,
654 wrs_needed
, c3cn
->wr_avail
, c3cn
->wr_unacked
);
657 if (likely(skb_flags(skb
) & C3CB_FLAG_NEED_HDR
)) {
658 if ((req_completion
&&
659 c3cn
->wr_unacked
== wrs_needed
) ||
660 (skb_flags(skb
) & C3CB_FLAG_COMPL
) ||
661 c3cn
->wr_unacked
>= c3cn
->wr_max
/ 2) {
663 c3cn
->wr_unacked
= 0;
665 len
+= ulp_extra_len(skb
);
666 make_tx_data_wr(c3cn
, skb
, len
, req_completion
);
667 c3cn
->snd_nxt
+= len
;
668 skb_flags(skb
) &= ~C3CB_FLAG_NEED_HDR
;
671 total_size
+= skb
->truesize
;
672 set_arp_failure_handler(skb
, arp_failure_discard
);
673 l2t_send(cdev
, skb
, c3cn
->l2t
);
679 * process_cpl_msg: -> host
680 * Top-level CPL message processing used by most CPL messages that
681 * pertain to connections.
683 static inline void process_cpl_msg(void (*fn
)(struct s3_conn
*,
685 struct s3_conn
*c3cn
,
688 spin_lock_bh(&c3cn
->lock
);
690 spin_unlock_bh(&c3cn
->lock
);
694 * process_cpl_msg_ref: -> host
695 * Similar to process_cpl_msg() but takes an extra connection reference around
696 * the call to the handler. Should be used if the handler may drop a
697 * connection reference.
699 static inline void process_cpl_msg_ref(void (*fn
) (struct s3_conn
*,
701 struct s3_conn
*c3cn
,
705 process_cpl_msg(fn
, c3cn
, skb
);
710 * Process a CPL_ACT_ESTABLISH message: -> host
711 * Updates connection state from an active establish CPL message. Runs with
712 * the connection lock held.
715 static inline void s3_free_atid(struct t3cdev
*cdev
, unsigned int tid
)
717 struct s3_conn
*c3cn
= cxgb3_free_atid(cdev
, tid
);
722 static void c3cn_established(struct s3_conn
*c3cn
, u32 snd_isn
,
725 c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn
, c3cn
->state
);
727 c3cn
->write_seq
= c3cn
->snd_nxt
= c3cn
->snd_una
= snd_isn
;
730 * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
733 if (cxgb3_rcv_win
> (M_RCV_BUFSIZ
<< 10))
734 c3cn
->rcv_wup
-= cxgb3_rcv_win
- (M_RCV_BUFSIZ
<< 10);
736 dst_confirm(c3cn
->dst_cache
);
740 c3cn_set_state(c3cn
, C3CN_STATE_ESTABLISHED
);
743 static void process_act_establish(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
745 struct cpl_act_establish
*req
= cplhdr(skb
);
746 u32 rcv_isn
= ntohl(req
->rcv_isn
); /* real RCV_ISN + 1 */
748 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
749 c3cn
, c3cn
->state
, c3cn
->flags
);
751 if (unlikely(c3cn
->state
!= C3CN_STATE_CONNECTING
))
752 cxgb3i_log_error("TID %u expected SYN_SENT, got EST., s %u\n",
753 c3cn
->tid
, c3cn
->state
);
755 c3cn
->copied_seq
= c3cn
->rcv_wup
= c3cn
->rcv_nxt
= rcv_isn
;
756 c3cn_established(c3cn
, ntohl(req
->snd_isn
), ntohs(req
->tcp_opt
));
760 if (unlikely(c3cn_flag(c3cn
, C3CN_ACTIVE_CLOSE_NEEDED
)))
761 /* upper layer has requested closing */
762 send_abort_req(c3cn
);
764 if (skb_queue_len(&c3cn
->write_queue
))
765 c3cn_push_tx_frames(c3cn
, 1);
766 cxgb3i_conn_tx_open(c3cn
);
770 static int do_act_establish(struct t3cdev
*cdev
, struct sk_buff
*skb
,
773 struct cpl_act_establish
*req
= cplhdr(skb
);
774 unsigned int tid
= GET_TID(req
);
775 unsigned int atid
= G_PASS_OPEN_TID(ntohl(req
->tos_tid
));
776 struct s3_conn
*c3cn
= ctx
;
777 struct cxgb3i_sdev_data
*cdata
= CXGB3_SDEV_DATA(cdev
);
779 c3cn_conn_debug("rcv, tid 0x%x, c3cn 0x%p, s %u, f 0x%lx.\n",
780 tid
, c3cn
, c3cn
->state
, c3cn
->flags
);
784 cxgb3_insert_tid(cdata
->cdev
, cdata
->client
, c3cn
, tid
);
785 s3_free_atid(cdev
, atid
);
787 c3cn
->qset
= G_QNUM(ntohl(skb
->csum
));
789 process_cpl_msg(process_act_establish
, c3cn
, skb
);
794 * Process a CPL_ACT_OPEN_RPL message: -> host
795 * Handle active open failures.
797 static int act_open_rpl_status_to_errno(int status
)
800 case CPL_ERR_CONN_RESET
:
801 return -ECONNREFUSED
;
802 case CPL_ERR_ARP_MISS
:
803 return -EHOSTUNREACH
;
804 case CPL_ERR_CONN_TIMEDOUT
:
806 case CPL_ERR_TCAM_FULL
:
808 case CPL_ERR_CONN_EXIST
:
809 cxgb3i_log_error("ACTIVE_OPEN_RPL: 4-tuple in use\n");
816 static void act_open_retry_timer(unsigned long data
)
819 struct s3_conn
*c3cn
= (struct s3_conn
*)data
;
821 c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn
, c3cn
->state
);
823 spin_lock_bh(&c3cn
->lock
);
824 skb
= alloc_skb(sizeof(struct cpl_act_open_req
), GFP_ATOMIC
);
826 fail_act_open(c3cn
, -ENOMEM
);
828 skb
->sk
= (struct sock
*)c3cn
;
829 set_arp_failure_handler(skb
, act_open_req_arp_failure
);
830 make_act_open_req(c3cn
, skb
, c3cn
->tid
, c3cn
->l2t
);
831 l2t_send(c3cn
->cdev
, skb
, c3cn
->l2t
);
833 spin_unlock_bh(&c3cn
->lock
);
837 static void process_act_open_rpl(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
839 struct cpl_act_open_rpl
*rpl
= cplhdr(skb
);
841 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
842 c3cn
, c3cn
->state
, c3cn
->flags
);
844 if (rpl
->status
== CPL_ERR_CONN_EXIST
&&
845 c3cn
->retry_timer
.function
!= act_open_retry_timer
) {
846 c3cn
->retry_timer
.function
= act_open_retry_timer
;
847 if (!mod_timer(&c3cn
->retry_timer
, jiffies
+ HZ
/ 2))
850 fail_act_open(c3cn
, act_open_rpl_status_to_errno(rpl
->status
));
854 static int do_act_open_rpl(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
856 struct s3_conn
*c3cn
= ctx
;
857 struct cpl_act_open_rpl
*rpl
= cplhdr(skb
);
859 c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, s %u, f 0x%lx.\n",
860 rpl
->status
, c3cn
, c3cn
->state
, c3cn
->flags
);
862 if (rpl
->status
!= CPL_ERR_TCAM_FULL
&&
863 rpl
->status
!= CPL_ERR_CONN_EXIST
&&
864 rpl
->status
!= CPL_ERR_ARP_MISS
)
865 cxgb3_queue_tid_release(cdev
, GET_TID(rpl
));
867 process_cpl_msg_ref(process_act_open_rpl
, c3cn
, skb
);
872 * Process PEER_CLOSE CPL messages: -> host
875 static void process_peer_close(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
877 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
878 c3cn
, c3cn
->state
, c3cn
->flags
);
880 if (c3cn_flag(c3cn
, C3CN_ABORT_RPL_PENDING
))
883 switch (c3cn
->state
) {
884 case C3CN_STATE_ESTABLISHED
:
885 c3cn_set_state(c3cn
, C3CN_STATE_PASSIVE_CLOSE
);
887 case C3CN_STATE_ACTIVE_CLOSE
:
888 c3cn_set_state(c3cn
, C3CN_STATE_CLOSE_WAIT_2
);
890 case C3CN_STATE_CLOSE_WAIT_1
:
893 case C3CN_STATE_ABORTING
:
896 cxgb3i_log_error("%s: peer close, TID %u in bad state %u\n",
897 c3cn
->cdev
->name
, c3cn
->tid
, c3cn
->state
);
900 cxgb3i_conn_closing(c3cn
);
905 static int do_peer_close(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
907 struct s3_conn
*c3cn
= ctx
;
909 c3cn_conn_debug("rcv, c3cn 0x%p, s %u, f 0x%lx.\n",
910 c3cn
, c3cn
->state
, c3cn
->flags
);
911 process_cpl_msg_ref(process_peer_close
, c3cn
, skb
);
916 * Process CLOSE_CONN_RPL CPL message: -> host
917 * Process a peer ACK to our FIN.
919 static void process_close_con_rpl(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
921 struct cpl_close_con_rpl
*rpl
= cplhdr(skb
);
923 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
924 c3cn
, c3cn
->state
, c3cn
->flags
);
926 c3cn
->snd_una
= ntohl(rpl
->snd_nxt
) - 1; /* exclude FIN */
928 if (c3cn_flag(c3cn
, C3CN_ABORT_RPL_PENDING
))
931 switch (c3cn
->state
) {
932 case C3CN_STATE_ACTIVE_CLOSE
:
933 c3cn_set_state(c3cn
, C3CN_STATE_CLOSE_WAIT_1
);
935 case C3CN_STATE_CLOSE_WAIT_1
:
936 case C3CN_STATE_CLOSE_WAIT_2
:
939 case C3CN_STATE_ABORTING
:
942 cxgb3i_log_error("%s: close_rpl, TID %u in bad state %u\n",
943 c3cn
->cdev
->name
, c3cn
->tid
, c3cn
->state
);
950 static int do_close_con_rpl(struct t3cdev
*cdev
, struct sk_buff
*skb
,
953 struct s3_conn
*c3cn
= ctx
;
955 c3cn_conn_debug("rcv, c3cn 0x%p, s %u, f 0x%lx.\n",
956 c3cn
, c3cn
->state
, c3cn
->flags
);
958 process_cpl_msg_ref(process_close_con_rpl
, c3cn
, skb
);
963 * Process ABORT_REQ_RSS CPL message: -> host
964 * Process abort requests. If we are waiting for an ABORT_RPL we ignore this
965 * request except that we need to reply to it.
968 static int abort_status_to_errno(struct s3_conn
*c3cn
, int abort_reason
,
971 switch (abort_reason
) {
972 case CPL_ERR_BAD_SYN
: /* fall through */
973 case CPL_ERR_CONN_RESET
:
974 return c3cn
->state
> C3CN_STATE_ESTABLISHED
?
975 -EPIPE
: -ECONNRESET
;
976 case CPL_ERR_XMIT_TIMEDOUT
:
977 case CPL_ERR_PERSIST_TIMEDOUT
:
978 case CPL_ERR_FINWAIT2_TIMEDOUT
:
979 case CPL_ERR_KEEPALIVE_TIMEDOUT
:
986 static void process_abort_req(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
988 int rst_status
= CPL_ABORT_NO_RST
;
989 const struct cpl_abort_req_rss
*req
= cplhdr(skb
);
991 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
992 c3cn
, c3cn
->state
, c3cn
->flags
);
994 if (!c3cn_flag(c3cn
, C3CN_ABORT_REQ_RCVD
)) {
995 c3cn_set_flag(c3cn
, C3CN_ABORT_REQ_RCVD
);
996 c3cn_set_state(c3cn
, C3CN_STATE_ABORTING
);
1001 c3cn_clear_flag(c3cn
, C3CN_ABORT_REQ_RCVD
);
1002 send_abort_rpl(c3cn
, rst_status
);
1004 if (!c3cn_flag(c3cn
, C3CN_ABORT_RPL_PENDING
)) {
1006 abort_status_to_errno(c3cn
, req
->status
, &rst_status
);
1011 static int do_abort_req(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
1013 const struct cpl_abort_req_rss
*req
= cplhdr(skb
);
1014 struct s3_conn
*c3cn
= ctx
;
1016 c3cn_conn_debug("rcv, c3cn 0x%p, s 0x%x, f 0x%lx.\n",
1017 c3cn
, c3cn
->state
, c3cn
->flags
);
1019 if (req
->status
== CPL_ERR_RTX_NEG_ADVICE
||
1020 req
->status
== CPL_ERR_PERSIST_NEG_ADVICE
) {
1025 process_cpl_msg_ref(process_abort_req
, c3cn
, skb
);
1030 * Process ABORT_RPL_RSS CPL message: -> host
1031 * Process abort replies. We only process these messages if we anticipate
1032 * them as the coordination between SW and HW in this area is somewhat lacking
1033 * and sometimes we get ABORT_RPLs after we are done with the connection that
1034 * originated the ABORT_REQ.
1036 static void process_abort_rpl(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
1038 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1039 c3cn
, c3cn
->state
, c3cn
->flags
);
1041 if (c3cn_flag(c3cn
, C3CN_ABORT_RPL_PENDING
)) {
1042 if (!c3cn_flag(c3cn
, C3CN_ABORT_RPL_RCVD
))
1043 c3cn_set_flag(c3cn
, C3CN_ABORT_RPL_RCVD
);
1045 c3cn_clear_flag(c3cn
, C3CN_ABORT_RPL_RCVD
);
1046 c3cn_clear_flag(c3cn
, C3CN_ABORT_RPL_PENDING
);
1047 if (c3cn_flag(c3cn
, C3CN_ABORT_REQ_RCVD
))
1048 cxgb3i_log_error("%s tid %u, ABORT_RPL_RSS\n",
1049 c3cn
->cdev
->name
, c3cn
->tid
);
1056 static int do_abort_rpl(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
1058 struct cpl_abort_rpl_rss
*rpl
= cplhdr(skb
);
1059 struct s3_conn
*c3cn
= ctx
;
1061 c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, s %u, 0x%lx.\n",
1062 rpl
->status
, c3cn
, c3cn
? c3cn
->state
: 0,
1063 c3cn
? c3cn
->flags
: 0UL);
1066 * Ignore replies to post-close aborts indicating that the abort was
1067 * requested too late. These connections are terminated when we get
1068 * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
1069 * arrives the TID is either no longer used or it has been recycled.
1071 if (rpl
->status
== CPL_ERR_ABORT_FAILED
)
1075 * Sometimes we've already closed the connection, e.g., a post-close
1076 * abort races with ABORT_REQ_RSS, the latter frees the connection
1077 * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
1078 * but FW turns the ABORT_REQ into a regular one and so we get
1079 * ABORT_RPL_RSS with status 0 and no connection.
1084 process_cpl_msg_ref(process_abort_rpl
, c3cn
, skb
);
1093 * Process RX_ISCSI_HDR CPL message: -> host
1094 * Handle received PDUs, the payload could be DDP'ed. If not, the payload
1095 * follow after the bhs.
1097 static void process_rx_iscsi_hdr(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
1099 struct cpl_iscsi_hdr
*hdr_cpl
= cplhdr(skb
);
1100 struct cpl_iscsi_hdr_norss data_cpl
;
1101 struct cpl_rx_data_ddp_norss ddp_cpl
;
1102 unsigned int hdr_len
, data_len
, status
;
1106 if (unlikely(c3cn
->state
>= C3CN_STATE_PASSIVE_CLOSE
)) {
1107 if (c3cn
->state
!= C3CN_STATE_ABORTING
)
1108 send_abort_req(c3cn
);
1113 skb_tcp_seq(skb
) = ntohl(hdr_cpl
->seq
);
1116 skb_reset_transport_header(skb
);
1117 __skb_pull(skb
, sizeof(struct cpl_iscsi_hdr
));
1119 len
= hdr_len
= ntohs(hdr_cpl
->len
);
1120 /* msg coalesce is off or not enough data received */
1121 if (skb
->len
<= hdr_len
) {
1122 cxgb3i_log_error("%s: TID %u, ISCSI_HDR, skb len %u < %u.\n",
1123 c3cn
->cdev
->name
, c3cn
->tid
,
1128 err
= skb_copy_bits(skb
, skb
->len
- sizeof(ddp_cpl
), &ddp_cpl
,
1133 skb_ulp_mode(skb
) = ULP2_FLAG_DATA_READY
;
1134 skb_rx_pdulen(skb
) = ntohs(ddp_cpl
.len
);
1135 skb_rx_ddigest(skb
) = ntohl(ddp_cpl
.ulp_crc
);
1136 status
= ntohl(ddp_cpl
.ddp_status
);
1138 c3cn_rx_debug("rx skb 0x%p, len %u, pdulen %u, ddp status 0x%x.\n",
1139 skb
, skb
->len
, skb_rx_pdulen(skb
), status
);
1141 if (status
& (1 << RX_DDP_STATUS_HCRC_SHIFT
))
1142 skb_ulp_mode(skb
) |= ULP2_FLAG_HCRC_ERROR
;
1143 if (status
& (1 << RX_DDP_STATUS_DCRC_SHIFT
))
1144 skb_ulp_mode(skb
) |= ULP2_FLAG_DCRC_ERROR
;
1145 if (status
& (1 << RX_DDP_STATUS_PAD_SHIFT
))
1146 skb_ulp_mode(skb
) |= ULP2_FLAG_PAD_ERROR
;
1148 if (skb
->len
> (hdr_len
+ sizeof(ddp_cpl
))) {
1149 err
= skb_copy_bits(skb
, hdr_len
, &data_cpl
, sizeof(data_cpl
));
1152 data_len
= ntohs(data_cpl
.len
);
1153 len
+= sizeof(data_cpl
) + data_len
;
1154 } else if (status
& (1 << RX_DDP_STATUS_DDP_SHIFT
))
1155 skb_ulp_mode(skb
) |= ULP2_FLAG_DATA_DDPED
;
1157 c3cn
->rcv_nxt
= ntohl(ddp_cpl
.seq
) + skb_rx_pdulen(skb
);
1158 __pskb_trim(skb
, len
);
1159 __skb_queue_tail(&c3cn
->receive_queue
, skb
);
1160 cxgb3i_conn_pdu_ready(c3cn
);
1165 send_abort_req(c3cn
);
1169 static int do_iscsi_hdr(struct t3cdev
*t3dev
, struct sk_buff
*skb
, void *ctx
)
1171 struct s3_conn
*c3cn
= ctx
;
1173 process_cpl_msg(process_rx_iscsi_hdr
, c3cn
, skb
);
1178 * Process TX_DATA_ACK CPL messages: -> host
1179 * Process an acknowledgment of WR completion. Advance snd_una and send the
1180 * next batch of work requests from the write queue.
1182 static void check_wr_invariants(struct s3_conn
*c3cn
)
1184 int pending
= count_pending_wrs(c3cn
);
1186 if (unlikely(c3cn
->wr_avail
+ pending
!= c3cn
->wr_max
))
1187 cxgb3i_log_error("TID %u: credit imbalance: avail %u, "
1188 "pending %u, total should be %u\n",
1189 c3cn
->tid
, c3cn
->wr_avail
, pending
,
1193 static void process_wr_ack(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
1195 struct cpl_wr_ack
*hdr
= cplhdr(skb
);
1196 unsigned int credits
= ntohs(hdr
->credits
);
1197 u32 snd_una
= ntohl(hdr
->snd_una
);
1199 c3cn_tx_debug("%u WR credits, avail %u, unack %u, TID %u, state %u.\n",
1200 credits
, c3cn
->wr_avail
, c3cn
->wr_unacked
,
1201 c3cn
->tid
, c3cn
->state
);
1203 c3cn
->wr_avail
+= credits
;
1204 if (c3cn
->wr_unacked
> c3cn
->wr_max
- c3cn
->wr_avail
)
1205 c3cn
->wr_unacked
= c3cn
->wr_max
- c3cn
->wr_avail
;
1208 struct sk_buff
*p
= peek_wr(c3cn
);
1211 cxgb3i_log_error("%u WR_ACK credits for TID %u with "
1212 "nothing pending, state %u\n",
1213 credits
, c3cn
->tid
, c3cn
->state
);
1216 if (unlikely(credits
< p
->csum
)) {
1217 struct tx_data_wr
*w
= cplhdr(p
);
1218 cxgb3i_log_error("TID %u got %u WR credits need %u, "
1219 "len %u, main body %u, frags %u, "
1220 "seq # %u, ACK una %u, ACK nxt %u, "
1221 "WR_AVAIL %u, WRs pending %u\n",
1222 c3cn
->tid
, credits
, p
->csum
, p
->len
,
1223 p
->len
- p
->data_len
,
1224 skb_shinfo(p
)->nr_frags
,
1225 ntohl(w
->sndseq
), snd_una
,
1226 ntohl(hdr
->snd_nxt
), c3cn
->wr_avail
,
1227 count_pending_wrs(c3cn
) - credits
);
1237 check_wr_invariants(c3cn
);
1239 if (unlikely(before(snd_una
, c3cn
->snd_una
))) {
1240 cxgb3i_log_error("TID %u, unexpected sequence # %u in WR_ACK "
1242 c3cn
->tid
, snd_una
, c3cn
->snd_una
);
1246 if (c3cn
->snd_una
!= snd_una
) {
1247 c3cn
->snd_una
= snd_una
;
1248 dst_confirm(c3cn
->dst_cache
);
1251 if (skb_queue_len(&c3cn
->write_queue
)) {
1252 if (c3cn_push_tx_frames(c3cn
, 0))
1253 cxgb3i_conn_tx_open(c3cn
);
1255 cxgb3i_conn_tx_open(c3cn
);
1260 static int do_wr_ack(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
1262 struct s3_conn
*c3cn
= ctx
;
1264 process_cpl_msg(process_wr_ack
, c3cn
, skb
);
1269 * for each connection, pre-allocate skbs needed for close/abort requests. So
1270 * that we can service the request right away.
1272 static void c3cn_free_cpl_skbs(struct s3_conn
*c3cn
)
1274 if (c3cn
->cpl_close
)
1275 kfree_skb(c3cn
->cpl_close
);
1276 if (c3cn
->cpl_abort_req
)
1277 kfree_skb(c3cn
->cpl_abort_req
);
1278 if (c3cn
->cpl_abort_rpl
)
1279 kfree_skb(c3cn
->cpl_abort_rpl
);
1282 static int c3cn_alloc_cpl_skbs(struct s3_conn
*c3cn
)
1284 c3cn
->cpl_close
= alloc_skb(sizeof(struct cpl_close_con_req
),
1286 if (!c3cn
->cpl_close
)
1288 skb_put(c3cn
->cpl_close
, sizeof(struct cpl_close_con_req
));
1290 c3cn
->cpl_abort_req
= alloc_skb(sizeof(struct cpl_abort_req
),
1292 if (!c3cn
->cpl_abort_req
)
1294 skb_put(c3cn
->cpl_abort_req
, sizeof(struct cpl_abort_req
));
1296 c3cn
->cpl_abort_rpl
= alloc_skb(sizeof(struct cpl_abort_rpl
),
1298 if (!c3cn
->cpl_abort_rpl
)
1300 skb_put(c3cn
->cpl_abort_rpl
, sizeof(struct cpl_abort_rpl
));
1305 c3cn_free_cpl_skbs(c3cn
);
1310 * c3cn_release_offload_resources - release offload resource
1311 * @c3cn: the offloaded iscsi tcp connection.
1312 * Release resources held by an offload connection (TID, L2T entry, etc.)
1314 static void c3cn_release_offload_resources(struct s3_conn
*c3cn
)
1316 struct t3cdev
*cdev
= c3cn
->cdev
;
1317 unsigned int tid
= c3cn
->tid
;
1320 c3cn_free_cpl_skbs(c3cn
);
1322 if (c3cn
->wr_avail
!= c3cn
->wr_max
) {
1323 purge_wr_queue(c3cn
);
1324 reset_wr_list(c3cn
);
1329 l2t_release(L2DATA(cdev
), c3cn
->l2t
);
1332 if (c3cn
->state
== C3CN_STATE_CONNECTING
)
1334 s3_free_atid(cdev
, tid
);
1337 cxgb3_remove_tid(cdev
, (void *)c3cn
, tid
);
1342 c3cn
->dst_cache
= NULL
;
1347 * cxgb3i_c3cn_create - allocate and initialize an s3_conn structure
1348 * returns the s3_conn structure allocated.
1350 struct s3_conn
*cxgb3i_c3cn_create(void)
1352 struct s3_conn
*c3cn
;
1354 c3cn
= kzalloc(sizeof(*c3cn
), GFP_KERNEL
);
1358 /* pre-allocate close/abort cpl, so we don't need to wait for memory
1359 when close/abort is requested. */
1360 if (c3cn_alloc_cpl_skbs(c3cn
) < 0)
1363 c3cn_conn_debug("alloc c3cn 0x%p.\n", c3cn
);
1366 spin_lock_init(&c3cn
->lock
);
1367 atomic_set(&c3cn
->refcnt
, 1);
1368 skb_queue_head_init(&c3cn
->receive_queue
);
1369 skb_queue_head_init(&c3cn
->write_queue
);
1370 setup_timer(&c3cn
->retry_timer
, NULL
, (unsigned long)c3cn
);
1371 rwlock_init(&c3cn
->callback_lock
);
1380 static void c3cn_active_close(struct s3_conn
*c3cn
)
1385 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1386 c3cn
, c3cn
->state
, c3cn
->flags
);
1388 dst_confirm(c3cn
->dst_cache
);
1391 spin_lock_bh(&c3cn
->lock
);
1393 data_lost
= skb_queue_len(&c3cn
->receive_queue
);
1394 __skb_queue_purge(&c3cn
->receive_queue
);
1396 switch (c3cn
->state
) {
1397 case C3CN_STATE_CLOSED
:
1398 case C3CN_STATE_ACTIVE_CLOSE
:
1399 case C3CN_STATE_CLOSE_WAIT_1
:
1400 case C3CN_STATE_CLOSE_WAIT_2
:
1401 case C3CN_STATE_ABORTING
:
1402 /* nothing need to be done */
1404 case C3CN_STATE_CONNECTING
:
1405 /* defer until cpl_act_open_rpl or cpl_act_establish */
1406 c3cn_set_flag(c3cn
, C3CN_ACTIVE_CLOSE_NEEDED
);
1408 case C3CN_STATE_ESTABLISHED
:
1410 c3cn_set_state(c3cn
, C3CN_STATE_ACTIVE_CLOSE
);
1412 case C3CN_STATE_PASSIVE_CLOSE
:
1414 c3cn_set_state(c3cn
, C3CN_STATE_CLOSE_WAIT_2
);
1420 /* Unread data was tossed, zap the connection. */
1421 send_abort_req(c3cn
);
1423 send_close_req(c3cn
);
1426 spin_unlock_bh(&c3cn
->lock
);
1431 * cxgb3i_c3cn_release - close and release an iscsi tcp connection and any
1433 * @c3cn: the iscsi tcp connection
1435 void cxgb3i_c3cn_release(struct s3_conn
*c3cn
)
1437 c3cn_conn_debug("c3cn 0x%p, s %u, f 0x%lx.\n",
1438 c3cn
, c3cn
->state
, c3cn
->flags
);
1439 if (unlikely(c3cn
->state
== C3CN_STATE_CONNECTING
))
1440 c3cn_set_flag(c3cn
, C3CN_ACTIVE_CLOSE_NEEDED
);
1441 else if (likely(c3cn
->state
!= C3CN_STATE_CLOSED
))
1442 c3cn_active_close(c3cn
);
1446 static int is_cxgb3_dev(struct net_device
*dev
)
1448 struct cxgb3i_sdev_data
*cdata
;
1449 struct net_device
*ndev
= dev
;
1451 if (dev
->priv_flags
& IFF_802_1Q_VLAN
)
1452 ndev
= vlan_dev_real_dev(dev
);
1454 write_lock(&cdata_rwlock
);
1455 list_for_each_entry(cdata
, &cdata_list
, list
) {
1456 struct adap_ports
*ports
= &cdata
->ports
;
1459 for (i
= 0; i
< ports
->nports
; i
++)
1460 if (ndev
== ports
->lldevs
[i
]) {
1461 write_unlock(&cdata_rwlock
);
1465 write_unlock(&cdata_rwlock
);
1470 * cxgb3_egress_dev - return the cxgb3 egress device
1471 * @root_dev: the root device anchoring the search
1472 * @c3cn: the connection used to determine egress port in bonding mode
1473 * @context: in bonding mode, indicates a connection set up or failover
1475 * Return egress device or NULL if the egress device isn't one of our ports.
1477 static struct net_device
*cxgb3_egress_dev(struct net_device
*root_dev
,
1478 struct s3_conn
*c3cn
,
1482 if (root_dev
->priv_flags
& IFF_802_1Q_VLAN
)
1483 root_dev
= vlan_dev_real_dev(root_dev
);
1484 else if (is_cxgb3_dev(root_dev
))
1492 static struct rtable
*find_route(struct net_device
*dev
,
1493 __be32 saddr
, __be32 daddr
,
1494 __be16 sport
, __be16 dport
)
1498 .oif
= dev
? dev
->ifindex
: 0,
1504 .proto
= IPPROTO_TCP
,
1508 .dport
= dport
} } };
1510 if (ip_route_output_flow(&init_net
, &rt
, &fl
, NULL
, 0))
1516 * Assign offload parameters to some connection fields.
1518 static void init_offload_conn(struct s3_conn
*c3cn
,
1519 struct t3cdev
*cdev
,
1520 struct dst_entry
*dst
)
1522 BUG_ON(c3cn
->cdev
!= cdev
);
1523 c3cn
->wr_max
= c3cn
->wr_avail
= T3C_DATA(cdev
)->max_wrs
- 1;
1524 c3cn
->wr_unacked
= 0;
1525 c3cn
->mss_idx
= select_mss(c3cn
, dst_mtu(dst
));
1527 reset_wr_list(c3cn
);
1530 static int initiate_act_open(struct s3_conn
*c3cn
, struct net_device
*dev
)
1532 struct cxgb3i_sdev_data
*cdata
= NDEV2CDATA(dev
);
1533 struct t3cdev
*cdev
= cdata
->cdev
;
1534 struct dst_entry
*dst
= c3cn
->dst_cache
;
1535 struct sk_buff
*skb
;
1537 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1538 c3cn
, c3cn
->state
, c3cn
->flags
);
1540 * Initialize connection data. Note that the flags and ULP mode are
1541 * initialized higher up ...
1545 c3cn
->tid
= cxgb3_alloc_atid(cdev
, cdata
->client
, c3cn
);
1550 c3cn
->l2t
= t3_l2t_get(cdev
, dst
->neighbour
, dev
);
1554 skb
= alloc_skb(sizeof(struct cpl_act_open_req
), GFP_KERNEL
);
1558 skb
->sk
= (struct sock
*)c3cn
;
1559 set_arp_failure_handler(skb
, act_open_req_arp_failure
);
1563 init_offload_conn(c3cn
, cdev
, dst
);
1566 make_act_open_req(c3cn
, skb
, c3cn
->tid
, c3cn
->l2t
);
1567 l2t_send(cdev
, skb
, c3cn
->l2t
);
1571 l2t_release(L2DATA(cdev
), c3cn
->l2t
);
1573 s3_free_atid(cdev
, c3cn
->tid
);
1580 * cxgb3i_find_dev - find the interface associated with the given address
1581 * @ipaddr: ip address
1583 static struct net_device
*
1584 cxgb3i_find_dev(struct net_device
*dev
, __be32 ipaddr
)
1590 memset(&fl
, 0, sizeof(fl
));
1591 fl
.nl_u
.ip4_u
.daddr
= ipaddr
;
1593 err
= ip_route_output_key(dev
? dev_net(dev
) : &init_net
, &rt
, &fl
);
1595 return (&rt
->dst
)->dev
;
1601 * cxgb3i_c3cn_connect - initiates an iscsi tcp connection to a given address
1602 * @c3cn: the iscsi tcp connection
1603 * @usin: destination address
1605 * return 0 if active open request is sent, < 0 otherwise.
1607 int cxgb3i_c3cn_connect(struct net_device
*dev
, struct s3_conn
*c3cn
,
1608 struct sockaddr_in
*usin
)
1611 struct cxgb3i_sdev_data
*cdata
;
1612 struct t3cdev
*cdev
;
1614 struct net_device
*dstdev
;
1617 c3cn_conn_debug("c3cn 0x%p, dev 0x%p.\n", c3cn
, dev
);
1619 if (usin
->sin_family
!= AF_INET
)
1620 return -EAFNOSUPPORT
;
1622 c3cn
->daddr
.sin_port
= usin
->sin_port
;
1623 c3cn
->daddr
.sin_addr
.s_addr
= usin
->sin_addr
.s_addr
;
1625 dstdev
= cxgb3i_find_dev(dev
, usin
->sin_addr
.s_addr
);
1626 if (!dstdev
|| !is_cxgb3_dev(dstdev
))
1627 return -ENETUNREACH
;
1629 if (dstdev
->priv_flags
& IFF_802_1Q_VLAN
)
1632 rt
= find_route(dev
, c3cn
->saddr
.sin_addr
.s_addr
,
1633 c3cn
->daddr
.sin_addr
.s_addr
,
1634 c3cn
->saddr
.sin_port
,
1635 c3cn
->daddr
.sin_port
);
1637 c3cn_conn_debug("NO route to 0x%x, port %u, dev %s.\n",
1638 c3cn
->daddr
.sin_addr
.s_addr
,
1639 ntohs(c3cn
->daddr
.sin_port
),
1640 dev
? dev
->name
: "any");
1641 return -ENETUNREACH
;
1644 if (rt
->rt_flags
& (RTCF_MULTICAST
| RTCF_BROADCAST
)) {
1645 c3cn_conn_debug("multi-cast route to 0x%x, port %u, dev %s.\n",
1646 c3cn
->daddr
.sin_addr
.s_addr
,
1647 ntohs(c3cn
->daddr
.sin_port
),
1648 dev
? dev
->name
: "any");
1650 return -ENETUNREACH
;
1653 if (!c3cn
->saddr
.sin_addr
.s_addr
)
1654 c3cn
->saddr
.sin_addr
.s_addr
= rt
->rt_src
;
1656 /* now commit destination to connection */
1657 c3cn
->dst_cache
= &rt
->dst
;
1659 /* try to establish an offloaded connection */
1660 dev
= cxgb3_egress_dev(c3cn
->dst_cache
->dev
, c3cn
, 0);
1662 c3cn_conn_debug("c3cn 0x%p, egress dev NULL.\n", c3cn
);
1663 return -ENETUNREACH
;
1665 cdata
= NDEV2CDATA(dev
);
1668 /* get a source port if one hasn't been provided */
1669 err
= c3cn_get_port(c3cn
, cdata
);
1673 c3cn_conn_debug("c3cn 0x%p get port %u.\n",
1674 c3cn
, ntohs(c3cn
->saddr
.sin_port
));
1676 sipv4
= cxgb3i_get_private_ipv4addr(dev
);
1678 c3cn_conn_debug("c3cn 0x%p, iscsi ip not configured.\n", c3cn
);
1679 sipv4
= c3cn
->saddr
.sin_addr
.s_addr
;
1680 cxgb3i_set_private_ipv4addr(dev
, sipv4
);
1682 c3cn
->saddr
.sin_addr
.s_addr
= sipv4
;
1684 c3cn_conn_debug("c3cn 0x%p, %pI4,%u-%pI4,%u SYN_SENT.\n",
1686 &c3cn
->saddr
.sin_addr
.s_addr
,
1687 ntohs(c3cn
->saddr
.sin_port
),
1688 &c3cn
->daddr
.sin_addr
.s_addr
,
1689 ntohs(c3cn
->daddr
.sin_port
));
1691 c3cn_set_state(c3cn
, C3CN_STATE_CONNECTING
);
1692 if (!initiate_act_open(c3cn
, dev
))
1696 * If we get here, we don't have an offload connection so simply
1702 * This trashes the connection and releases the local port,
1705 c3cn_conn_debug("c3cn 0x%p -> CLOSED.\n", c3cn
);
1706 c3cn_set_state(c3cn
, C3CN_STATE_CLOSED
);
1708 c3cn_put_port(c3cn
);
1713 * cxgb3i_c3cn_rx_credits - ack received tcp data.
1714 * @c3cn: iscsi tcp connection
1715 * @copied: # of bytes processed
1717 * Called after some received data has been read. It returns RX credits
1718 * to the HW for the amount of data processed.
1720 void cxgb3i_c3cn_rx_credits(struct s3_conn
*c3cn
, int copied
)
1722 struct t3cdev
*cdev
;
1724 u32 credits
, dack
= 0;
1726 if (c3cn
->state
!= C3CN_STATE_ESTABLISHED
)
1729 credits
= c3cn
->copied_seq
- c3cn
->rcv_wup
;
1730 if (unlikely(!credits
))
1735 if (unlikely(cxgb3_rx_credit_thres
== 0))
1738 dack
= F_RX_DACK_CHANGE
| V_RX_DACK_MODE(1);
1741 * For coalescing to work effectively ensure the receive window has
1742 * at least 16KB left.
1744 must_send
= credits
+ 16384 >= cxgb3_rcv_win
;
1746 if (must_send
|| credits
>= cxgb3_rx_credit_thres
)
1747 c3cn
->rcv_wup
+= send_rx_credits(c3cn
, credits
, dack
);
1751 * cxgb3i_c3cn_send_pdus - send the skbs containing iscsi pdus
1752 * @c3cn: iscsi tcp connection
1753 * @skb: skb contains the iscsi pdu
1755 * Add a list of skbs to a connection send queue. The skbs must comply with
1756 * the max size limit of the device and have a headroom of at least
1757 * TX_HEADER_LEN bytes.
1758 * Return # of bytes queued.
1760 int cxgb3i_c3cn_send_pdus(struct s3_conn
*c3cn
, struct sk_buff
*skb
)
1762 struct sk_buff
*next
;
1763 int err
, copied
= 0;
1765 spin_lock_bh(&c3cn
->lock
);
1767 if (c3cn
->state
!= C3CN_STATE_ESTABLISHED
) {
1768 c3cn_tx_debug("c3cn 0x%p, not in est. state %u.\n",
1775 c3cn_tx_debug("c3cn 0x%p, err %d.\n", c3cn
, c3cn
->err
);
1780 if (c3cn
->write_seq
- c3cn
->snd_una
>= cxgb3_snd_win
) {
1781 c3cn_tx_debug("c3cn 0x%p, snd %u - %u > %u.\n",
1782 c3cn
, c3cn
->write_seq
, c3cn
->snd_una
,
1789 int frags
= skb_shinfo(skb
)->nr_frags
+
1790 (skb
->len
!= skb
->data_len
);
1792 if (unlikely(skb_headroom(skb
) < TX_HEADER_LEN
)) {
1793 c3cn_tx_debug("c3cn 0x%p, skb head.\n", c3cn
);
1798 if (frags
>= SKB_WR_LIST_SIZE
) {
1799 cxgb3i_log_error("c3cn 0x%p, tx frags %d, len %u,%u.\n",
1800 c3cn
, skb_shinfo(skb
)->nr_frags
,
1801 skb
->len
, skb
->data_len
);
1808 skb_entail(c3cn
, skb
, C3CB_FLAG_NO_APPEND
| C3CB_FLAG_NEED_HDR
);
1810 c3cn
->write_seq
+= skb
->len
+ ulp_extra_len(skb
);
1814 if (likely(skb_queue_len(&c3cn
->write_queue
)))
1815 c3cn_push_tx_frames(c3cn
, 1);
1816 spin_unlock_bh(&c3cn
->lock
);
1820 if (copied
== 0 && err
== -EPIPE
)
1821 copied
= c3cn
->err
? c3cn
->err
: -EPIPE
;
1827 static void sdev_data_cleanup(struct cxgb3i_sdev_data
*cdata
)
1829 struct adap_ports
*ports
= &cdata
->ports
;
1830 struct s3_conn
*c3cn
;
1833 for (i
= 0; i
< cxgb3_max_connect
; i
++) {
1834 if (cdata
->sport_conn
[i
]) {
1835 c3cn
= cdata
->sport_conn
[i
];
1836 cdata
->sport_conn
[i
] = NULL
;
1838 spin_lock_bh(&c3cn
->lock
);
1840 c3cn_set_flag(c3cn
, C3CN_OFFLOAD_DOWN
);
1842 spin_unlock_bh(&c3cn
->lock
);
1846 for (i
= 0; i
< ports
->nports
; i
++)
1847 NDEV2CDATA(ports
->lldevs
[i
]) = NULL
;
1849 cxgb3i_free_big_mem(cdata
);
1852 void cxgb3i_sdev_cleanup(void)
1854 struct cxgb3i_sdev_data
*cdata
;
1856 write_lock(&cdata_rwlock
);
1857 list_for_each_entry(cdata
, &cdata_list
, list
) {
1858 list_del(&cdata
->list
);
1859 sdev_data_cleanup(cdata
);
1861 write_unlock(&cdata_rwlock
);
1864 int cxgb3i_sdev_init(cxgb3_cpl_handler_func
*cpl_handlers
)
1866 cpl_handlers
[CPL_ACT_ESTABLISH
] = do_act_establish
;
1867 cpl_handlers
[CPL_ACT_OPEN_RPL
] = do_act_open_rpl
;
1868 cpl_handlers
[CPL_PEER_CLOSE
] = do_peer_close
;
1869 cpl_handlers
[CPL_ABORT_REQ_RSS
] = do_abort_req
;
1870 cpl_handlers
[CPL_ABORT_RPL_RSS
] = do_abort_rpl
;
1871 cpl_handlers
[CPL_CLOSE_CON_RPL
] = do_close_con_rpl
;
1872 cpl_handlers
[CPL_TX_DMA_ACK
] = do_wr_ack
;
1873 cpl_handlers
[CPL_ISCSI_HDR
] = do_iscsi_hdr
;
1875 if (cxgb3_max_connect
> CXGB3I_MAX_CONN
)
1876 cxgb3_max_connect
= CXGB3I_MAX_CONN
;
1881 * cxgb3i_sdev_add - allocate and initialize resources for each adapter found
1882 * @cdev: t3cdev adapter
1883 * @client: cxgb3 driver client
1885 void cxgb3i_sdev_add(struct t3cdev
*cdev
, struct cxgb3_client
*client
)
1887 struct cxgb3i_sdev_data
*cdata
;
1888 struct ofld_page_info rx_page_info
;
1889 unsigned int wr_len
;
1890 int mapsize
= cxgb3_max_connect
* sizeof(struct s3_conn
*);
1893 cdata
= cxgb3i_alloc_big_mem(sizeof(*cdata
) + mapsize
, GFP_KERNEL
);
1895 cxgb3i_log_warn("t3dev 0x%p, offload up, OOM %d.\n",
1900 if (cdev
->ctl(cdev
, GET_WR_LEN
, &wr_len
) < 0 ||
1901 cdev
->ctl(cdev
, GET_PORTS
, &cdata
->ports
) < 0 ||
1902 cdev
->ctl(cdev
, GET_RX_PAGE_INFO
, &rx_page_info
) < 0) {
1903 cxgb3i_log_warn("t3dev 0x%p, offload up, ioctl failed.\n",
1908 s3_init_wr_tab(wr_len
);
1910 spin_lock_init(&cdata
->lock
);
1911 INIT_LIST_HEAD(&cdata
->list
);
1913 cdata
->client
= client
;
1915 for (i
= 0; i
< cdata
->ports
.nports
; i
++)
1916 NDEV2CDATA(cdata
->ports
.lldevs
[i
]) = cdata
;
1918 write_lock(&cdata_rwlock
);
1919 list_add_tail(&cdata
->list
, &cdata_list
);
1920 write_unlock(&cdata_rwlock
);
1922 cxgb3i_log_info("t3dev 0x%p, offload up, added.\n", cdev
);
1926 cxgb3i_free_big_mem(cdata
);
1930 * cxgb3i_sdev_remove - free the allocated resources for the adapter
1931 * @cdev: t3cdev adapter
1933 void cxgb3i_sdev_remove(struct t3cdev
*cdev
)
1935 struct cxgb3i_sdev_data
*cdata
= CXGB3_SDEV_DATA(cdev
);
1937 cxgb3i_log_info("t3dev 0x%p, offload down, remove.\n", cdev
);
1939 write_lock(&cdata_rwlock
);
1940 list_del(&cdata
->list
);
1941 write_unlock(&cdata_rwlock
);
1943 sdev_data_cleanup(cdata
);