2 * Copyright (c) 2016 Chelsio Communications, Inc.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
9 #include <linux/module.h>
10 #include <linux/list.h>
11 #include <linux/workqueue.h>
12 #include <linux/skbuff.h>
13 #include <linux/timer.h>
14 #include <linux/notifier.h>
15 #include <linux/inetdevice.h>
17 #include <linux/tcp.h>
18 #include <linux/if_vlan.h>
20 #include <net/neighbour.h>
21 #include <net/netevent.h>
22 #include <net/route.h>
24 #include <net/ip6_route.h>
25 #include <net/addrconf.h>
27 #include <libcxgb_cm.h>
31 static void cxgbit_init_wr_wait(struct cxgbit_wr_wait
*wr_waitp
)
34 reinit_completion(&wr_waitp
->completion
);
38 cxgbit_wake_up(struct cxgbit_wr_wait
*wr_waitp
, const char *func
, u8 ret
)
40 if (ret
== CPL_ERR_NONE
)
46 pr_err("%s: err:%u", func
, ret
);
48 complete(&wr_waitp
->completion
);
52 cxgbit_wait_for_reply(struct cxgbit_device
*cdev
,
53 struct cxgbit_wr_wait
*wr_waitp
, u32 tid
, u32 timeout
,
58 if (!test_bit(CDEV_STATE_UP
, &cdev
->flags
)) {
63 ret
= wait_for_completion_timeout(&wr_waitp
->completion
, timeout
* HZ
);
65 pr_info("%s - Device %s not responding tid %u\n",
66 func
, pci_name(cdev
->lldi
.pdev
), tid
);
67 wr_waitp
->ret
= -ETIMEDOUT
;
71 pr_info("%s: FW reply %d tid %u\n",
72 pci_name(cdev
->lldi
.pdev
), wr_waitp
->ret
, tid
);
76 /* Returns whether a CPL status conveys negative advice.
78 static int cxgbit_is_neg_adv(unsigned int status
)
80 return status
== CPL_ERR_RTX_NEG_ADVICE
||
81 status
== CPL_ERR_PERSIST_NEG_ADVICE
||
82 status
== CPL_ERR_KEEPALV_NEG_ADVICE
;
85 static int cxgbit_np_hashfn(const struct cxgbit_np
*cnp
)
87 return ((unsigned long)cnp
>> 10) & (NP_INFO_HASH_SIZE
- 1);
90 static struct np_info
*
91 cxgbit_np_hash_add(struct cxgbit_device
*cdev
, struct cxgbit_np
*cnp
,
94 struct np_info
*p
= kzalloc(sizeof(*p
), GFP_KERNEL
);
97 int bucket
= cxgbit_np_hashfn(cnp
);
101 spin_lock(&cdev
->np_lock
);
102 p
->next
= cdev
->np_hash_tab
[bucket
];
103 cdev
->np_hash_tab
[bucket
] = p
;
104 spin_unlock(&cdev
->np_lock
);
111 cxgbit_np_hash_find(struct cxgbit_device
*cdev
, struct cxgbit_np
*cnp
)
113 int stid
= -1, bucket
= cxgbit_np_hashfn(cnp
);
116 spin_lock(&cdev
->np_lock
);
117 for (p
= cdev
->np_hash_tab
[bucket
]; p
; p
= p
->next
) {
123 spin_unlock(&cdev
->np_lock
);
128 static int cxgbit_np_hash_del(struct cxgbit_device
*cdev
, struct cxgbit_np
*cnp
)
130 int stid
= -1, bucket
= cxgbit_np_hashfn(cnp
);
131 struct np_info
*p
, **prev
= &cdev
->np_hash_tab
[bucket
];
133 spin_lock(&cdev
->np_lock
);
134 for (p
= *prev
; p
; prev
= &p
->next
, p
= p
->next
) {
142 spin_unlock(&cdev
->np_lock
);
147 void _cxgbit_free_cnp(struct kref
*kref
)
149 struct cxgbit_np
*cnp
;
151 cnp
= container_of(kref
, struct cxgbit_np
, kref
);
156 cxgbit_create_server6(struct cxgbit_device
*cdev
, unsigned int stid
,
157 struct cxgbit_np
*cnp
)
159 struct sockaddr_in6
*sin6
= (struct sockaddr_in6
*)
160 &cnp
->com
.local_addr
;
164 pr_debug("%s: dev = %s; stid = %u; sin6_port = %u\n",
165 __func__
, cdev
->lldi
.ports
[0]->name
, stid
, sin6
->sin6_port
);
167 addr_type
= ipv6_addr_type((const struct in6_addr
*)
169 if (addr_type
!= IPV6_ADDR_ANY
) {
170 ret
= cxgb4_clip_get(cdev
->lldi
.ports
[0],
171 (const u32
*)&sin6
->sin6_addr
.s6_addr
, 1);
173 pr_err("Unable to find clip table entry. laddr %pI6. Error:%d.\n",
174 sin6
->sin6_addr
.s6_addr
, ret
);
180 cxgbit_init_wr_wait(&cnp
->com
.wr_wait
);
182 ret
= cxgb4_create_server6(cdev
->lldi
.ports
[0],
183 stid
, &sin6
->sin6_addr
,
185 cdev
->lldi
.rxq_ids
[0]);
187 ret
= cxgbit_wait_for_reply(cdev
, &cnp
->com
.wr_wait
,
190 ret
= net_xmit_errno(ret
);
195 if (ret
!= -ETIMEDOUT
)
196 cxgb4_clip_release(cdev
->lldi
.ports
[0],
197 (const u32
*)&sin6
->sin6_addr
.s6_addr
, 1);
199 pr_err("create server6 err %d stid %d laddr %pI6 lport %d\n",
200 ret
, stid
, sin6
->sin6_addr
.s6_addr
,
201 ntohs(sin6
->sin6_port
));
208 cxgbit_create_server4(struct cxgbit_device
*cdev
, unsigned int stid
,
209 struct cxgbit_np
*cnp
)
211 struct sockaddr_in
*sin
= (struct sockaddr_in
*)
212 &cnp
->com
.local_addr
;
215 pr_debug("%s: dev = %s; stid = %u; sin_port = %u\n",
216 __func__
, cdev
->lldi
.ports
[0]->name
, stid
, sin
->sin_port
);
219 cxgbit_init_wr_wait(&cnp
->com
.wr_wait
);
221 ret
= cxgb4_create_server(cdev
->lldi
.ports
[0],
222 stid
, sin
->sin_addr
.s_addr
,
224 cdev
->lldi
.rxq_ids
[0]);
226 ret
= cxgbit_wait_for_reply(cdev
,
230 ret
= net_xmit_errno(ret
);
235 pr_err("create server failed err %d stid %d laddr %pI4 lport %d\n",
236 ret
, stid
, &sin
->sin_addr
, ntohs(sin
->sin_port
));
240 struct cxgbit_device
*cxgbit_find_device(struct net_device
*ndev
, u8
*port_id
)
242 struct cxgbit_device
*cdev
;
245 list_for_each_entry(cdev
, &cdev_list_head
, list
) {
246 struct cxgb4_lld_info
*lldi
= &cdev
->lldi
;
248 for (i
= 0; i
< lldi
->nports
; i
++) {
249 if (lldi
->ports
[i
] == ndev
) {
260 static struct net_device
*cxgbit_get_real_dev(struct net_device
*ndev
)
262 if (ndev
->priv_flags
& IFF_BONDING
) {
263 pr_err("Bond devices are not supported. Interface:%s\n",
268 if (is_vlan_dev(ndev
))
269 return vlan_dev_real_dev(ndev
);
274 static struct net_device
*cxgbit_ipv4_netdev(__be32 saddr
)
276 struct net_device
*ndev
;
278 ndev
= __ip_dev_find(&init_net
, saddr
, false);
282 return cxgbit_get_real_dev(ndev
);
285 static struct net_device
*cxgbit_ipv6_netdev(struct in6_addr
*addr6
)
287 struct net_device
*ndev
= NULL
;
290 if (IS_ENABLED(CONFIG_IPV6
)) {
291 for_each_netdev_rcu(&init_net
, ndev
)
292 if (ipv6_chk_addr(&init_net
, addr6
, ndev
, 1)) {
299 return cxgbit_get_real_dev(ndev
);
302 static struct cxgbit_device
*cxgbit_find_np_cdev(struct cxgbit_np
*cnp
)
304 struct sockaddr_storage
*sockaddr
= &cnp
->com
.local_addr
;
305 int ss_family
= sockaddr
->ss_family
;
306 struct net_device
*ndev
= NULL
;
307 struct cxgbit_device
*cdev
= NULL
;
310 if (ss_family
== AF_INET
) {
311 struct sockaddr_in
*sin
;
313 sin
= (struct sockaddr_in
*)sockaddr
;
314 ndev
= cxgbit_ipv4_netdev(sin
->sin_addr
.s_addr
);
315 } else if (ss_family
== AF_INET6
) {
316 struct sockaddr_in6
*sin6
;
318 sin6
= (struct sockaddr_in6
*)sockaddr
;
319 ndev
= cxgbit_ipv6_netdev(&sin6
->sin6_addr
);
324 cdev
= cxgbit_find_device(ndev
, NULL
);
330 static bool cxgbit_inaddr_any(struct cxgbit_np
*cnp
)
332 struct sockaddr_storage
*sockaddr
= &cnp
->com
.local_addr
;
333 int ss_family
= sockaddr
->ss_family
;
336 if (ss_family
== AF_INET
) {
337 struct sockaddr_in
*sin
;
339 sin
= (struct sockaddr_in
*)sockaddr
;
340 if (sin
->sin_addr
.s_addr
== htonl(INADDR_ANY
))
342 } else if (ss_family
== AF_INET6
) {
343 struct sockaddr_in6
*sin6
;
345 sin6
= (struct sockaddr_in6
*)sockaddr
;
346 addr_type
= ipv6_addr_type((const struct in6_addr
*)
348 if (addr_type
== IPV6_ADDR_ANY
)
355 __cxgbit_setup_cdev_np(struct cxgbit_device
*cdev
, struct cxgbit_np
*cnp
)
358 int ss_family
= cnp
->com
.local_addr
.ss_family
;
360 if (!test_bit(CDEV_STATE_UP
, &cdev
->flags
))
363 stid
= cxgb4_alloc_stid(cdev
->lldi
.tids
, ss_family
, cnp
);
367 if (!cxgbit_np_hash_add(cdev
, cnp
, stid
)) {
368 cxgb4_free_stid(cdev
->lldi
.tids
, stid
, ss_family
);
372 if (ss_family
== AF_INET
)
373 ret
= cxgbit_create_server4(cdev
, stid
, cnp
);
375 ret
= cxgbit_create_server6(cdev
, stid
, cnp
);
378 if (ret
!= -ETIMEDOUT
)
379 cxgb4_free_stid(cdev
->lldi
.tids
, stid
,
381 cxgbit_np_hash_del(cdev
, cnp
);
387 static int cxgbit_setup_cdev_np(struct cxgbit_np
*cnp
)
389 struct cxgbit_device
*cdev
;
392 mutex_lock(&cdev_list_lock
);
393 cdev
= cxgbit_find_np_cdev(cnp
);
397 if (cxgbit_np_hash_find(cdev
, cnp
) >= 0)
400 if (__cxgbit_setup_cdev_np(cdev
, cnp
))
403 cnp
->com
.cdev
= cdev
;
406 mutex_unlock(&cdev_list_lock
);
410 static int cxgbit_setup_all_np(struct cxgbit_np
*cnp
)
412 struct cxgbit_device
*cdev
;
416 mutex_lock(&cdev_list_lock
);
417 list_for_each_entry(cdev
, &cdev_list_head
, list
) {
418 if (cxgbit_np_hash_find(cdev
, cnp
) >= 0) {
419 mutex_unlock(&cdev_list_lock
);
424 list_for_each_entry(cdev
, &cdev_list_head
, list
) {
425 ret
= __cxgbit_setup_cdev_np(cdev
, cnp
);
426 if (ret
== -ETIMEDOUT
)
432 mutex_unlock(&cdev_list_lock
);
434 return count
? 0 : -1;
437 int cxgbit_setup_np(struct iscsi_np
*np
, struct sockaddr_storage
*ksockaddr
)
439 struct cxgbit_np
*cnp
;
442 if ((ksockaddr
->ss_family
!= AF_INET
) &&
443 (ksockaddr
->ss_family
!= AF_INET6
))
446 cnp
= kzalloc(sizeof(*cnp
), GFP_KERNEL
);
450 init_waitqueue_head(&cnp
->accept_wait
);
451 init_completion(&cnp
->com
.wr_wait
.completion
);
452 init_completion(&cnp
->accept_comp
);
453 INIT_LIST_HEAD(&cnp
->np_accept_list
);
454 spin_lock_init(&cnp
->np_accept_lock
);
455 kref_init(&cnp
->kref
);
456 memcpy(&np
->np_sockaddr
, ksockaddr
,
457 sizeof(struct sockaddr_storage
));
458 memcpy(&cnp
->com
.local_addr
, &np
->np_sockaddr
,
459 sizeof(cnp
->com
.local_addr
));
462 cnp
->com
.cdev
= NULL
;
464 if (cxgbit_inaddr_any(cnp
))
465 ret
= cxgbit_setup_all_np(cnp
);
467 ret
= cxgbit_setup_cdev_np(cnp
);
474 np
->np_context
= cnp
;
475 cnp
->com
.state
= CSK_STATE_LISTEN
;
480 cxgbit_set_conn_info(struct iscsi_np
*np
, struct iscsi_conn
*conn
,
481 struct cxgbit_sock
*csk
)
483 conn
->login_family
= np
->np_sockaddr
.ss_family
;
484 conn
->login_sockaddr
= csk
->com
.remote_addr
;
485 conn
->local_sockaddr
= csk
->com
.local_addr
;
488 int cxgbit_accept_np(struct iscsi_np
*np
, struct iscsi_conn
*conn
)
490 struct cxgbit_np
*cnp
= np
->np_context
;
491 struct cxgbit_sock
*csk
;
495 ret
= wait_for_completion_interruptible(&cnp
->accept_comp
);
499 spin_lock_bh(&np
->np_thread_lock
);
500 if (np
->np_thread_state
>= ISCSI_NP_THREAD_RESET
) {
501 spin_unlock_bh(&np
->np_thread_lock
);
503 * No point in stalling here when np_thread
504 * is in state RESET/SHUTDOWN/EXIT - bail
508 spin_unlock_bh(&np
->np_thread_lock
);
510 spin_lock_bh(&cnp
->np_accept_lock
);
511 if (list_empty(&cnp
->np_accept_list
)) {
512 spin_unlock_bh(&cnp
->np_accept_lock
);
516 csk
= list_first_entry(&cnp
->np_accept_list
,
520 list_del_init(&csk
->accept_node
);
521 spin_unlock_bh(&cnp
->np_accept_lock
);
525 cxgbit_set_conn_info(np
, conn
, csk
);
530 __cxgbit_free_cdev_np(struct cxgbit_device
*cdev
, struct cxgbit_np
*cnp
)
535 stid
= cxgbit_np_hash_del(cdev
, cnp
);
538 if (!test_bit(CDEV_STATE_UP
, &cdev
->flags
))
541 if (cnp
->np
->np_sockaddr
.ss_family
== AF_INET6
)
545 cxgbit_init_wr_wait(&cnp
->com
.wr_wait
);
546 ret
= cxgb4_remove_server(cdev
->lldi
.ports
[0], stid
,
547 cdev
->lldi
.rxq_ids
[0], ipv6
);
550 ret
= net_xmit_errno(ret
);
557 ret
= cxgbit_wait_for_reply(cdev
, &cnp
->com
.wr_wait
,
559 if (ret
== -ETIMEDOUT
)
562 if (ipv6
&& cnp
->com
.cdev
) {
563 struct sockaddr_in6
*sin6
;
565 sin6
= (struct sockaddr_in6
*)&cnp
->com
.local_addr
;
566 cxgb4_clip_release(cdev
->lldi
.ports
[0],
567 (const u32
*)&sin6
->sin6_addr
.s6_addr
,
571 cxgb4_free_stid(cdev
->lldi
.tids
, stid
,
572 cnp
->com
.local_addr
.ss_family
);
576 static void cxgbit_free_all_np(struct cxgbit_np
*cnp
)
578 struct cxgbit_device
*cdev
;
581 mutex_lock(&cdev_list_lock
);
582 list_for_each_entry(cdev
, &cdev_list_head
, list
) {
583 ret
= __cxgbit_free_cdev_np(cdev
, cnp
);
584 if (ret
== -ETIMEDOUT
)
587 mutex_unlock(&cdev_list_lock
);
590 static void cxgbit_free_cdev_np(struct cxgbit_np
*cnp
)
592 struct cxgbit_device
*cdev
;
595 mutex_lock(&cdev_list_lock
);
596 list_for_each_entry(cdev
, &cdev_list_head
, list
) {
597 if (cdev
== cnp
->com
.cdev
) {
605 __cxgbit_free_cdev_np(cdev
, cnp
);
607 mutex_unlock(&cdev_list_lock
);
610 void cxgbit_free_np(struct iscsi_np
*np
)
612 struct cxgbit_np
*cnp
= np
->np_context
;
614 cnp
->com
.state
= CSK_STATE_DEAD
;
616 cxgbit_free_cdev_np(cnp
);
618 cxgbit_free_all_np(cnp
);
620 np
->np_context
= NULL
;
624 static void cxgbit_send_halfclose(struct cxgbit_sock
*csk
)
627 struct cpl_close_con_req
*req
;
628 unsigned int len
= roundup(sizeof(struct cpl_close_con_req
), 16);
630 skb
= alloc_skb(len
, GFP_ATOMIC
);
634 req
= (struct cpl_close_con_req
*)__skb_put(skb
, len
);
637 set_wr_txq(skb
, CPL_PRIORITY_DATA
, csk
->txq_idx
);
638 INIT_TP_WR(req
, csk
->tid
);
639 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ
,
643 cxgbit_skcb_flags(skb
) |= SKCBF_TX_FLAG_COMPL
;
644 __skb_queue_tail(&csk
->txq
, skb
);
645 cxgbit_push_tx_frames(csk
);
648 static void cxgbit_arp_failure_discard(void *handle
, struct sk_buff
*skb
)
650 pr_debug("%s cxgbit_device %p\n", __func__
, handle
);
654 static void cxgbit_abort_arp_failure(void *handle
, struct sk_buff
*skb
)
656 struct cxgbit_device
*cdev
= handle
;
657 struct cpl_abort_req
*req
= cplhdr(skb
);
659 pr_debug("%s cdev %p\n", __func__
, cdev
);
660 req
->cmd
= CPL_ABORT_NO_RST
;
661 cxgbit_ofld_send(cdev
, skb
);
664 static int cxgbit_send_abort_req(struct cxgbit_sock
*csk
)
666 struct cpl_abort_req
*req
;
667 unsigned int len
= roundup(sizeof(*req
), 16);
670 pr_debug("%s: csk %p tid %u; state %d\n",
671 __func__
, csk
, csk
->tid
, csk
->com
.state
);
673 __skb_queue_purge(&csk
->txq
);
675 if (!test_and_set_bit(CSK_TX_DATA_SENT
, &csk
->com
.flags
))
676 cxgbit_send_tx_flowc_wr(csk
);
678 skb
= __skb_dequeue(&csk
->skbq
);
679 req
= (struct cpl_abort_req
*)__skb_put(skb
, len
);
682 set_wr_txq(skb
, CPL_PRIORITY_DATA
, csk
->txq_idx
);
683 t4_set_arp_err_handler(skb
, csk
->com
.cdev
, cxgbit_abort_arp_failure
);
684 INIT_TP_WR(req
, csk
->tid
);
685 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ
,
687 req
->cmd
= CPL_ABORT_SEND_RST
;
688 return cxgbit_l2t_send(csk
->com
.cdev
, skb
, csk
->l2t
);
691 void cxgbit_free_conn(struct iscsi_conn
*conn
)
693 struct cxgbit_sock
*csk
= conn
->context
;
694 bool release
= false;
696 pr_debug("%s: state %d\n",
697 __func__
, csk
->com
.state
);
699 spin_lock_bh(&csk
->lock
);
700 switch (csk
->com
.state
) {
701 case CSK_STATE_ESTABLISHED
:
702 if (conn
->conn_state
== TARG_CONN_STATE_IN_LOGOUT
) {
703 csk
->com
.state
= CSK_STATE_CLOSING
;
704 cxgbit_send_halfclose(csk
);
706 csk
->com
.state
= CSK_STATE_ABORTING
;
707 cxgbit_send_abort_req(csk
);
710 case CSK_STATE_CLOSING
:
711 csk
->com
.state
= CSK_STATE_MORIBUND
;
712 cxgbit_send_halfclose(csk
);
718 pr_err("%s: csk %p; state %d\n",
719 __func__
, csk
, csk
->com
.state
);
721 spin_unlock_bh(&csk
->lock
);
727 static void cxgbit_set_emss(struct cxgbit_sock
*csk
, u16 opt
)
729 csk
->emss
= csk
->com
.cdev
->lldi
.mtus
[TCPOPT_MSS_G(opt
)] -
730 ((csk
->com
.remote_addr
.ss_family
== AF_INET
) ?
731 sizeof(struct iphdr
) : sizeof(struct ipv6hdr
)) -
732 sizeof(struct tcphdr
);
733 csk
->mss
= csk
->emss
;
734 if (TCPOPT_TSTAMP_G(opt
))
735 csk
->emss
-= round_up(TCPOLEN_TIMESTAMP
, 4);
739 pr_info("Warning: misaligned mtu idx %u mss %u emss=%u\n",
740 TCPOPT_MSS_G(opt
), csk
->mss
, csk
->emss
);
741 pr_debug("%s mss_idx %u mss %u emss=%u\n", __func__
, TCPOPT_MSS_G(opt
),
742 csk
->mss
, csk
->emss
);
745 static void cxgbit_free_skb(struct cxgbit_sock
*csk
)
749 __skb_queue_purge(&csk
->txq
);
750 __skb_queue_purge(&csk
->rxq
);
751 __skb_queue_purge(&csk
->backlogq
);
752 __skb_queue_purge(&csk
->ppodq
);
753 __skb_queue_purge(&csk
->skbq
);
755 while ((skb
= cxgbit_sock_dequeue_wr(csk
)))
758 __kfree_skb(csk
->lro_hskb
);
761 void _cxgbit_free_csk(struct kref
*kref
)
763 struct cxgbit_sock
*csk
;
764 struct cxgbit_device
*cdev
;
766 csk
= container_of(kref
, struct cxgbit_sock
, kref
);
768 pr_debug("%s csk %p state %d\n", __func__
, csk
, csk
->com
.state
);
770 if (csk
->com
.local_addr
.ss_family
== AF_INET6
) {
771 struct sockaddr_in6
*sin6
= (struct sockaddr_in6
*)
772 &csk
->com
.local_addr
;
773 cxgb4_clip_release(csk
->com
.cdev
->lldi
.ports
[0],
775 &sin6
->sin6_addr
.s6_addr
, 1);
778 cxgb4_remove_tid(csk
->com
.cdev
->lldi
.tids
, 0, csk
->tid
);
779 dst_release(csk
->dst
);
780 cxgb4_l2t_release(csk
->l2t
);
782 cdev
= csk
->com
.cdev
;
783 spin_lock_bh(&cdev
->cskq
.lock
);
784 list_del(&csk
->list
);
785 spin_unlock_bh(&cdev
->cskq
.lock
);
787 cxgbit_free_skb(csk
);
788 cxgbit_put_cdev(cdev
);
793 static void cxgbit_set_tcp_window(struct cxgbit_sock
*csk
, struct port_info
*pi
)
795 unsigned int linkspeed
;
798 linkspeed
= pi
->link_cfg
.speed
;
799 scale
= linkspeed
/ SPEED_10000
;
801 #define CXGBIT_10G_RCV_WIN (256 * 1024)
802 csk
->rcv_win
= CXGBIT_10G_RCV_WIN
;
804 csk
->rcv_win
*= scale
;
806 #define CXGBIT_10G_SND_WIN (256 * 1024)
807 csk
->snd_win
= CXGBIT_10G_SND_WIN
;
809 csk
->snd_win
*= scale
;
811 pr_debug("%s snd_win %d rcv_win %d\n",
812 __func__
, csk
->snd_win
, csk
->rcv_win
);
815 #ifdef CONFIG_CHELSIO_T4_DCB
816 static u8
cxgbit_get_iscsi_dcb_state(struct net_device
*ndev
)
818 return ndev
->dcbnl_ops
->getstate(ndev
);
821 static int cxgbit_select_priority(int pri_mask
)
826 return (ffs(pri_mask
) - 1);
829 static u8
cxgbit_get_iscsi_dcb_priority(struct net_device
*ndev
, u16 local_port
)
834 struct dcb_app iscsi_dcb_app
= {
835 .protocol
= local_port
838 ret
= (int)ndev
->dcbnl_ops
->getcap(ndev
, DCB_CAP_ATTR_DCBX
, &caps
);
843 if (caps
& DCB_CAP_DCBX_VER_IEEE
) {
844 iscsi_dcb_app
.selector
= IEEE_8021QAZ_APP_SEL_ANY
;
846 ret
= dcb_ieee_getapp_mask(ndev
, &iscsi_dcb_app
);
848 } else if (caps
& DCB_CAP_DCBX_VER_CEE
) {
849 iscsi_dcb_app
.selector
= DCB_APP_IDTYPE_PORTNUM
;
851 ret
= dcb_getapp(ndev
, &iscsi_dcb_app
);
854 pr_info("iSCSI priority is set to %u\n", cxgbit_select_priority(ret
));
856 return cxgbit_select_priority(ret
);
861 cxgbit_offload_init(struct cxgbit_sock
*csk
, int iptype
, __u8
*peer_ip
,
862 u16 local_port
, struct dst_entry
*dst
,
863 struct cxgbit_device
*cdev
)
867 struct net_device
*ndev
;
868 u16 rxq_idx
, port_id
;
869 #ifdef CONFIG_CHELSIO_T4_DCB
873 n
= dst_neigh_lookup(dst
, peer_ip
);
879 if (n
->dev
->flags
& IFF_LOOPBACK
) {
881 ndev
= cxgbit_ipv4_netdev(*(__be32
*)peer_ip
);
882 else if (IS_ENABLED(CONFIG_IPV6
))
883 ndev
= cxgbit_ipv6_netdev((struct in6_addr
*)peer_ip
);
892 csk
->l2t
= cxgb4_l2t_get(cdev
->lldi
.l2t
,
896 csk
->mtu
= ndev
->mtu
;
897 csk
->tx_chan
= cxgb4_port_chan(ndev
);
898 csk
->smac_idx
= (cxgb4_port_viid(ndev
) & 0x7F) << 1;
899 step
= cdev
->lldi
.ntxq
/
901 csk
->txq_idx
= cxgb4_port_idx(ndev
) * step
;
902 step
= cdev
->lldi
.nrxq
/
904 csk
->ctrlq_idx
= cxgb4_port_idx(ndev
);
905 csk
->rss_qid
= cdev
->lldi
.rxq_ids
[
906 cxgb4_port_idx(ndev
) * step
];
907 csk
->port_id
= cxgb4_port_idx(ndev
);
908 cxgbit_set_tcp_window(csk
,
909 (struct port_info
*)netdev_priv(ndev
));
911 ndev
= cxgbit_get_real_dev(n
->dev
);
917 #ifdef CONFIG_CHELSIO_T4_DCB
918 if (cxgbit_get_iscsi_dcb_state(ndev
))
919 priority
= cxgbit_get_iscsi_dcb_priority(ndev
,
922 csk
->dcb_priority
= priority
;
924 csk
->l2t
= cxgb4_l2t_get(cdev
->lldi
.l2t
, n
, ndev
, priority
);
926 csk
->l2t
= cxgb4_l2t_get(cdev
->lldi
.l2t
, n
, ndev
, 0);
930 port_id
= cxgb4_port_idx(ndev
);
931 csk
->mtu
= dst_mtu(dst
);
932 csk
->tx_chan
= cxgb4_port_chan(ndev
);
933 csk
->smac_idx
= (cxgb4_port_viid(ndev
) & 0x7F) << 1;
934 step
= cdev
->lldi
.ntxq
/
936 csk
->txq_idx
= (port_id
* step
) +
937 (cdev
->selectq
[port_id
][0]++ % step
);
938 csk
->ctrlq_idx
= cxgb4_port_idx(ndev
);
939 step
= cdev
->lldi
.nrxq
/
941 rxq_idx
= (port_id
* step
) +
942 (cdev
->selectq
[port_id
][1]++ % step
);
943 csk
->rss_qid
= cdev
->lldi
.rxq_ids
[rxq_idx
];
944 csk
->port_id
= port_id
;
945 cxgbit_set_tcp_window(csk
,
946 (struct port_info
*)netdev_priv(ndev
));
955 int cxgbit_ofld_send(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
959 if (!test_bit(CDEV_STATE_UP
, &cdev
->flags
)) {
961 pr_err("%s - device not up - dropping\n", __func__
);
965 ret
= cxgb4_ofld_send(cdev
->lldi
.ports
[0], skb
);
968 return ret
< 0 ? ret
: 0;
971 static void cxgbit_release_tid(struct cxgbit_device
*cdev
, u32 tid
)
973 struct cpl_tid_release
*req
;
974 unsigned int len
= roundup(sizeof(*req
), 16);
977 skb
= alloc_skb(len
, GFP_ATOMIC
);
981 req
= (struct cpl_tid_release
*)__skb_put(skb
, len
);
984 INIT_TP_WR(req
, tid
);
985 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(
986 CPL_TID_RELEASE
, tid
));
987 set_wr_txq(skb
, CPL_PRIORITY_SETUP
, 0);
988 cxgbit_ofld_send(cdev
, skb
);
992 cxgbit_l2t_send(struct cxgbit_device
*cdev
, struct sk_buff
*skb
,
993 struct l2t_entry
*l2e
)
997 if (!test_bit(CDEV_STATE_UP
, &cdev
->flags
)) {
999 pr_err("%s - device not up - dropping\n", __func__
);
1003 ret
= cxgb4_l2t_send(cdev
->lldi
.ports
[0], skb
, l2e
);
1006 return ret
< 0 ? ret
: 0;
1010 cxgbit_best_mtu(const unsigned short *mtus
, unsigned short mtu
,
1011 unsigned int *idx
, int use_ts
, int ipv6
)
1013 unsigned short hdr_size
= (ipv6
? sizeof(struct ipv6hdr
) :
1014 sizeof(struct iphdr
)) +
1015 sizeof(struct tcphdr
) +
1016 (use_ts
? round_up(TCPOLEN_TIMESTAMP
,
1018 unsigned short data_size
= mtu
- hdr_size
;
1020 cxgb4_best_aligned_mtu(mtus
, hdr_size
, data_size
, 8, idx
);
1023 static void cxgbit_send_rx_credits(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1025 if (csk
->com
.state
!= CSK_STATE_ESTABLISHED
) {
1030 cxgbit_ofld_send(csk
->com
.cdev
, skb
);
1034 * CPL connection rx data ack: host ->
1035 * Send RX credits through an RX_DATA_ACK CPL message.
1036 * Returns the number of credits sent.
1038 int cxgbit_rx_data_ack(struct cxgbit_sock
*csk
)
1040 struct sk_buff
*skb
;
1041 struct cpl_rx_data_ack
*req
;
1042 unsigned int len
= roundup(sizeof(*req
), 16);
1044 skb
= alloc_skb(len
, GFP_KERNEL
);
1048 req
= (struct cpl_rx_data_ack
*)__skb_put(skb
, len
);
1049 memset(req
, 0, len
);
1051 set_wr_txq(skb
, CPL_PRIORITY_ACK
, csk
->ctrlq_idx
);
1052 INIT_TP_WR(req
, csk
->tid
);
1053 OPCODE_TID(req
) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK
,
1055 req
->credit_dack
= cpu_to_be32(RX_DACK_CHANGE_F
| RX_DACK_MODE_V(1) |
1056 RX_CREDITS_V(csk
->rx_credits
));
1058 csk
->rx_credits
= 0;
1060 spin_lock_bh(&csk
->lock
);
1061 if (csk
->lock_owner
) {
1062 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_send_rx_credits
;
1063 __skb_queue_tail(&csk
->backlogq
, skb
);
1064 spin_unlock_bh(&csk
->lock
);
1068 cxgbit_send_rx_credits(csk
, skb
);
1069 spin_unlock_bh(&csk
->lock
);
1074 #define FLOWC_WR_NPARAMS_MIN 9
1075 #define FLOWC_WR_NPARAMS_MAX 11
1076 static int cxgbit_alloc_csk_skb(struct cxgbit_sock
*csk
)
1078 struct sk_buff
*skb
;
1082 flowclen
= offsetof(struct fw_flowc_wr
,
1083 mnemval
[FLOWC_WR_NPARAMS_MAX
]);
1085 len
= max_t(u32
, sizeof(struct cpl_abort_req
),
1086 sizeof(struct cpl_abort_rpl
));
1088 len
= max(len
, flowclen
);
1089 len
= roundup(len
, 16);
1091 for (i
= 0; i
< 3; i
++) {
1092 skb
= alloc_skb(len
, GFP_ATOMIC
);
1095 __skb_queue_tail(&csk
->skbq
, skb
);
1098 skb
= alloc_skb(LRO_SKB_MIN_HEADROOM
, GFP_ATOMIC
);
1102 memset(skb
->data
, 0, LRO_SKB_MIN_HEADROOM
);
1103 csk
->lro_hskb
= skb
;
1107 __skb_queue_purge(&csk
->skbq
);
1111 static u32
cxgbit_compute_wscale(u32 win
)
1115 while (wscale
< 14 && (65535 << wscale
) < win
)
1121 cxgbit_pass_accept_rpl(struct cxgbit_sock
*csk
, struct cpl_pass_accept_req
*req
)
1123 struct sk_buff
*skb
;
1124 const struct tcphdr
*tcph
;
1125 struct cpl_t5_pass_accept_rpl
*rpl5
;
1126 unsigned int len
= roundup(sizeof(*rpl5
), 16);
1127 unsigned int mtu_idx
;
1133 pr_debug("%s csk %p tid %u\n", __func__
, csk
, csk
->tid
);
1135 skb
= alloc_skb(len
, GFP_ATOMIC
);
1137 cxgbit_put_csk(csk
);
1141 rpl5
= (struct cpl_t5_pass_accept_rpl
*)__skb_put(skb
, len
);
1142 memset(rpl5
, 0, len
);
1144 INIT_TP_WR(rpl5
, csk
->tid
);
1145 OPCODE_TID(rpl5
) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL
,
1147 cxgbit_best_mtu(csk
->com
.cdev
->lldi
.mtus
, csk
->mtu
, &mtu_idx
,
1149 (csk
->com
.remote_addr
.ss_family
== AF_INET
) ? 0 : 1);
1150 wscale
= cxgbit_compute_wscale(csk
->rcv_win
);
1152 * Specify the largest window that will fit in opt0. The
1153 * remainder will be specified in the rx_data_ack.
1155 win
= csk
->rcv_win
>> 10;
1156 if (win
> RCV_BUFSIZ_M
)
1158 opt0
= TCAM_BYPASS_F
|
1159 WND_SCALE_V(wscale
) |
1160 MSS_IDX_V(mtu_idx
) |
1161 L2T_IDX_V(csk
->l2t
->idx
) |
1162 TX_CHAN_V(csk
->tx_chan
) |
1163 SMAC_SEL_V(csk
->smac_idx
) |
1164 DSCP_V(csk
->tos
>> 2) |
1165 ULP_MODE_V(ULP_MODE_ISCSI
) |
1168 opt2
= RX_CHANNEL_V(0) |
1169 RSS_QUEUE_VALID_F
| RSS_QUEUE_V(csk
->rss_qid
);
1171 if (req
->tcpopt
.tstamp
)
1172 opt2
|= TSTAMPS_EN_F
;
1173 if (req
->tcpopt
.sack
)
1176 opt2
|= WND_SCALE_EN_F
;
1178 hlen
= ntohl(req
->hdr_len
);
1179 tcph
= (const void *)(req
+ 1) + ETH_HDR_LEN_G(hlen
) +
1182 if (tcph
->ece
&& tcph
->cwr
)
1183 opt2
|= CCTRL_ECN_V(1);
1185 opt2
|= RX_COALESCE_V(3);
1186 opt2
|= CONG_CNTRL_V(CONG_ALG_NEWRENO
);
1189 rpl5
->iss
= cpu_to_be32((prandom_u32() & ~7UL) - 1);
1191 opt2
|= T5_OPT_2_VALID_F
;
1193 rpl5
->opt0
= cpu_to_be64(opt0
);
1194 rpl5
->opt2
= cpu_to_be32(opt2
);
1195 set_wr_txq(skb
, CPL_PRIORITY_SETUP
, csk
->ctrlq_idx
);
1196 t4_set_arp_err_handler(skb
, NULL
, cxgbit_arp_failure_discard
);
1197 cxgbit_l2t_send(csk
->com
.cdev
, skb
, csk
->l2t
);
1201 cxgbit_pass_accept_req(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1203 struct cxgbit_sock
*csk
= NULL
;
1204 struct cxgbit_np
*cnp
;
1205 struct cpl_pass_accept_req
*req
= cplhdr(skb
);
1206 unsigned int stid
= PASS_OPEN_TID_G(ntohl(req
->tos_stid
));
1207 struct tid_info
*t
= cdev
->lldi
.tids
;
1208 unsigned int tid
= GET_TID(req
);
1209 u16 peer_mss
= ntohs(req
->tcpopt
.mss
);
1210 unsigned short hdrs
;
1212 struct dst_entry
*dst
;
1213 __u8 local_ip
[16], peer_ip
[16];
1214 __be16 local_port
, peer_port
;
1218 pr_debug("%s: cdev = %p; stid = %u; tid = %u\n",
1219 __func__
, cdev
, stid
, tid
);
1221 cnp
= lookup_stid(t
, stid
);
1223 pr_err("%s connect request on invalid stid %d\n",
1228 if (cnp
->com
.state
!= CSK_STATE_LISTEN
) {
1229 pr_err("%s - listening parent not in CSK_STATE_LISTEN\n",
1234 csk
= lookup_tid(t
, tid
);
1236 pr_err("%s csk not null tid %u\n",
1241 cxgb_get_4tuple(req
, cdev
->lldi
.adapter_type
, &iptype
, local_ip
,
1242 peer_ip
, &local_port
, &peer_port
);
1244 /* Find output route */
1246 pr_debug("%s parent sock %p tid %u laddr %pI4 raddr %pI4 "
1247 "lport %d rport %d peer_mss %d\n"
1248 , __func__
, cnp
, tid
,
1249 local_ip
, peer_ip
, ntohs(local_port
),
1250 ntohs(peer_port
), peer_mss
);
1251 dst
= cxgb_find_route(&cdev
->lldi
, cxgbit_get_real_dev
,
1252 *(__be32
*)local_ip
,
1254 local_port
, peer_port
,
1255 PASS_OPEN_TOS_G(ntohl(req
->tos_stid
)));
1257 pr_debug("%s parent sock %p tid %u laddr %pI6 raddr %pI6 "
1258 "lport %d rport %d peer_mss %d\n"
1259 , __func__
, cnp
, tid
,
1260 local_ip
, peer_ip
, ntohs(local_port
),
1261 ntohs(peer_port
), peer_mss
);
1262 dst
= cxgb_find_route6(&cdev
->lldi
, cxgbit_get_real_dev
,
1264 local_port
, peer_port
,
1265 PASS_OPEN_TOS_G(ntohl(req
->tos_stid
)),
1266 ((struct sockaddr_in6
*)
1267 &cnp
->com
.local_addr
)->sin6_scope_id
);
1270 pr_err("%s - failed to find dst entry!\n",
1275 csk
= kzalloc(sizeof(*csk
), GFP_ATOMIC
);
1281 ret
= cxgbit_offload_init(csk
, iptype
, peer_ip
, ntohs(local_port
),
1284 pr_err("%s - failed to allocate l2t entry!\n",
1291 kref_init(&csk
->kref
);
1292 init_completion(&csk
->com
.wr_wait
.completion
);
1294 INIT_LIST_HEAD(&csk
->accept_node
);
1296 hdrs
= (iptype
== 4 ? sizeof(struct iphdr
) : sizeof(struct ipv6hdr
)) +
1297 sizeof(struct tcphdr
) + (req
->tcpopt
.tstamp
? 12 : 0);
1298 if (peer_mss
&& csk
->mtu
> (peer_mss
+ hdrs
))
1299 csk
->mtu
= peer_mss
+ hdrs
;
1301 csk
->com
.state
= CSK_STATE_CONNECTING
;
1302 csk
->com
.cdev
= cdev
;
1304 csk
->tos
= PASS_OPEN_TOS_G(ntohl(req
->tos_stid
));
1307 csk
->wr_cred
= cdev
->lldi
.wr_cred
-
1308 DIV_ROUND_UP(sizeof(struct cpl_abort_req
), 16);
1309 csk
->wr_max_cred
= csk
->wr_cred
;
1310 csk
->wr_una_cred
= 0;
1313 struct sockaddr_in
*sin
= (struct sockaddr_in
*)
1314 &csk
->com
.local_addr
;
1315 sin
->sin_family
= AF_INET
;
1316 sin
->sin_port
= local_port
;
1317 sin
->sin_addr
.s_addr
= *(__be32
*)local_ip
;
1319 sin
= (struct sockaddr_in
*)&csk
->com
.remote_addr
;
1320 sin
->sin_family
= AF_INET
;
1321 sin
->sin_port
= peer_port
;
1322 sin
->sin_addr
.s_addr
= *(__be32
*)peer_ip
;
1324 struct sockaddr_in6
*sin6
= (struct sockaddr_in6
*)
1325 &csk
->com
.local_addr
;
1327 sin6
->sin6_family
= PF_INET6
;
1328 sin6
->sin6_port
= local_port
;
1329 memcpy(sin6
->sin6_addr
.s6_addr
, local_ip
, 16);
1330 cxgb4_clip_get(cdev
->lldi
.ports
[0],
1331 (const u32
*)&sin6
->sin6_addr
.s6_addr
,
1334 sin6
= (struct sockaddr_in6
*)&csk
->com
.remote_addr
;
1335 sin6
->sin6_family
= PF_INET6
;
1336 sin6
->sin6_port
= peer_port
;
1337 memcpy(sin6
->sin6_addr
.s6_addr
, peer_ip
, 16);
1340 skb_queue_head_init(&csk
->rxq
);
1341 skb_queue_head_init(&csk
->txq
);
1342 skb_queue_head_init(&csk
->ppodq
);
1343 skb_queue_head_init(&csk
->backlogq
);
1344 skb_queue_head_init(&csk
->skbq
);
1345 cxgbit_sock_reset_wr_list(csk
);
1346 spin_lock_init(&csk
->lock
);
1347 init_waitqueue_head(&csk
->waitq
);
1348 init_waitqueue_head(&csk
->ack_waitq
);
1349 csk
->lock_owner
= false;
1351 if (cxgbit_alloc_csk_skb(csk
)) {
1357 cxgbit_get_cdev(cdev
);
1359 spin_lock(&cdev
->cskq
.lock
);
1360 list_add_tail(&csk
->list
, &cdev
->cskq
.list
);
1361 spin_unlock(&cdev
->cskq
.lock
);
1363 cxgb4_insert_tid(t
, csk
, tid
);
1364 cxgbit_pass_accept_rpl(csk
, req
);
1368 cxgbit_release_tid(cdev
, tid
);
1374 cxgbit_tx_flowc_wr_credits(struct cxgbit_sock
*csk
, u32
*nparamsp
,
1377 u32 nparams
, flowclen16
, flowclen
;
1379 nparams
= FLOWC_WR_NPARAMS_MIN
;
1381 if (csk
->snd_wscale
)
1384 #ifdef CONFIG_CHELSIO_T4_DCB
1387 flowclen
= offsetof(struct fw_flowc_wr
, mnemval
[nparams
]);
1388 flowclen16
= DIV_ROUND_UP(flowclen
, 16);
1389 flowclen
= flowclen16
* 16;
1391 * Return the number of 16-byte credits used by the flowc request.
1392 * Pass back the nparams and actual flowc length if requested.
1395 *nparamsp
= nparams
;
1397 *flowclenp
= flowclen
;
1401 u32
cxgbit_send_tx_flowc_wr(struct cxgbit_sock
*csk
)
1403 struct cxgbit_device
*cdev
= csk
->com
.cdev
;
1404 struct fw_flowc_wr
*flowc
;
1405 u32 nparams
, flowclen16
, flowclen
;
1406 struct sk_buff
*skb
;
1409 #ifdef CONFIG_CHELSIO_T4_DCB
1410 u16 vlan
= ((struct l2t_entry
*)csk
->l2t
)->vlan
;
1413 flowclen16
= cxgbit_tx_flowc_wr_credits(csk
, &nparams
, &flowclen
);
1415 skb
= __skb_dequeue(&csk
->skbq
);
1416 flowc
= (struct fw_flowc_wr
*)__skb_put(skb
, flowclen
);
1417 memset(flowc
, 0, flowclen
);
1419 flowc
->op_to_nparams
= cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR
) |
1420 FW_FLOWC_WR_NPARAMS_V(nparams
));
1421 flowc
->flowid_len16
= cpu_to_be32(FW_WR_LEN16_V(flowclen16
) |
1422 FW_WR_FLOWID_V(csk
->tid
));
1423 flowc
->mnemval
[0].mnemonic
= FW_FLOWC_MNEM_PFNVFN
;
1424 flowc
->mnemval
[0].val
= cpu_to_be32(FW_PFVF_CMD_PFN_V
1425 (csk
->com
.cdev
->lldi
.pf
));
1426 flowc
->mnemval
[1].mnemonic
= FW_FLOWC_MNEM_CH
;
1427 flowc
->mnemval
[1].val
= cpu_to_be32(csk
->tx_chan
);
1428 flowc
->mnemval
[2].mnemonic
= FW_FLOWC_MNEM_PORT
;
1429 flowc
->mnemval
[2].val
= cpu_to_be32(csk
->tx_chan
);
1430 flowc
->mnemval
[3].mnemonic
= FW_FLOWC_MNEM_IQID
;
1431 flowc
->mnemval
[3].val
= cpu_to_be32(csk
->rss_qid
);
1432 flowc
->mnemval
[4].mnemonic
= FW_FLOWC_MNEM_SNDNXT
;
1433 flowc
->mnemval
[4].val
= cpu_to_be32(csk
->snd_nxt
);
1434 flowc
->mnemval
[5].mnemonic
= FW_FLOWC_MNEM_RCVNXT
;
1435 flowc
->mnemval
[5].val
= cpu_to_be32(csk
->rcv_nxt
);
1436 flowc
->mnemval
[6].mnemonic
= FW_FLOWC_MNEM_SNDBUF
;
1437 flowc
->mnemval
[6].val
= cpu_to_be32(csk
->snd_win
);
1438 flowc
->mnemval
[7].mnemonic
= FW_FLOWC_MNEM_MSS
;
1439 flowc
->mnemval
[7].val
= cpu_to_be32(csk
->emss
);
1441 flowc
->mnemval
[8].mnemonic
= FW_FLOWC_MNEM_TXDATAPLEN_MAX
;
1442 if (test_bit(CDEV_ISO_ENABLE
, &cdev
->flags
))
1443 flowc
->mnemval
[8].val
= cpu_to_be32(CXGBIT_MAX_ISO_PAYLOAD
);
1445 flowc
->mnemval
[8].val
= cpu_to_be32(16384);
1449 if (csk
->snd_wscale
) {
1450 flowc
->mnemval
[index
].mnemonic
= FW_FLOWC_MNEM_RCV_SCALE
;
1451 flowc
->mnemval
[index
].val
= cpu_to_be32(csk
->snd_wscale
);
1455 #ifdef CONFIG_CHELSIO_T4_DCB
1456 flowc
->mnemval
[index
].mnemonic
= FW_FLOWC_MNEM_DCBPRIO
;
1457 if (vlan
== VLAN_NONE
) {
1458 pr_warn("csk %u without VLAN Tag on DCB Link\n", csk
->tid
);
1459 flowc
->mnemval
[index
].val
= cpu_to_be32(0);
1461 flowc
->mnemval
[index
].val
= cpu_to_be32(
1462 (vlan
& VLAN_PRIO_MASK
) >> VLAN_PRIO_SHIFT
);
1465 pr_debug("%s: csk %p; tx_chan = %u; rss_qid = %u; snd_seq = %u;"
1466 " rcv_seq = %u; snd_win = %u; emss = %u\n",
1467 __func__
, csk
, csk
->tx_chan
, csk
->rss_qid
, csk
->snd_nxt
,
1468 csk
->rcv_nxt
, csk
->snd_win
, csk
->emss
);
1469 set_wr_txq(skb
, CPL_PRIORITY_DATA
, csk
->txq_idx
);
1470 cxgbit_ofld_send(csk
->com
.cdev
, skb
);
1474 int cxgbit_setup_conn_digest(struct cxgbit_sock
*csk
)
1476 struct sk_buff
*skb
;
1477 struct cpl_set_tcb_field
*req
;
1478 u8 hcrc
= csk
->submode
& CXGBIT_SUBMODE_HCRC
;
1479 u8 dcrc
= csk
->submode
& CXGBIT_SUBMODE_DCRC
;
1480 unsigned int len
= roundup(sizeof(*req
), 16);
1483 skb
= alloc_skb(len
, GFP_KERNEL
);
1487 /* set up ulp submode */
1488 req
= (struct cpl_set_tcb_field
*)__skb_put(skb
, len
);
1489 memset(req
, 0, len
);
1491 INIT_TP_WR(req
, csk
->tid
);
1492 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD
, csk
->tid
));
1493 req
->reply_ctrl
= htons(NO_REPLY_V(0) | QUEUENO_V(csk
->rss_qid
));
1494 req
->word_cookie
= htons(0);
1495 req
->mask
= cpu_to_be64(0x3 << 4);
1496 req
->val
= cpu_to_be64(((hcrc
? ULP_CRC_HEADER
: 0) |
1497 (dcrc
? ULP_CRC_DATA
: 0)) << 4);
1498 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, csk
->ctrlq_idx
);
1500 cxgbit_get_csk(csk
);
1501 cxgbit_init_wr_wait(&csk
->com
.wr_wait
);
1503 cxgbit_ofld_send(csk
->com
.cdev
, skb
);
1505 ret
= cxgbit_wait_for_reply(csk
->com
.cdev
,
1507 csk
->tid
, 5, __func__
);
1514 int cxgbit_setup_conn_pgidx(struct cxgbit_sock
*csk
, u32 pg_idx
)
1516 struct sk_buff
*skb
;
1517 struct cpl_set_tcb_field
*req
;
1518 unsigned int len
= roundup(sizeof(*req
), 16);
1521 skb
= alloc_skb(len
, GFP_KERNEL
);
1525 req
= (struct cpl_set_tcb_field
*)__skb_put(skb
, len
);
1526 memset(req
, 0, len
);
1528 INIT_TP_WR(req
, csk
->tid
);
1529 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD
, csk
->tid
));
1530 req
->reply_ctrl
= htons(NO_REPLY_V(0) | QUEUENO_V(csk
->rss_qid
));
1531 req
->word_cookie
= htons(0);
1532 req
->mask
= cpu_to_be64(0x3 << 8);
1533 req
->val
= cpu_to_be64(pg_idx
<< 8);
1534 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, csk
->ctrlq_idx
);
1536 cxgbit_get_csk(csk
);
1537 cxgbit_init_wr_wait(&csk
->com
.wr_wait
);
1539 cxgbit_ofld_send(csk
->com
.cdev
, skb
);
1541 ret
= cxgbit_wait_for_reply(csk
->com
.cdev
,
1543 csk
->tid
, 5, __func__
);
1551 cxgbit_pass_open_rpl(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1553 struct cpl_pass_open_rpl
*rpl
= cplhdr(skb
);
1554 struct tid_info
*t
= cdev
->lldi
.tids
;
1555 unsigned int stid
= GET_TID(rpl
);
1556 struct cxgbit_np
*cnp
= lookup_stid(t
, stid
);
1558 pr_debug("%s: cnp = %p; stid = %u; status = %d\n",
1559 __func__
, cnp
, stid
, rpl
->status
);
1562 pr_info("%s stid %d lookup failure\n", __func__
, stid
);
1566 cxgbit_wake_up(&cnp
->com
.wr_wait
, __func__
, rpl
->status
);
1567 cxgbit_put_cnp(cnp
);
1571 cxgbit_close_listsrv_rpl(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1573 struct cpl_close_listsvr_rpl
*rpl
= cplhdr(skb
);
1574 struct tid_info
*t
= cdev
->lldi
.tids
;
1575 unsigned int stid
= GET_TID(rpl
);
1576 struct cxgbit_np
*cnp
= lookup_stid(t
, stid
);
1578 pr_debug("%s: cnp = %p; stid = %u; status = %d\n",
1579 __func__
, cnp
, stid
, rpl
->status
);
1582 pr_info("%s stid %d lookup failure\n", __func__
, stid
);
1586 cxgbit_wake_up(&cnp
->com
.wr_wait
, __func__
, rpl
->status
);
1587 cxgbit_put_cnp(cnp
);
1591 cxgbit_pass_establish(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1593 struct cpl_pass_establish
*req
= cplhdr(skb
);
1594 struct tid_info
*t
= cdev
->lldi
.tids
;
1595 unsigned int tid
= GET_TID(req
);
1596 struct cxgbit_sock
*csk
;
1597 struct cxgbit_np
*cnp
;
1598 u16 tcp_opt
= be16_to_cpu(req
->tcp_opt
);
1599 u32 snd_isn
= be32_to_cpu(req
->snd_isn
);
1600 u32 rcv_isn
= be32_to_cpu(req
->rcv_isn
);
1602 csk
= lookup_tid(t
, tid
);
1603 if (unlikely(!csk
)) {
1604 pr_err("can't find connection for tid %u.\n", tid
);
1609 pr_debug("%s: csk %p; tid %u; cnp %p\n",
1610 __func__
, csk
, tid
, cnp
);
1612 csk
->write_seq
= snd_isn
;
1613 csk
->snd_una
= snd_isn
;
1614 csk
->snd_nxt
= snd_isn
;
1616 csk
->rcv_nxt
= rcv_isn
;
1618 if (csk
->rcv_win
> (RCV_BUFSIZ_M
<< 10))
1619 csk
->rx_credits
= (csk
->rcv_win
- (RCV_BUFSIZ_M
<< 10));
1621 csk
->snd_wscale
= TCPOPT_SND_WSCALE_G(tcp_opt
);
1622 cxgbit_set_emss(csk
, tcp_opt
);
1623 dst_confirm(csk
->dst
);
1624 csk
->com
.state
= CSK_STATE_ESTABLISHED
;
1625 spin_lock_bh(&cnp
->np_accept_lock
);
1626 list_add_tail(&csk
->accept_node
, &cnp
->np_accept_list
);
1627 spin_unlock_bh(&cnp
->np_accept_lock
);
1628 complete(&cnp
->accept_comp
);
1633 static void cxgbit_queue_rx_skb(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1635 cxgbit_skcb_flags(skb
) = 0;
1636 spin_lock_bh(&csk
->rxq
.lock
);
1637 __skb_queue_tail(&csk
->rxq
, skb
);
1638 spin_unlock_bh(&csk
->rxq
.lock
);
1639 wake_up(&csk
->waitq
);
1642 static void cxgbit_peer_close(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1644 pr_debug("%s: csk %p; tid %u; state %d\n",
1645 __func__
, csk
, csk
->tid
, csk
->com
.state
);
1647 switch (csk
->com
.state
) {
1648 case CSK_STATE_ESTABLISHED
:
1649 csk
->com
.state
= CSK_STATE_CLOSING
;
1650 cxgbit_queue_rx_skb(csk
, skb
);
1652 case CSK_STATE_CLOSING
:
1653 /* simultaneous close */
1654 csk
->com
.state
= CSK_STATE_MORIBUND
;
1656 case CSK_STATE_MORIBUND
:
1657 csk
->com
.state
= CSK_STATE_DEAD
;
1658 cxgbit_put_csk(csk
);
1660 case CSK_STATE_ABORTING
:
1663 pr_info("%s: cpl_peer_close in bad state %d\n",
1664 __func__
, csk
->com
.state
);
1670 static void cxgbit_close_con_rpl(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1672 pr_debug("%s: csk %p; tid %u; state %d\n",
1673 __func__
, csk
, csk
->tid
, csk
->com
.state
);
1675 switch (csk
->com
.state
) {
1676 case CSK_STATE_CLOSING
:
1677 csk
->com
.state
= CSK_STATE_MORIBUND
;
1679 case CSK_STATE_MORIBUND
:
1680 csk
->com
.state
= CSK_STATE_DEAD
;
1681 cxgbit_put_csk(csk
);
1683 case CSK_STATE_ABORTING
:
1684 case CSK_STATE_DEAD
:
1687 pr_info("%s: cpl_close_con_rpl in bad state %d\n",
1688 __func__
, csk
->com
.state
);
1694 static void cxgbit_abort_req_rss(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1696 struct cpl_abort_req_rss
*hdr
= cplhdr(skb
);
1697 unsigned int tid
= GET_TID(hdr
);
1698 struct cpl_abort_rpl
*rpl
;
1699 struct sk_buff
*rpl_skb
;
1700 bool release
= false;
1701 bool wakeup_thread
= false;
1702 unsigned int len
= roundup(sizeof(*rpl
), 16);
1704 pr_debug("%s: csk %p; tid %u; state %d\n",
1705 __func__
, csk
, tid
, csk
->com
.state
);
1707 if (cxgbit_is_neg_adv(hdr
->status
)) {
1708 pr_err("%s: got neg advise %d on tid %u\n",
1709 __func__
, hdr
->status
, tid
);
1713 switch (csk
->com
.state
) {
1714 case CSK_STATE_CONNECTING
:
1715 case CSK_STATE_MORIBUND
:
1716 csk
->com
.state
= CSK_STATE_DEAD
;
1719 case CSK_STATE_ESTABLISHED
:
1720 csk
->com
.state
= CSK_STATE_DEAD
;
1721 wakeup_thread
= true;
1723 case CSK_STATE_CLOSING
:
1724 csk
->com
.state
= CSK_STATE_DEAD
;
1728 case CSK_STATE_ABORTING
:
1731 pr_info("%s: cpl_abort_req_rss in bad state %d\n",
1732 __func__
, csk
->com
.state
);
1733 csk
->com
.state
= CSK_STATE_DEAD
;
1736 __skb_queue_purge(&csk
->txq
);
1738 if (!test_and_set_bit(CSK_TX_DATA_SENT
, &csk
->com
.flags
))
1739 cxgbit_send_tx_flowc_wr(csk
);
1741 rpl_skb
= __skb_dequeue(&csk
->skbq
);
1742 set_wr_txq(skb
, CPL_PRIORITY_DATA
, csk
->txq_idx
);
1744 rpl
= (struct cpl_abort_rpl
*)__skb_put(rpl_skb
, len
);
1745 memset(rpl
, 0, len
);
1747 INIT_TP_WR(rpl
, csk
->tid
);
1748 OPCODE_TID(rpl
) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL
, tid
));
1749 rpl
->cmd
= CPL_ABORT_NO_RST
;
1750 cxgbit_ofld_send(csk
->com
.cdev
, rpl_skb
);
1752 if (wakeup_thread
) {
1753 cxgbit_queue_rx_skb(csk
, skb
);
1758 cxgbit_put_csk(csk
);
1763 static void cxgbit_abort_rpl_rss(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1765 pr_debug("%s: csk %p; tid %u; state %d\n",
1766 __func__
, csk
, csk
->tid
, csk
->com
.state
);
1768 switch (csk
->com
.state
) {
1769 case CSK_STATE_ABORTING
:
1770 csk
->com
.state
= CSK_STATE_DEAD
;
1771 cxgbit_put_csk(csk
);
1774 pr_info("%s: cpl_abort_rpl_rss in state %d\n",
1775 __func__
, csk
->com
.state
);
1781 static bool cxgbit_credit_err(const struct cxgbit_sock
*csk
)
1783 const struct sk_buff
*skb
= csk
->wr_pending_head
;
1786 if (unlikely(csk
->wr_cred
> csk
->wr_max_cred
)) {
1787 pr_err("csk 0x%p, tid %u, credit %u > %u\n",
1788 csk
, csk
->tid
, csk
->wr_cred
, csk
->wr_max_cred
);
1793 credit
+= skb
->csum
;
1794 skb
= cxgbit_skcb_tx_wr_next(skb
);
1797 if (unlikely((csk
->wr_cred
+ credit
) != csk
->wr_max_cred
)) {
1798 pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n",
1799 csk
, csk
->tid
, csk
->wr_cred
,
1800 credit
, csk
->wr_max_cred
);
1808 static void cxgbit_fw4_ack(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1810 struct cpl_fw4_ack
*rpl
= (struct cpl_fw4_ack
*)cplhdr(skb
);
1811 u32 credits
= rpl
->credits
;
1812 u32 snd_una
= ntohl(rpl
->snd_una
);
1814 csk
->wr_cred
+= credits
;
1815 if (csk
->wr_una_cred
> (csk
->wr_max_cred
- csk
->wr_cred
))
1816 csk
->wr_una_cred
= csk
->wr_max_cred
- csk
->wr_cred
;
1819 struct sk_buff
*p
= cxgbit_sock_peek_wr(csk
);
1822 pr_err("csk 0x%p,%u, cr %u,%u+%u, empty.\n",
1823 csk
, csk
->tid
, credits
,
1824 csk
->wr_cred
, csk
->wr_una_cred
);
1828 if (unlikely(credits
< p
->csum
)) {
1829 pr_warn("csk 0x%p,%u, cr %u,%u+%u, < %u.\n",
1831 credits
, csk
->wr_cred
, csk
->wr_una_cred
,
1837 cxgbit_sock_dequeue_wr(csk
);
1842 if (unlikely(cxgbit_credit_err(csk
))) {
1843 cxgbit_queue_rx_skb(csk
, skb
);
1847 if (rpl
->seq_vld
& CPL_FW4_ACK_FLAGS_SEQVAL
) {
1848 if (unlikely(before(snd_una
, csk
->snd_una
))) {
1849 pr_warn("csk 0x%p,%u, snd_una %u/%u.",
1850 csk
, csk
->tid
, snd_una
,
1855 if (csk
->snd_una
!= snd_una
) {
1856 csk
->snd_una
= snd_una
;
1857 dst_confirm(csk
->dst
);
1858 wake_up(&csk
->ack_waitq
);
1862 if (skb_queue_len(&csk
->txq
))
1863 cxgbit_push_tx_frames(csk
);
1869 static void cxgbit_set_tcb_rpl(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1871 struct cxgbit_sock
*csk
;
1872 struct cpl_set_tcb_rpl
*rpl
= (struct cpl_set_tcb_rpl
*)skb
->data
;
1873 unsigned int tid
= GET_TID(rpl
);
1874 struct cxgb4_lld_info
*lldi
= &cdev
->lldi
;
1875 struct tid_info
*t
= lldi
->tids
;
1877 csk
= lookup_tid(t
, tid
);
1879 pr_err("can't find connection for tid %u.\n", tid
);
1881 cxgbit_wake_up(&csk
->com
.wr_wait
, __func__
, rpl
->status
);
1883 cxgbit_put_csk(csk
);
1886 static void cxgbit_rx_data(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1888 struct cxgbit_sock
*csk
;
1889 struct cpl_rx_data
*cpl
= cplhdr(skb
);
1890 unsigned int tid
= GET_TID(cpl
);
1891 struct cxgb4_lld_info
*lldi
= &cdev
->lldi
;
1892 struct tid_info
*t
= lldi
->tids
;
1894 csk
= lookup_tid(t
, tid
);
1895 if (unlikely(!csk
)) {
1896 pr_err("can't find conn. for tid %u.\n", tid
);
1900 cxgbit_queue_rx_skb(csk
, skb
);
1907 __cxgbit_process_rx_cpl(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1909 spin_lock(&csk
->lock
);
1910 if (csk
->lock_owner
) {
1911 __skb_queue_tail(&csk
->backlogq
, skb
);
1912 spin_unlock(&csk
->lock
);
1916 cxgbit_skcb_rx_backlog_fn(skb
)(csk
, skb
);
1917 spin_unlock(&csk
->lock
);
1920 static void cxgbit_process_rx_cpl(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1922 cxgbit_get_csk(csk
);
1923 __cxgbit_process_rx_cpl(csk
, skb
);
1924 cxgbit_put_csk(csk
);
1927 static void cxgbit_rx_cpl(struct cxgbit_device
*cdev
, struct sk_buff
*skb
)
1929 struct cxgbit_sock
*csk
;
1930 struct cpl_tx_data
*cpl
= cplhdr(skb
);
1931 struct cxgb4_lld_info
*lldi
= &cdev
->lldi
;
1932 struct tid_info
*t
= lldi
->tids
;
1933 unsigned int tid
= GET_TID(cpl
);
1934 u8 opcode
= cxgbit_skcb_rx_opcode(skb
);
1939 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_fw4_ack
;
1942 case CPL_PEER_CLOSE
:
1943 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_peer_close
;
1945 case CPL_CLOSE_CON_RPL
:
1946 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_close_con_rpl
;
1948 case CPL_ABORT_REQ_RSS
:
1949 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_abort_req_rss
;
1951 case CPL_ABORT_RPL_RSS
:
1952 cxgbit_skcb_rx_backlog_fn(skb
) = cxgbit_abort_rpl_rss
;
1958 csk
= lookup_tid(t
, tid
);
1959 if (unlikely(!csk
)) {
1960 pr_err("can't find conn. for tid %u.\n", tid
);
1965 cxgbit_process_rx_cpl(csk
, skb
);
1967 __cxgbit_process_rx_cpl(csk
, skb
);
1974 cxgbit_cplhandler_func cxgbit_cplhandlers
[NUM_CPL_CMDS
] = {
1975 [CPL_PASS_OPEN_RPL
] = cxgbit_pass_open_rpl
,
1976 [CPL_CLOSE_LISTSRV_RPL
] = cxgbit_close_listsrv_rpl
,
1977 [CPL_PASS_ACCEPT_REQ
] = cxgbit_pass_accept_req
,
1978 [CPL_PASS_ESTABLISH
] = cxgbit_pass_establish
,
1979 [CPL_SET_TCB_RPL
] = cxgbit_set_tcb_rpl
,
1980 [CPL_RX_DATA
] = cxgbit_rx_data
,
1981 [CPL_FW4_ACK
] = cxgbit_rx_cpl
,
1982 [CPL_PEER_CLOSE
] = cxgbit_rx_cpl
,
1983 [CPL_CLOSE_CON_RPL
] = cxgbit_rx_cpl
,
1984 [CPL_ABORT_REQ_RSS
] = cxgbit_rx_cpl
,
1985 [CPL_ABORT_RPL_RSS
] = cxgbit_rx_cpl
,