2 * Copyright (c) 2006-2007 Chelsio, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/list.h>
34 #include <net/neighbour.h>
35 #include <linux/notifier.h>
36 #include <asm/atomic.h>
37 #include <linux/proc_fs.h>
38 #include <linux/if_vlan.h>
39 #include <net/netevent.h>
40 #include <linux/highmem.h>
41 #include <linux/vmalloc.h>
45 #include "cxgb3_ioctl.h"
46 #include "cxgb3_ctl_defs.h"
47 #include "cxgb3_defs.h"
49 #include "firmware_exports.h"
50 #include "cxgb3_offload.h"
52 static LIST_HEAD(client_list
);
53 static LIST_HEAD(ofld_dev_list
);
54 static DEFINE_MUTEX(cxgb3_db_lock
);
56 static DEFINE_RWLOCK(adapter_list_lock
);
57 static LIST_HEAD(adapter_list
);
59 static const unsigned int MAX_ATIDS
= 64 * 1024;
60 static const unsigned int ATID_BASE
= 0x10000;
62 static inline int offload_activated(struct t3cdev
*tdev
)
64 const struct adapter
*adapter
= tdev2adap(tdev
);
66 return (test_bit(OFFLOAD_DEVMAP_BIT
, &adapter
->open_device_map
));
70 * cxgb3_register_client - register an offload client
73 * Add the client to the client list,
74 * and call backs the client for each activated offload device
76 void cxgb3_register_client(struct cxgb3_client
*client
)
80 mutex_lock(&cxgb3_db_lock
);
81 list_add_tail(&client
->client_list
, &client_list
);
84 list_for_each_entry(tdev
, &ofld_dev_list
, ofld_dev_list
) {
85 if (offload_activated(tdev
))
89 mutex_unlock(&cxgb3_db_lock
);
92 EXPORT_SYMBOL(cxgb3_register_client
);
95 * cxgb3_unregister_client - unregister an offload client
98 * Remove the client to the client list,
99 * and call backs the client for each activated offload device.
101 void cxgb3_unregister_client(struct cxgb3_client
*client
)
105 mutex_lock(&cxgb3_db_lock
);
106 list_del(&client
->client_list
);
108 if (client
->remove
) {
109 list_for_each_entry(tdev
, &ofld_dev_list
, ofld_dev_list
) {
110 if (offload_activated(tdev
))
111 client
->remove(tdev
);
114 mutex_unlock(&cxgb3_db_lock
);
117 EXPORT_SYMBOL(cxgb3_unregister_client
);
120 * cxgb3_add_clients - activate registered clients for an offload device
121 * @tdev: the offload device
123 * Call backs all registered clients once a offload device is activated
125 void cxgb3_add_clients(struct t3cdev
*tdev
)
127 struct cxgb3_client
*client
;
129 mutex_lock(&cxgb3_db_lock
);
130 list_for_each_entry(client
, &client_list
, client_list
) {
134 mutex_unlock(&cxgb3_db_lock
);
138 * cxgb3_remove_clients - deactivates registered clients
139 * for an offload device
140 * @tdev: the offload device
142 * Call backs all registered clients once a offload device is deactivated
144 void cxgb3_remove_clients(struct t3cdev
*tdev
)
146 struct cxgb3_client
*client
;
148 mutex_lock(&cxgb3_db_lock
);
149 list_for_each_entry(client
, &client_list
, client_list
) {
151 client
->remove(tdev
);
153 mutex_unlock(&cxgb3_db_lock
);
156 static struct net_device
*get_iff_from_mac(struct adapter
*adapter
,
157 const unsigned char *mac
,
162 for_each_port(adapter
, i
) {
163 struct vlan_group
*grp
;
164 struct net_device
*dev
= adapter
->port
[i
];
165 const struct port_info
*p
= netdev_priv(dev
);
167 if (!memcmp(dev
->dev_addr
, mac
, ETH_ALEN
)) {
168 if (vlan
&& vlan
!= VLAN_VID_MASK
) {
172 dev
= vlan_group_get_device(grp
, vlan
);
182 static int cxgb_ulp_iscsi_ctl(struct adapter
*adapter
, unsigned int req
,
186 struct ulp_iscsi_info
*uiip
= data
;
189 case ULP_ISCSI_GET_PARAMS
:
190 uiip
->pdev
= adapter
->pdev
;
191 uiip
->llimit
= t3_read_reg(adapter
, A_ULPRX_ISCSI_LLIMIT
);
192 uiip
->ulimit
= t3_read_reg(adapter
, A_ULPRX_ISCSI_ULIMIT
);
193 uiip
->tagmask
= t3_read_reg(adapter
, A_ULPRX_ISCSI_TAGMASK
);
195 * On tx, the iscsi pdu has to be <= tx page size and has to
196 * fit into the Tx PM FIFO.
198 uiip
->max_txsz
= min(adapter
->params
.tp
.tx_pg_size
,
199 t3_read_reg(adapter
, A_PM1_TX_CFG
) >> 17);
200 /* on rx, the iscsi pdu has to be < rx page size and the
201 whole pdu + cpl headers has to fit into one sge buffer */
202 uiip
->max_rxsz
= min_t(unsigned int,
203 adapter
->params
.tp
.rx_pg_size
,
204 (adapter
->sge
.qs
[0].fl
[1].buf_size
-
205 sizeof(struct cpl_rx_data
) * 2 -
206 sizeof(struct cpl_rx_data_ddp
)));
208 case ULP_ISCSI_SET_PARAMS
:
209 t3_write_reg(adapter
, A_ULPRX_ISCSI_TAGMASK
, uiip
->tagmask
);
210 /* set MaxRxData and MaxCoalesceSize to 16224 */
211 t3_write_reg(adapter
, A_TP_PARA_REG2
, 0x3f603f60);
212 /* program the ddp page sizes */
215 unsigned int val
= 0;
216 for (i
= 0; i
< 4; i
++)
217 val
|= (uiip
->pgsz_factor
[i
] & 0xF) << (8 * i
);
219 t3_write_reg(adapter
, A_ULPRX_ISCSI_PSZ
, val
);
228 /* Response queue used for RDMA events. */
229 #define ASYNC_NOTIF_RSPQ 0
231 static int cxgb_rdma_ctl(struct adapter
*adapter
, unsigned int req
, void *data
)
236 case RDMA_GET_PARAMS
: {
237 struct rdma_info
*rdma
= data
;
238 struct pci_dev
*pdev
= adapter
->pdev
;
240 rdma
->udbell_physbase
= pci_resource_start(pdev
, 2);
241 rdma
->udbell_len
= pci_resource_len(pdev
, 2);
243 t3_read_reg(adapter
, A_ULPTX_TPT_LLIMIT
);
244 rdma
->tpt_top
= t3_read_reg(adapter
, A_ULPTX_TPT_ULIMIT
);
246 t3_read_reg(adapter
, A_ULPTX_PBL_LLIMIT
);
247 rdma
->pbl_top
= t3_read_reg(adapter
, A_ULPTX_PBL_ULIMIT
);
248 rdma
->rqt_base
= t3_read_reg(adapter
, A_ULPRX_RQ_LLIMIT
);
249 rdma
->rqt_top
= t3_read_reg(adapter
, A_ULPRX_RQ_ULIMIT
);
250 rdma
->kdb_addr
= adapter
->regs
+ A_SG_KDOORBELL
;
256 struct rdma_cq_op
*rdma
= data
;
258 /* may be called in any context */
259 spin_lock_irqsave(&adapter
->sge
.reg_lock
, flags
);
260 ret
= t3_sge_cqcntxt_op(adapter
, rdma
->id
, rdma
->op
,
262 spin_unlock_irqrestore(&adapter
->sge
.reg_lock
, flags
);
266 struct ch_mem_range
*t
= data
;
269 if ((t
->addr
& 7) || (t
->len
& 7))
271 if (t
->mem_id
== MEM_CM
)
273 else if (t
->mem_id
== MEM_PMRX
)
274 mem
= &adapter
->pmrx
;
275 else if (t
->mem_id
== MEM_PMTX
)
276 mem
= &adapter
->pmtx
;
281 t3_mc7_bd_read(mem
, t
->addr
/ 8, t
->len
/ 8,
288 struct rdma_cq_setup
*rdma
= data
;
290 spin_lock_irq(&adapter
->sge
.reg_lock
);
292 t3_sge_init_cqcntxt(adapter
, rdma
->id
,
293 rdma
->base_addr
, rdma
->size
,
295 rdma
->ovfl_mode
, rdma
->credits
,
297 spin_unlock_irq(&adapter
->sge
.reg_lock
);
300 case RDMA_CQ_DISABLE
:
301 spin_lock_irq(&adapter
->sge
.reg_lock
);
302 ret
= t3_sge_disable_cqcntxt(adapter
, *(unsigned int *)data
);
303 spin_unlock_irq(&adapter
->sge
.reg_lock
);
305 case RDMA_CTRL_QP_SETUP
:{
306 struct rdma_ctrlqp_setup
*rdma
= data
;
308 spin_lock_irq(&adapter
->sge
.reg_lock
);
309 ret
= t3_sge_init_ecntxt(adapter
, FW_RI_SGEEC_START
, 0,
312 rdma
->base_addr
, rdma
->size
,
313 FW_RI_TID_START
, 1, 0);
314 spin_unlock_irq(&adapter
->sge
.reg_lock
);
318 spin_lock(&adapter
->stats_lock
);
319 t3_tp_get_mib_stats(adapter
, (struct tp_mib_stats
*)data
);
320 spin_unlock(&adapter
->stats_lock
);
329 static int cxgb_offload_ctl(struct t3cdev
*tdev
, unsigned int req
, void *data
)
331 struct adapter
*adapter
= tdev2adap(tdev
);
332 struct tid_range
*tid
;
334 struct iff_mac
*iffmacp
;
335 struct ddp_params
*ddpp
;
336 struct adap_ports
*ports
;
337 struct ofld_page_info
*rx_page_info
;
338 struct tp_params
*tp
= &adapter
->params
.tp
;
342 case GET_MAX_OUTSTANDING_WR
:
343 *(unsigned int *)data
= FW_WR_NUM
;
346 *(unsigned int *)data
= WR_FLITS
;
348 case GET_TX_MAX_CHUNK
:
349 *(unsigned int *)data
= 1 << 20; /* 1MB */
353 tid
->num
= t3_mc5_size(&adapter
->mc5
) -
354 adapter
->params
.mc5
.nroutes
-
355 adapter
->params
.mc5
.nfilters
- adapter
->params
.mc5
.nservers
;
360 tid
->num
= adapter
->params
.mc5
.nservers
;
361 tid
->base
= t3_mc5_size(&adapter
->mc5
) - tid
->num
-
362 adapter
->params
.mc5
.nfilters
- adapter
->params
.mc5
.nroutes
;
364 case GET_L2T_CAPACITY
:
365 *(unsigned int *)data
= 2048;
370 mtup
->mtus
= adapter
->params
.mtus
;
372 case GET_IFF_FROM_MAC
:
374 iffmacp
->dev
= get_iff_from_mac(adapter
, iffmacp
->mac_addr
,
380 ddpp
->llimit
= t3_read_reg(adapter
, A_ULPRX_TDDP_LLIMIT
);
381 ddpp
->ulimit
= t3_read_reg(adapter
, A_ULPRX_TDDP_ULIMIT
);
382 ddpp
->tag_mask
= t3_read_reg(adapter
, A_ULPRX_TDDP_TAGMASK
);
386 ports
->nports
= adapter
->params
.nports
;
387 for_each_port(adapter
, i
)
388 ports
->lldevs
[i
] = adapter
->port
[i
];
390 case ULP_ISCSI_GET_PARAMS
:
391 case ULP_ISCSI_SET_PARAMS
:
392 if (!offload_running(adapter
))
394 return cxgb_ulp_iscsi_ctl(adapter
, req
, data
);
395 case RDMA_GET_PARAMS
:
398 case RDMA_CQ_DISABLE
:
399 case RDMA_CTRL_QP_SETUP
:
402 if (!offload_running(adapter
))
404 return cxgb_rdma_ctl(adapter
, req
, data
);
405 case GET_RX_PAGE_INFO
:
407 rx_page_info
->page_size
= tp
->rx_pg_size
;
408 rx_page_info
->num
= tp
->rx_num_pgs
;
417 * Dummy handler for Rx offload packets in case we get an offload packet before
418 * proper processing is setup. This complains and drops the packet as it isn't
419 * normal to get offload packets at this stage.
421 static int rx_offload_blackhole(struct t3cdev
*dev
, struct sk_buff
**skbs
,
425 dev_kfree_skb_any(skbs
[n
]);
429 static void dummy_neigh_update(struct t3cdev
*dev
, struct neighbour
*neigh
)
433 void cxgb3_set_dummy_ops(struct t3cdev
*dev
)
435 dev
->recv
= rx_offload_blackhole
;
436 dev
->neigh_update
= dummy_neigh_update
;
440 * Free an active-open TID.
442 void *cxgb3_free_atid(struct t3cdev
*tdev
, int atid
)
444 struct tid_info
*t
= &(T3C_DATA(tdev
))->tid_maps
;
445 union active_open_entry
*p
= atid2entry(t
, atid
);
446 void *ctx
= p
->t3c_tid
.ctx
;
448 spin_lock_bh(&t
->atid_lock
);
452 spin_unlock_bh(&t
->atid_lock
);
457 EXPORT_SYMBOL(cxgb3_free_atid
);
460 * Free a server TID and return it to the free pool.
462 void cxgb3_free_stid(struct t3cdev
*tdev
, int stid
)
464 struct tid_info
*t
= &(T3C_DATA(tdev
))->tid_maps
;
465 union listen_entry
*p
= stid2entry(t
, stid
);
467 spin_lock_bh(&t
->stid_lock
);
471 spin_unlock_bh(&t
->stid_lock
);
474 EXPORT_SYMBOL(cxgb3_free_stid
);
476 void cxgb3_insert_tid(struct t3cdev
*tdev
, struct cxgb3_client
*client
,
477 void *ctx
, unsigned int tid
)
479 struct tid_info
*t
= &(T3C_DATA(tdev
))->tid_maps
;
481 t
->tid_tab
[tid
].client
= client
;
482 t
->tid_tab
[tid
].ctx
= ctx
;
483 atomic_inc(&t
->tids_in_use
);
486 EXPORT_SYMBOL(cxgb3_insert_tid
);
489 * Populate a TID_RELEASE WR. The skb must be already propely sized.
491 static inline void mk_tid_release(struct sk_buff
*skb
, unsigned int tid
)
493 struct cpl_tid_release
*req
;
495 skb
->priority
= CPL_PRIORITY_SETUP
;
496 req
= (struct cpl_tid_release
*)__skb_put(skb
, sizeof(*req
));
497 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_FORWARD
));
498 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE
, tid
));
501 static void t3_process_tid_release_list(struct work_struct
*work
)
503 struct t3c_data
*td
= container_of(work
, struct t3c_data
,
506 struct t3cdev
*tdev
= td
->dev
;
509 spin_lock_bh(&td
->tid_release_lock
);
510 while (td
->tid_release_list
) {
511 struct t3c_tid_entry
*p
= td
->tid_release_list
;
513 td
->tid_release_list
= (struct t3c_tid_entry
*)p
->ctx
;
514 spin_unlock_bh(&td
->tid_release_lock
);
516 skb
= alloc_skb(sizeof(struct cpl_tid_release
),
517 GFP_KERNEL
| __GFP_NOFAIL
);
518 mk_tid_release(skb
, p
- td
->tid_maps
.tid_tab
);
519 cxgb3_ofld_send(tdev
, skb
);
521 spin_lock_bh(&td
->tid_release_lock
);
523 spin_unlock_bh(&td
->tid_release_lock
);
526 /* use ctx as a next pointer in the tid release list */
527 void cxgb3_queue_tid_release(struct t3cdev
*tdev
, unsigned int tid
)
529 struct t3c_data
*td
= T3C_DATA(tdev
);
530 struct t3c_tid_entry
*p
= &td
->tid_maps
.tid_tab
[tid
];
532 spin_lock_bh(&td
->tid_release_lock
);
533 p
->ctx
= (void *)td
->tid_release_list
;
535 td
->tid_release_list
= p
;
537 schedule_work(&td
->tid_release_task
);
538 spin_unlock_bh(&td
->tid_release_lock
);
541 EXPORT_SYMBOL(cxgb3_queue_tid_release
);
544 * Remove a tid from the TID table. A client may defer processing its last
545 * CPL message if it is locked at the time it arrives, and while the message
546 * sits in the client's backlog the TID may be reused for another connection.
547 * To handle this we atomically switch the TID association if it still points
548 * to the original client context.
550 void cxgb3_remove_tid(struct t3cdev
*tdev
, void *ctx
, unsigned int tid
)
552 struct tid_info
*t
= &(T3C_DATA(tdev
))->tid_maps
;
554 BUG_ON(tid
>= t
->ntids
);
555 if (tdev
->type
== T3A
)
556 (void)cmpxchg(&t
->tid_tab
[tid
].ctx
, ctx
, NULL
);
560 skb
= alloc_skb(sizeof(struct cpl_tid_release
), GFP_ATOMIC
);
562 mk_tid_release(skb
, tid
);
563 cxgb3_ofld_send(tdev
, skb
);
564 t
->tid_tab
[tid
].ctx
= NULL
;
566 cxgb3_queue_tid_release(tdev
, tid
);
568 atomic_dec(&t
->tids_in_use
);
571 EXPORT_SYMBOL(cxgb3_remove_tid
);
573 int cxgb3_alloc_atid(struct t3cdev
*tdev
, struct cxgb3_client
*client
,
577 struct tid_info
*t
= &(T3C_DATA(tdev
))->tid_maps
;
579 spin_lock_bh(&t
->atid_lock
);
581 t
->atids_in_use
+ atomic_read(&t
->tids_in_use
) + MC5_MIN_TIDS
<=
583 union active_open_entry
*p
= t
->afree
;
585 atid
= (p
- t
->atid_tab
) + t
->atid_base
;
587 p
->t3c_tid
.ctx
= ctx
;
588 p
->t3c_tid
.client
= client
;
591 spin_unlock_bh(&t
->atid_lock
);
595 EXPORT_SYMBOL(cxgb3_alloc_atid
);
597 int cxgb3_alloc_stid(struct t3cdev
*tdev
, struct cxgb3_client
*client
,
601 struct tid_info
*t
= &(T3C_DATA(tdev
))->tid_maps
;
603 spin_lock_bh(&t
->stid_lock
);
605 union listen_entry
*p
= t
->sfree
;
607 stid
= (p
- t
->stid_tab
) + t
->stid_base
;
609 p
->t3c_tid
.ctx
= ctx
;
610 p
->t3c_tid
.client
= client
;
613 spin_unlock_bh(&t
->stid_lock
);
617 EXPORT_SYMBOL(cxgb3_alloc_stid
);
619 /* Get the t3cdev associated with a net_device */
620 struct t3cdev
*dev2t3cdev(struct net_device
*dev
)
622 const struct port_info
*pi
= netdev_priv(dev
);
624 return (struct t3cdev
*)pi
->adapter
;
627 EXPORT_SYMBOL(dev2t3cdev
);
629 static int do_smt_write_rpl(struct t3cdev
*dev
, struct sk_buff
*skb
)
631 struct cpl_smt_write_rpl
*rpl
= cplhdr(skb
);
633 if (rpl
->status
!= CPL_ERR_NONE
)
635 "Unexpected SMT_WRITE_RPL status %u for entry %u\n",
636 rpl
->status
, GET_TID(rpl
));
638 return CPL_RET_BUF_DONE
;
641 static int do_l2t_write_rpl(struct t3cdev
*dev
, struct sk_buff
*skb
)
643 struct cpl_l2t_write_rpl
*rpl
= cplhdr(skb
);
645 if (rpl
->status
!= CPL_ERR_NONE
)
647 "Unexpected L2T_WRITE_RPL status %u for entry %u\n",
648 rpl
->status
, GET_TID(rpl
));
650 return CPL_RET_BUF_DONE
;
653 static int do_rte_write_rpl(struct t3cdev
*dev
, struct sk_buff
*skb
)
655 struct cpl_rte_write_rpl
*rpl
= cplhdr(skb
);
657 if (rpl
->status
!= CPL_ERR_NONE
)
659 "Unexpected RTE_WRITE_RPL status %u for entry %u\n",
660 rpl
->status
, GET_TID(rpl
));
662 return CPL_RET_BUF_DONE
;
665 static int do_act_open_rpl(struct t3cdev
*dev
, struct sk_buff
*skb
)
667 struct cpl_act_open_rpl
*rpl
= cplhdr(skb
);
668 unsigned int atid
= G_TID(ntohl(rpl
->atid
));
669 struct t3c_tid_entry
*t3c_tid
;
671 t3c_tid
= lookup_atid(&(T3C_DATA(dev
))->tid_maps
, atid
);
672 if (t3c_tid
&& t3c_tid
->ctx
&& t3c_tid
->client
&&
673 t3c_tid
->client
->handlers
&&
674 t3c_tid
->client
->handlers
[CPL_ACT_OPEN_RPL
]) {
675 return t3c_tid
->client
->handlers
[CPL_ACT_OPEN_RPL
] (dev
, skb
,
679 printk(KERN_ERR
"%s: received clientless CPL command 0x%x\n",
680 dev
->name
, CPL_ACT_OPEN_RPL
);
681 return CPL_RET_BUF_DONE
| CPL_RET_BAD_MSG
;
685 static int do_stid_rpl(struct t3cdev
*dev
, struct sk_buff
*skb
)
687 union opcode_tid
*p
= cplhdr(skb
);
688 unsigned int stid
= G_TID(ntohl(p
->opcode_tid
));
689 struct t3c_tid_entry
*t3c_tid
;
691 t3c_tid
= lookup_stid(&(T3C_DATA(dev
))->tid_maps
, stid
);
692 if (t3c_tid
&& t3c_tid
->ctx
&& t3c_tid
->client
->handlers
&&
693 t3c_tid
->client
->handlers
[p
->opcode
]) {
694 return t3c_tid
->client
->handlers
[p
->opcode
] (dev
, skb
,
697 printk(KERN_ERR
"%s: received clientless CPL command 0x%x\n",
698 dev
->name
, p
->opcode
);
699 return CPL_RET_BUF_DONE
| CPL_RET_BAD_MSG
;
703 static int do_hwtid_rpl(struct t3cdev
*dev
, struct sk_buff
*skb
)
705 union opcode_tid
*p
= cplhdr(skb
);
706 unsigned int hwtid
= G_TID(ntohl(p
->opcode_tid
));
707 struct t3c_tid_entry
*t3c_tid
;
709 t3c_tid
= lookup_tid(&(T3C_DATA(dev
))->tid_maps
, hwtid
);
710 if (t3c_tid
&& t3c_tid
->ctx
&& t3c_tid
->client
->handlers
&&
711 t3c_tid
->client
->handlers
[p
->opcode
]) {
712 return t3c_tid
->client
->handlers
[p
->opcode
]
713 (dev
, skb
, t3c_tid
->ctx
);
715 printk(KERN_ERR
"%s: received clientless CPL command 0x%x\n",
716 dev
->name
, p
->opcode
);
717 return CPL_RET_BUF_DONE
| CPL_RET_BAD_MSG
;
721 static int do_cr(struct t3cdev
*dev
, struct sk_buff
*skb
)
723 struct cpl_pass_accept_req
*req
= cplhdr(skb
);
724 unsigned int stid
= G_PASS_OPEN_TID(ntohl(req
->tos_tid
));
725 struct tid_info
*t
= &(T3C_DATA(dev
))->tid_maps
;
726 struct t3c_tid_entry
*t3c_tid
;
727 unsigned int tid
= GET_TID(req
);
729 if (unlikely(tid
>= t
->ntids
)) {
730 printk("%s: passive open TID %u too large\n",
732 t3_fatal_err(tdev2adap(dev
));
733 return CPL_RET_BUF_DONE
;
736 t3c_tid
= lookup_stid(t
, stid
);
737 if (t3c_tid
&& t3c_tid
->ctx
&& t3c_tid
->client
->handlers
&&
738 t3c_tid
->client
->handlers
[CPL_PASS_ACCEPT_REQ
]) {
739 return t3c_tid
->client
->handlers
[CPL_PASS_ACCEPT_REQ
]
740 (dev
, skb
, t3c_tid
->ctx
);
742 printk(KERN_ERR
"%s: received clientless CPL command 0x%x\n",
743 dev
->name
, CPL_PASS_ACCEPT_REQ
);
744 return CPL_RET_BUF_DONE
| CPL_RET_BAD_MSG
;
749 * Returns an sk_buff for a reply CPL message of size len. If the input
750 * sk_buff has no other users it is trimmed and reused, otherwise a new buffer
751 * is allocated. The input skb must be of size at least len. Note that this
752 * operation does not destroy the original skb data even if it decides to reuse
755 static struct sk_buff
*cxgb3_get_cpl_reply_skb(struct sk_buff
*skb
, size_t len
,
758 if (likely(!skb_cloned(skb
))) {
759 BUG_ON(skb
->len
< len
);
760 __skb_trim(skb
, len
);
763 skb
= alloc_skb(len
, gfp
);
770 static int do_abort_req_rss(struct t3cdev
*dev
, struct sk_buff
*skb
)
772 union opcode_tid
*p
= cplhdr(skb
);
773 unsigned int hwtid
= G_TID(ntohl(p
->opcode_tid
));
774 struct t3c_tid_entry
*t3c_tid
;
776 t3c_tid
= lookup_tid(&(T3C_DATA(dev
))->tid_maps
, hwtid
);
777 if (t3c_tid
&& t3c_tid
->ctx
&& t3c_tid
->client
->handlers
&&
778 t3c_tid
->client
->handlers
[p
->opcode
]) {
779 return t3c_tid
->client
->handlers
[p
->opcode
]
780 (dev
, skb
, t3c_tid
->ctx
);
782 struct cpl_abort_req_rss
*req
= cplhdr(skb
);
783 struct cpl_abort_rpl
*rpl
;
784 struct sk_buff
*reply_skb
;
785 unsigned int tid
= GET_TID(req
);
786 u8 cmd
= req
->status
;
788 if (req
->status
== CPL_ERR_RTX_NEG_ADVICE
||
789 req
->status
== CPL_ERR_PERSIST_NEG_ADVICE
)
792 reply_skb
= cxgb3_get_cpl_reply_skb(skb
,
798 printk("do_abort_req_rss: couldn't get skb!\n");
801 reply_skb
->priority
= CPL_PRIORITY_DATA
;
802 __skb_put(reply_skb
, sizeof(struct cpl_abort_rpl
));
803 rpl
= cplhdr(reply_skb
);
805 htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL
));
806 rpl
->wr
.wr_lo
= htonl(V_WR_TID(tid
));
807 OPCODE_TID(rpl
) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL
, tid
));
809 cxgb3_ofld_send(dev
, reply_skb
);
811 return CPL_RET_BUF_DONE
;
815 static int do_act_establish(struct t3cdev
*dev
, struct sk_buff
*skb
)
817 struct cpl_act_establish
*req
= cplhdr(skb
);
818 unsigned int atid
= G_PASS_OPEN_TID(ntohl(req
->tos_tid
));
819 struct tid_info
*t
= &(T3C_DATA(dev
))->tid_maps
;
820 struct t3c_tid_entry
*t3c_tid
;
821 unsigned int tid
= GET_TID(req
);
823 if (unlikely(tid
>= t
->ntids
)) {
824 printk("%s: active establish TID %u too large\n",
826 t3_fatal_err(tdev2adap(dev
));
827 return CPL_RET_BUF_DONE
;
830 t3c_tid
= lookup_atid(t
, atid
);
831 if (t3c_tid
&& t3c_tid
->ctx
&& t3c_tid
->client
->handlers
&&
832 t3c_tid
->client
->handlers
[CPL_ACT_ESTABLISH
]) {
833 return t3c_tid
->client
->handlers
[CPL_ACT_ESTABLISH
]
834 (dev
, skb
, t3c_tid
->ctx
);
836 printk(KERN_ERR
"%s: received clientless CPL command 0x%x\n",
837 dev
->name
, CPL_ACT_ESTABLISH
);
838 return CPL_RET_BUF_DONE
| CPL_RET_BAD_MSG
;
842 static int do_trace(struct t3cdev
*dev
, struct sk_buff
*skb
)
844 struct cpl_trace_pkt
*p
= cplhdr(skb
);
846 skb
->protocol
= htons(0xffff);
847 skb
->dev
= dev
->lldev
;
848 skb_pull(skb
, sizeof(*p
));
849 skb_reset_mac_header(skb
);
850 netif_receive_skb(skb
);
855 * That skb would better have come from process_responses() where we abuse
856 * ->priority and ->csum to carry our data. NB: if we get to per-arch
857 * ->csum, the things might get really interesting here.
860 static inline u32
get_hwtid(struct sk_buff
*skb
)
862 return ntohl((__force __be32
)skb
->priority
) >> 8 & 0xfffff;
865 static inline u32
get_opcode(struct sk_buff
*skb
)
867 return G_OPCODE(ntohl((__force __be32
)skb
->csum
));
870 static int do_term(struct t3cdev
*dev
, struct sk_buff
*skb
)
872 unsigned int hwtid
= get_hwtid(skb
);
873 unsigned int opcode
= get_opcode(skb
);
874 struct t3c_tid_entry
*t3c_tid
;
876 t3c_tid
= lookup_tid(&(T3C_DATA(dev
))->tid_maps
, hwtid
);
877 if (t3c_tid
&& t3c_tid
->ctx
&& t3c_tid
->client
->handlers
&&
878 t3c_tid
->client
->handlers
[opcode
]) {
879 return t3c_tid
->client
->handlers
[opcode
] (dev
, skb
,
882 printk(KERN_ERR
"%s: received clientless CPL command 0x%x\n",
884 return CPL_RET_BUF_DONE
| CPL_RET_BAD_MSG
;
888 static int nb_callback(struct notifier_block
*self
, unsigned long event
,
892 case (NETEVENT_NEIGH_UPDATE
):{
893 cxgb_neigh_update((struct neighbour
*)ctx
);
896 case (NETEVENT_PMTU_UPDATE
):
898 case (NETEVENT_REDIRECT
):{
899 struct netevent_redirect
*nr
= ctx
;
900 cxgb_redirect(nr
->old
, nr
->new);
901 cxgb_neigh_update(nr
->new->neighbour
);
910 static struct notifier_block nb
= {
911 .notifier_call
= nb_callback
915 * Process a received packet with an unknown/unexpected CPL opcode.
917 static int do_bad_cpl(struct t3cdev
*dev
, struct sk_buff
*skb
)
919 printk(KERN_ERR
"%s: received bad CPL command 0x%x\n", dev
->name
,
921 return CPL_RET_BUF_DONE
| CPL_RET_BAD_MSG
;
925 * Handlers for each CPL opcode
927 static cpl_handler_func cpl_handlers
[NUM_CPL_CMDS
];
930 * Add a new handler to the CPL dispatch table. A NULL handler may be supplied
931 * to unregister an existing handler.
933 void t3_register_cpl_handler(unsigned int opcode
, cpl_handler_func h
)
935 if (opcode
< NUM_CPL_CMDS
)
936 cpl_handlers
[opcode
] = h
? h
: do_bad_cpl
;
938 printk(KERN_ERR
"T3C: handler registration for "
939 "opcode %x failed\n", opcode
);
942 EXPORT_SYMBOL(t3_register_cpl_handler
);
945 * T3CDEV's receive method.
947 int process_rx(struct t3cdev
*dev
, struct sk_buff
**skbs
, int n
)
950 struct sk_buff
*skb
= *skbs
++;
951 unsigned int opcode
= get_opcode(skb
);
952 int ret
= cpl_handlers
[opcode
] (dev
, skb
);
955 if (ret
& CPL_RET_UNKNOWN_TID
) {
956 union opcode_tid
*p
= cplhdr(skb
);
958 printk(KERN_ERR
"%s: CPL message (opcode %u) had "
959 "unknown TID %u\n", dev
->name
, opcode
,
960 G_TID(ntohl(p
->opcode_tid
)));
963 if (ret
& CPL_RET_BUF_DONE
)
970 * Sends an sk_buff to a T3C driver after dealing with any active network taps.
972 int cxgb3_ofld_send(struct t3cdev
*dev
, struct sk_buff
*skb
)
977 r
= dev
->send(dev
, skb
);
982 EXPORT_SYMBOL(cxgb3_ofld_send
);
984 static int is_offloading(struct net_device
*dev
)
986 struct adapter
*adapter
;
989 read_lock_bh(&adapter_list_lock
);
990 list_for_each_entry(adapter
, &adapter_list
, adapter_list
) {
991 for_each_port(adapter
, i
) {
992 if (dev
== adapter
->port
[i
]) {
993 read_unlock_bh(&adapter_list_lock
);
998 read_unlock_bh(&adapter_list_lock
);
1002 void cxgb_neigh_update(struct neighbour
*neigh
)
1004 struct net_device
*dev
= neigh
->dev
;
1006 if (dev
&& (is_offloading(dev
))) {
1007 struct t3cdev
*tdev
= dev2t3cdev(dev
);
1010 t3_l2t_update(tdev
, neigh
);
1014 static void set_l2t_ix(struct t3cdev
*tdev
, u32 tid
, struct l2t_entry
*e
)
1016 struct sk_buff
*skb
;
1017 struct cpl_set_tcb_field
*req
;
1019 skb
= alloc_skb(sizeof(*req
), GFP_ATOMIC
);
1021 printk(KERN_ERR
"%s: cannot allocate skb!\n", __FUNCTION__
);
1024 skb
->priority
= CPL_PRIORITY_CONTROL
;
1025 req
= (struct cpl_set_tcb_field
*)skb_put(skb
, sizeof(*req
));
1026 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_FORWARD
));
1027 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD
, tid
));
1030 req
->word
= htons(W_TCB_L2T_IX
);
1031 req
->mask
= cpu_to_be64(V_TCB_L2T_IX(M_TCB_L2T_IX
));
1032 req
->val
= cpu_to_be64(V_TCB_L2T_IX(e
->idx
));
1033 tdev
->send(tdev
, skb
);
1036 void cxgb_redirect(struct dst_entry
*old
, struct dst_entry
*new)
1038 struct net_device
*olddev
, *newdev
;
1039 struct tid_info
*ti
;
1040 struct t3cdev
*tdev
;
1043 struct l2t_entry
*e
;
1044 struct t3c_tid_entry
*te
;
1046 olddev
= old
->neighbour
->dev
;
1047 newdev
= new->neighbour
->dev
;
1048 if (!is_offloading(olddev
))
1050 if (!is_offloading(newdev
)) {
1051 printk(KERN_WARNING
"%s: Redirect to non-offload "
1052 "device ignored.\n", __FUNCTION__
);
1055 tdev
= dev2t3cdev(olddev
);
1057 if (tdev
!= dev2t3cdev(newdev
)) {
1058 printk(KERN_WARNING
"%s: Redirect to different "
1059 "offload device ignored.\n", __FUNCTION__
);
1063 /* Add new L2T entry */
1064 e
= t3_l2t_get(tdev
, new->neighbour
, newdev
);
1066 printk(KERN_ERR
"%s: couldn't allocate new l2t entry!\n",
1071 /* Walk tid table and notify clients of dst change. */
1072 ti
= &(T3C_DATA(tdev
))->tid_maps
;
1073 for (tid
= 0; tid
< ti
->ntids
; tid
++) {
1074 te
= lookup_tid(ti
, tid
);
1076 if (te
&& te
->ctx
&& te
->client
&& te
->client
->redirect
) {
1077 update_tcb
= te
->client
->redirect(te
->ctx
, old
, new, e
);
1079 l2t_hold(L2DATA(tdev
), e
);
1080 set_l2t_ix(tdev
, tid
, e
);
1084 l2t_release(L2DATA(tdev
), e
);
1088 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1089 * The allocated memory is cleared.
1091 void *cxgb_alloc_mem(unsigned long size
)
1093 void *p
= kmalloc(size
, GFP_KERNEL
);
1103 * Free memory allocated through t3_alloc_mem().
1105 void cxgb_free_mem(void *addr
)
1107 if (is_vmalloc_addr(addr
))
1114 * Allocate and initialize the TID tables. Returns 0 on success.
1116 static int init_tid_tabs(struct tid_info
*t
, unsigned int ntids
,
1117 unsigned int natids
, unsigned int nstids
,
1118 unsigned int atid_base
, unsigned int stid_base
)
1120 unsigned long size
= ntids
* sizeof(*t
->tid_tab
) +
1121 natids
* sizeof(*t
->atid_tab
) + nstids
* sizeof(*t
->stid_tab
);
1123 t
->tid_tab
= cxgb_alloc_mem(size
);
1127 t
->stid_tab
= (union listen_entry
*)&t
->tid_tab
[ntids
];
1128 t
->atid_tab
= (union active_open_entry
*)&t
->stid_tab
[nstids
];
1131 t
->stid_base
= stid_base
;
1134 t
->atid_base
= atid_base
;
1136 t
->stids_in_use
= t
->atids_in_use
= 0;
1137 atomic_set(&t
->tids_in_use
, 0);
1138 spin_lock_init(&t
->stid_lock
);
1139 spin_lock_init(&t
->atid_lock
);
1142 * Setup the free lists for stid_tab and atid_tab.
1146 t
->stid_tab
[nstids
- 1].next
= &t
->stid_tab
[nstids
];
1147 t
->sfree
= t
->stid_tab
;
1151 t
->atid_tab
[natids
- 1].next
= &t
->atid_tab
[natids
];
1152 t
->afree
= t
->atid_tab
;
1157 static void free_tid_maps(struct tid_info
*t
)
1159 cxgb_free_mem(t
->tid_tab
);
1162 static inline void add_adapter(struct adapter
*adap
)
1164 write_lock_bh(&adapter_list_lock
);
1165 list_add_tail(&adap
->adapter_list
, &adapter_list
);
1166 write_unlock_bh(&adapter_list_lock
);
1169 static inline void remove_adapter(struct adapter
*adap
)
1171 write_lock_bh(&adapter_list_lock
);
1172 list_del(&adap
->adapter_list
);
1173 write_unlock_bh(&adapter_list_lock
);
1176 int cxgb3_offload_activate(struct adapter
*adapter
)
1178 struct t3cdev
*dev
= &adapter
->tdev
;
1181 struct tid_range stid_range
, tid_range
;
1182 struct mtutab mtutab
;
1183 unsigned int l2t_capacity
;
1185 t
= kcalloc(1, sizeof(*t
), GFP_KERNEL
);
1190 if (dev
->ctl(dev
, GET_TX_MAX_CHUNK
, &t
->tx_max_chunk
) < 0 ||
1191 dev
->ctl(dev
, GET_MAX_OUTSTANDING_WR
, &t
->max_wrs
) < 0 ||
1192 dev
->ctl(dev
, GET_L2T_CAPACITY
, &l2t_capacity
) < 0 ||
1193 dev
->ctl(dev
, GET_MTUS
, &mtutab
) < 0 ||
1194 dev
->ctl(dev
, GET_TID_RANGE
, &tid_range
) < 0 ||
1195 dev
->ctl(dev
, GET_STID_RANGE
, &stid_range
) < 0)
1199 L2DATA(dev
) = t3_init_l2t(l2t_capacity
);
1203 natids
= min(tid_range
.num
/ 2, MAX_ATIDS
);
1204 err
= init_tid_tabs(&t
->tid_maps
, tid_range
.num
, natids
,
1205 stid_range
.num
, ATID_BASE
, stid_range
.base
);
1209 t
->mtus
= mtutab
.mtus
;
1210 t
->nmtus
= mtutab
.size
;
1212 INIT_WORK(&t
->tid_release_task
, t3_process_tid_release_list
);
1213 spin_lock_init(&t
->tid_release_lock
);
1214 INIT_LIST_HEAD(&t
->list_node
);
1218 dev
->recv
= process_rx
;
1219 dev
->neigh_update
= t3_l2t_update
;
1221 /* Register netevent handler once */
1222 if (list_empty(&adapter_list
))
1223 register_netevent_notifier(&nb
);
1225 add_adapter(adapter
);
1229 t3_free_l2t(L2DATA(dev
));
1236 void cxgb3_offload_deactivate(struct adapter
*adapter
)
1238 struct t3cdev
*tdev
= &adapter
->tdev
;
1239 struct t3c_data
*t
= T3C_DATA(tdev
);
1241 remove_adapter(adapter
);
1242 if (list_empty(&adapter_list
))
1243 unregister_netevent_notifier(&nb
);
1245 free_tid_maps(&t
->tid_maps
);
1246 T3C_DATA(tdev
) = NULL
;
1247 t3_free_l2t(L2DATA(tdev
));
1248 L2DATA(tdev
) = NULL
;
1252 static inline void register_tdev(struct t3cdev
*tdev
)
1256 mutex_lock(&cxgb3_db_lock
);
1257 snprintf(tdev
->name
, sizeof(tdev
->name
), "ofld_dev%d", unit
++);
1258 list_add_tail(&tdev
->ofld_dev_list
, &ofld_dev_list
);
1259 mutex_unlock(&cxgb3_db_lock
);
1262 static inline void unregister_tdev(struct t3cdev
*tdev
)
1264 mutex_lock(&cxgb3_db_lock
);
1265 list_del(&tdev
->ofld_dev_list
);
1266 mutex_unlock(&cxgb3_db_lock
);
1269 static inline int adap2type(struct adapter
*adapter
)
1273 switch (adapter
->params
.rev
) {
1288 void __devinit
cxgb3_adapter_ofld(struct adapter
*adapter
)
1290 struct t3cdev
*tdev
= &adapter
->tdev
;
1292 INIT_LIST_HEAD(&tdev
->ofld_dev_list
);
1294 cxgb3_set_dummy_ops(tdev
);
1295 tdev
->send
= t3_offload_tx
;
1296 tdev
->ctl
= cxgb_offload_ctl
;
1297 tdev
->type
= adap2type(adapter
);
1299 register_tdev(tdev
);
1302 void __devexit
cxgb3_adapter_unofld(struct adapter
*adapter
)
1304 struct t3cdev
*tdev
= &adapter
->tdev
;
1307 tdev
->neigh_update
= NULL
;
1309 unregister_tdev(tdev
);
1312 void __init
cxgb3_offload_init(void)
1316 for (i
= 0; i
< NUM_CPL_CMDS
; ++i
)
1317 cpl_handlers
[i
] = do_bad_cpl
;
1319 t3_register_cpl_handler(CPL_SMT_WRITE_RPL
, do_smt_write_rpl
);
1320 t3_register_cpl_handler(CPL_L2T_WRITE_RPL
, do_l2t_write_rpl
);
1321 t3_register_cpl_handler(CPL_RTE_WRITE_RPL
, do_rte_write_rpl
);
1322 t3_register_cpl_handler(CPL_PASS_OPEN_RPL
, do_stid_rpl
);
1323 t3_register_cpl_handler(CPL_CLOSE_LISTSRV_RPL
, do_stid_rpl
);
1324 t3_register_cpl_handler(CPL_PASS_ACCEPT_REQ
, do_cr
);
1325 t3_register_cpl_handler(CPL_PASS_ESTABLISH
, do_hwtid_rpl
);
1326 t3_register_cpl_handler(CPL_ABORT_RPL_RSS
, do_hwtid_rpl
);
1327 t3_register_cpl_handler(CPL_ABORT_RPL
, do_hwtid_rpl
);
1328 t3_register_cpl_handler(CPL_RX_URG_NOTIFY
, do_hwtid_rpl
);
1329 t3_register_cpl_handler(CPL_RX_DATA
, do_hwtid_rpl
);
1330 t3_register_cpl_handler(CPL_TX_DATA_ACK
, do_hwtid_rpl
);
1331 t3_register_cpl_handler(CPL_TX_DMA_ACK
, do_hwtid_rpl
);
1332 t3_register_cpl_handler(CPL_ACT_OPEN_RPL
, do_act_open_rpl
);
1333 t3_register_cpl_handler(CPL_PEER_CLOSE
, do_hwtid_rpl
);
1334 t3_register_cpl_handler(CPL_CLOSE_CON_RPL
, do_hwtid_rpl
);
1335 t3_register_cpl_handler(CPL_ABORT_REQ_RSS
, do_abort_req_rss
);
1336 t3_register_cpl_handler(CPL_ACT_ESTABLISH
, do_act_establish
);
1337 t3_register_cpl_handler(CPL_SET_TCB_RPL
, do_hwtid_rpl
);
1338 t3_register_cpl_handler(CPL_GET_TCB_RPL
, do_hwtid_rpl
);
1339 t3_register_cpl_handler(CPL_RDMA_TERMINATE
, do_term
);
1340 t3_register_cpl_handler(CPL_RDMA_EC_STATUS
, do_hwtid_rpl
);
1341 t3_register_cpl_handler(CPL_TRACE_PKT
, do_trace
);
1342 t3_register_cpl_handler(CPL_RX_DATA_DDP
, do_hwtid_rpl
);
1343 t3_register_cpl_handler(CPL_RX_DDP_COMPLETE
, do_hwtid_rpl
);
1344 t3_register_cpl_handler(CPL_ISCSI_HDR
, do_hwtid_rpl
);