2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5 * Copyright (c) 2004, 2005 Voltaire, Inc. All rights reserved.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 * $Id: ipoib_ib.c 1386 2004-12-27 16:23:17Z roland $
38 #include <linux/delay.h>
39 #include <linux/dma-mapping.h>
41 #include <rdma/ib_cache.h>
45 #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG_DATA
46 static int data_debug_level
;
48 module_param(data_debug_level
, int, 0644);
49 MODULE_PARM_DESC(data_debug_level
,
50 "Enable data path debug tracing if > 0");
53 #define IPOIB_OP_RECV (1ul << 31)
55 static DEFINE_MUTEX(pkey_mutex
);
57 struct ipoib_ah
*ipoib_create_ah(struct net_device
*dev
,
58 struct ib_pd
*pd
, struct ib_ah_attr
*attr
)
62 ah
= kmalloc(sizeof *ah
, GFP_KERNEL
);
70 ah
->ah
= ib_create_ah(pd
, attr
);
75 ipoib_dbg(netdev_priv(dev
), "Created ah %p\n", ah
->ah
);
80 void ipoib_free_ah(struct kref
*kref
)
82 struct ipoib_ah
*ah
= container_of(kref
, struct ipoib_ah
, ref
);
83 struct ipoib_dev_priv
*priv
= netdev_priv(ah
->dev
);
87 spin_lock_irqsave(&priv
->lock
, flags
);
88 list_add_tail(&ah
->list
, &priv
->dead_ahs
);
89 spin_unlock_irqrestore(&priv
->lock
, flags
);
92 static int ipoib_ib_post_receive(struct net_device
*dev
, int id
)
94 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
96 struct ib_recv_wr param
;
97 struct ib_recv_wr
*bad_wr
;
100 list
.addr
= priv
->rx_ring
[id
].mapping
;
101 list
.length
= IPOIB_BUF_SIZE
;
102 list
.lkey
= priv
->mr
->lkey
;
105 param
.wr_id
= id
| IPOIB_OP_RECV
;
106 param
.sg_list
= &list
;
109 ret
= ib_post_recv(priv
->qp
, ¶m
, &bad_wr
);
111 ipoib_warn(priv
, "receive failed for buf %d (%d)\n", id
, ret
);
112 dma_unmap_single(priv
->ca
->dma_device
,
113 priv
->rx_ring
[id
].mapping
,
114 IPOIB_BUF_SIZE
, DMA_FROM_DEVICE
);
115 dev_kfree_skb_any(priv
->rx_ring
[id
].skb
);
116 priv
->rx_ring
[id
].skb
= NULL
;
122 static int ipoib_alloc_rx_skb(struct net_device
*dev
, int id
)
124 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
128 skb
= dev_alloc_skb(IPOIB_BUF_SIZE
+ 4);
133 * IB will leave a 40 byte gap for a GRH and IPoIB adds a 4 byte
134 * header. So we need 4 more bytes to get to 48 and align the
135 * IP header to a multiple of 16.
139 addr
= dma_map_single(priv
->ca
->dma_device
,
140 skb
->data
, IPOIB_BUF_SIZE
,
142 if (unlikely(dma_mapping_error(addr
))) {
143 dev_kfree_skb_any(skb
);
147 priv
->rx_ring
[id
].skb
= skb
;
148 priv
->rx_ring
[id
].mapping
= addr
;
153 static int ipoib_ib_post_receives(struct net_device
*dev
)
155 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
158 for (i
= 0; i
< ipoib_recvq_size
; ++i
) {
159 if (ipoib_alloc_rx_skb(dev
, i
)) {
160 ipoib_warn(priv
, "failed to allocate receive buffer %d\n", i
);
163 if (ipoib_ib_post_receive(dev
, i
)) {
164 ipoib_warn(priv
, "ipoib_ib_post_receive failed for buf %d\n", i
);
172 static void ipoib_ib_handle_rx_wc(struct net_device
*dev
, struct ib_wc
*wc
)
174 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
175 unsigned int wr_id
= wc
->wr_id
& ~IPOIB_OP_RECV
;
179 ipoib_dbg_data(priv
, "recv completion: id %d, op %d, status: %d\n",
180 wr_id
, wc
->opcode
, wc
->status
);
182 if (unlikely(wr_id
>= ipoib_recvq_size
)) {
183 ipoib_warn(priv
, "recv completion event with wrid %d (> %d)\n",
184 wr_id
, ipoib_recvq_size
);
188 skb
= priv
->rx_ring
[wr_id
].skb
;
189 addr
= priv
->rx_ring
[wr_id
].mapping
;
191 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
192 if (wc
->status
!= IB_WC_WR_FLUSH_ERR
)
193 ipoib_warn(priv
, "failed recv event "
194 "(status=%d, wrid=%d vend_err %x)\n",
195 wc
->status
, wr_id
, wc
->vendor_err
);
196 dma_unmap_single(priv
->ca
->dma_device
, addr
,
197 IPOIB_BUF_SIZE
, DMA_FROM_DEVICE
);
198 dev_kfree_skb_any(skb
);
199 priv
->rx_ring
[wr_id
].skb
= NULL
;
204 * If we can't allocate a new RX buffer, dump
205 * this packet and reuse the old buffer.
207 if (unlikely(ipoib_alloc_rx_skb(dev
, wr_id
))) {
208 ++priv
->stats
.rx_dropped
;
212 ipoib_dbg_data(priv
, "received %d bytes, SLID 0x%04x\n",
213 wc
->byte_len
, wc
->slid
);
215 dma_unmap_single(priv
->ca
->dma_device
, addr
,
216 IPOIB_BUF_SIZE
, DMA_FROM_DEVICE
);
218 skb_put(skb
, wc
->byte_len
);
219 skb_pull(skb
, IB_GRH_BYTES
);
221 if (wc
->slid
!= priv
->local_lid
||
222 wc
->src_qp
!= priv
->qp
->qp_num
) {
223 skb
->protocol
= ((struct ipoib_header
*) skb
->data
)->proto
;
224 skb
->mac
.raw
= skb
->data
;
225 skb_pull(skb
, IPOIB_ENCAP_LEN
);
227 dev
->last_rx
= jiffies
;
228 ++priv
->stats
.rx_packets
;
229 priv
->stats
.rx_bytes
+= skb
->len
;
232 /* XXX get correct PACKET_ type here */
233 skb
->pkt_type
= PACKET_HOST
;
236 ipoib_dbg_data(priv
, "dropping loopback packet\n");
237 dev_kfree_skb_any(skb
);
241 if (unlikely(ipoib_ib_post_receive(dev
, wr_id
)))
242 ipoib_warn(priv
, "ipoib_ib_post_receive failed "
243 "for buf %d\n", wr_id
);
246 static void ipoib_ib_handle_tx_wc(struct net_device
*dev
, struct ib_wc
*wc
)
248 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
249 unsigned int wr_id
= wc
->wr_id
;
250 struct ipoib_tx_buf
*tx_req
;
253 ipoib_dbg_data(priv
, "send completion: id %d, op %d, status: %d\n",
254 wr_id
, wc
->opcode
, wc
->status
);
256 if (unlikely(wr_id
>= ipoib_sendq_size
)) {
257 ipoib_warn(priv
, "send completion event with wrid %d (> %d)\n",
258 wr_id
, ipoib_sendq_size
);
262 tx_req
= &priv
->tx_ring
[wr_id
];
264 dma_unmap_single(priv
->ca
->dma_device
,
265 pci_unmap_addr(tx_req
, mapping
),
269 ++priv
->stats
.tx_packets
;
270 priv
->stats
.tx_bytes
+= tx_req
->skb
->len
;
272 dev_kfree_skb_any(tx_req
->skb
);
274 spin_lock_irqsave(&priv
->tx_lock
, flags
);
276 if (netif_queue_stopped(dev
) &&
277 test_bit(IPOIB_FLAG_ADMIN_UP
, &priv
->flags
) &&
278 priv
->tx_head
- priv
->tx_tail
<= ipoib_sendq_size
>> 1)
279 netif_wake_queue(dev
);
280 spin_unlock_irqrestore(&priv
->tx_lock
, flags
);
282 if (wc
->status
!= IB_WC_SUCCESS
&&
283 wc
->status
!= IB_WC_WR_FLUSH_ERR
)
284 ipoib_warn(priv
, "failed send event "
285 "(status=%d, wrid=%d vend_err %x)\n",
286 wc
->status
, wr_id
, wc
->vendor_err
);
289 static void ipoib_ib_handle_wc(struct net_device
*dev
, struct ib_wc
*wc
)
291 if (wc
->wr_id
& IPOIB_OP_RECV
)
292 ipoib_ib_handle_rx_wc(dev
, wc
);
294 ipoib_ib_handle_tx_wc(dev
, wc
);
297 void ipoib_ib_completion(struct ib_cq
*cq
, void *dev_ptr
)
299 struct net_device
*dev
= (struct net_device
*) dev_ptr
;
300 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
303 ib_req_notify_cq(cq
, IB_CQ_NEXT_COMP
);
305 n
= ib_poll_cq(cq
, IPOIB_NUM_WC
, priv
->ibwc
);
306 for (i
= 0; i
< n
; ++i
)
307 ipoib_ib_handle_wc(dev
, priv
->ibwc
+ i
);
308 } while (n
== IPOIB_NUM_WC
);
311 static inline int post_send(struct ipoib_dev_priv
*priv
,
313 struct ib_ah
*address
, u32 qpn
,
314 dma_addr_t addr
, int len
)
316 struct ib_send_wr
*bad_wr
;
318 priv
->tx_sge
.addr
= addr
;
319 priv
->tx_sge
.length
= len
;
321 priv
->tx_wr
.wr_id
= wr_id
;
322 priv
->tx_wr
.wr
.ud
.remote_qpn
= qpn
;
323 priv
->tx_wr
.wr
.ud
.ah
= address
;
325 return ib_post_send(priv
->qp
, &priv
->tx_wr
, &bad_wr
);
328 void ipoib_send(struct net_device
*dev
, struct sk_buff
*skb
,
329 struct ipoib_ah
*address
, u32 qpn
)
331 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
332 struct ipoib_tx_buf
*tx_req
;
335 if (skb
->len
> dev
->mtu
+ INFINIBAND_ALEN
) {
336 ipoib_warn(priv
, "packet len %d (> %d) too long to send, dropping\n",
337 skb
->len
, dev
->mtu
+ INFINIBAND_ALEN
);
338 ++priv
->stats
.tx_dropped
;
339 ++priv
->stats
.tx_errors
;
340 dev_kfree_skb_any(skb
);
344 ipoib_dbg_data(priv
, "sending packet, length=%d address=%p qpn=0x%06x\n",
345 skb
->len
, address
, qpn
);
348 * We put the skb into the tx_ring _before_ we call post_send()
349 * because it's entirely possible that the completion handler will
350 * run before we execute anything after the post_send(). That
351 * means we have to make sure everything is properly recorded and
352 * our state is consistent before we call post_send().
354 tx_req
= &priv
->tx_ring
[priv
->tx_head
& (ipoib_sendq_size
- 1)];
356 addr
= dma_map_single(priv
->ca
->dma_device
, skb
->data
, skb
->len
,
358 pci_unmap_addr_set(tx_req
, mapping
, addr
);
360 if (unlikely(post_send(priv
, priv
->tx_head
& (ipoib_sendq_size
- 1),
361 address
->ah
, qpn
, addr
, skb
->len
))) {
362 ipoib_warn(priv
, "post_send failed\n");
363 ++priv
->stats
.tx_errors
;
364 dma_unmap_single(priv
->ca
->dma_device
, addr
, skb
->len
,
366 dev_kfree_skb_any(skb
);
368 dev
->trans_start
= jiffies
;
370 address
->last_send
= priv
->tx_head
;
373 if (priv
->tx_head
- priv
->tx_tail
== ipoib_sendq_size
) {
374 ipoib_dbg(priv
, "TX ring full, stopping kernel net queue\n");
375 netif_stop_queue(dev
);
380 static void __ipoib_reap_ah(struct net_device
*dev
)
382 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
383 struct ipoib_ah
*ah
, *tah
;
384 LIST_HEAD(remove_list
);
386 spin_lock_irq(&priv
->tx_lock
);
387 spin_lock(&priv
->lock
);
388 list_for_each_entry_safe(ah
, tah
, &priv
->dead_ahs
, list
)
389 if ((int) priv
->tx_tail
- (int) ah
->last_send
>= 0) {
391 ib_destroy_ah(ah
->ah
);
394 spin_unlock(&priv
->lock
);
395 spin_unlock_irq(&priv
->tx_lock
);
398 void ipoib_reap_ah(void *dev_ptr
)
400 struct net_device
*dev
= dev_ptr
;
401 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
403 __ipoib_reap_ah(dev
);
405 if (!test_bit(IPOIB_STOP_REAPER
, &priv
->flags
))
406 queue_delayed_work(ipoib_workqueue
, &priv
->ah_reap_task
, HZ
);
409 int ipoib_ib_dev_open(struct net_device
*dev
)
411 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
414 ret
= ipoib_init_qp(dev
);
416 ipoib_warn(priv
, "ipoib_init_qp returned %d\n", ret
);
420 ret
= ipoib_ib_post_receives(dev
);
422 ipoib_warn(priv
, "ipoib_ib_post_receives returned %d\n", ret
);
423 ipoib_ib_dev_stop(dev
);
427 clear_bit(IPOIB_STOP_REAPER
, &priv
->flags
);
428 queue_delayed_work(ipoib_workqueue
, &priv
->ah_reap_task
, HZ
);
430 set_bit(IPOIB_FLAG_INITIALIZED
, &priv
->flags
);
435 static void ipoib_pkey_dev_check_presence(struct net_device
*dev
)
437 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
440 if (ib_find_cached_pkey(priv
->ca
, priv
->port
, priv
->pkey
, &pkey_index
))
441 clear_bit(IPOIB_PKEY_ASSIGNED
, &priv
->flags
);
443 set_bit(IPOIB_PKEY_ASSIGNED
, &priv
->flags
);
446 int ipoib_ib_dev_up(struct net_device
*dev
)
448 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
450 ipoib_pkey_dev_check_presence(dev
);
452 if (!test_bit(IPOIB_PKEY_ASSIGNED
, &priv
->flags
)) {
453 ipoib_dbg(priv
, "PKEY is not assigned.\n");
457 set_bit(IPOIB_FLAG_OPER_UP
, &priv
->flags
);
459 return ipoib_mcast_start_thread(dev
);
462 int ipoib_ib_dev_down(struct net_device
*dev
, int flush
)
464 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
466 ipoib_dbg(priv
, "downing ib_dev\n");
468 clear_bit(IPOIB_FLAG_OPER_UP
, &priv
->flags
);
469 netif_carrier_off(dev
);
471 /* Shutdown the P_Key thread if still active */
472 if (!test_bit(IPOIB_PKEY_ASSIGNED
, &priv
->flags
)) {
473 mutex_lock(&pkey_mutex
);
474 set_bit(IPOIB_PKEY_STOP
, &priv
->flags
);
475 cancel_delayed_work(&priv
->pkey_task
);
476 mutex_unlock(&pkey_mutex
);
478 flush_workqueue(ipoib_workqueue
);
481 ipoib_mcast_stop_thread(dev
, flush
);
482 ipoib_mcast_dev_flush(dev
);
484 ipoib_flush_paths(dev
);
489 static int recvs_pending(struct net_device
*dev
)
491 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
495 for (i
= 0; i
< ipoib_recvq_size
; ++i
)
496 if (priv
->rx_ring
[i
].skb
)
502 int ipoib_ib_dev_stop(struct net_device
*dev
)
504 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
505 struct ib_qp_attr qp_attr
;
507 struct ipoib_tx_buf
*tx_req
;
510 clear_bit(IPOIB_FLAG_INITIALIZED
, &priv
->flags
);
513 * Move our QP to the error state and then reinitialize in
514 * when all work requests have completed or have been flushed.
516 qp_attr
.qp_state
= IB_QPS_ERR
;
517 if (ib_modify_qp(priv
->qp
, &qp_attr
, IB_QP_STATE
))
518 ipoib_warn(priv
, "Failed to modify QP to ERROR state\n");
520 /* Wait for all sends and receives to complete */
523 while (priv
->tx_head
!= priv
->tx_tail
|| recvs_pending(dev
)) {
524 if (time_after(jiffies
, begin
+ 5 * HZ
)) {
525 ipoib_warn(priv
, "timing out; %d sends %d receives not completed\n",
526 priv
->tx_head
- priv
->tx_tail
, recvs_pending(dev
));
529 * assume the HW is wedged and just free up
530 * all our pending work requests.
532 while ((int) priv
->tx_tail
- (int) priv
->tx_head
< 0) {
533 tx_req
= &priv
->tx_ring
[priv
->tx_tail
&
534 (ipoib_sendq_size
- 1)];
535 dma_unmap_single(priv
->ca
->dma_device
,
536 pci_unmap_addr(tx_req
, mapping
),
539 dev_kfree_skb_any(tx_req
->skb
);
543 for (i
= 0; i
< ipoib_recvq_size
; ++i
)
544 if (priv
->rx_ring
[i
].skb
) {
545 dma_unmap_single(priv
->ca
->dma_device
,
546 pci_unmap_addr(&priv
->rx_ring
[i
],
550 dev_kfree_skb_any(priv
->rx_ring
[i
].skb
);
551 priv
->rx_ring
[i
].skb
= NULL
;
560 ipoib_dbg(priv
, "All sends and receives done.\n");
563 qp_attr
.qp_state
= IB_QPS_RESET
;
564 if (ib_modify_qp(priv
->qp
, &qp_attr
, IB_QP_STATE
))
565 ipoib_warn(priv
, "Failed to modify QP to RESET state\n");
567 /* Wait for all AHs to be reaped */
568 set_bit(IPOIB_STOP_REAPER
, &priv
->flags
);
569 cancel_delayed_work(&priv
->ah_reap_task
);
570 flush_workqueue(ipoib_workqueue
);
574 while (!list_empty(&priv
->dead_ahs
)) {
575 __ipoib_reap_ah(dev
);
577 if (time_after(jiffies
, begin
+ HZ
)) {
578 ipoib_warn(priv
, "timing out; will leak address handles\n");
588 int ipoib_ib_dev_init(struct net_device
*dev
, struct ib_device
*ca
, int port
)
590 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
596 if (ipoib_transport_dev_init(dev
, ca
)) {
597 printk(KERN_WARNING
"%s: ipoib_transport_dev_init failed\n", ca
->name
);
601 if (dev
->flags
& IFF_UP
) {
602 if (ipoib_ib_dev_open(dev
)) {
603 ipoib_transport_dev_cleanup(dev
);
611 void ipoib_ib_dev_flush(void *_dev
)
613 struct net_device
*dev
= (struct net_device
*)_dev
;
614 struct ipoib_dev_priv
*priv
= netdev_priv(dev
), *cpriv
;
616 if (!test_bit(IPOIB_FLAG_INITIALIZED
, &priv
->flags
) ) {
617 ipoib_dbg(priv
, "Not flushing - IPOIB_FLAG_INITIALIZED not set.\n");
621 if (!test_bit(IPOIB_FLAG_ADMIN_UP
, &priv
->flags
)) {
622 ipoib_dbg(priv
, "Not flushing - IPOIB_FLAG_ADMIN_UP not set.\n");
626 ipoib_dbg(priv
, "flushing\n");
628 ipoib_ib_dev_down(dev
, 0);
631 * The device could have been brought down between the start and when
632 * we get here, don't bring it back up if it's not configured up
634 if (test_bit(IPOIB_FLAG_ADMIN_UP
, &priv
->flags
)) {
635 ipoib_ib_dev_up(dev
);
636 ipoib_mcast_restart_task(dev
);
639 mutex_lock(&priv
->vlan_mutex
);
641 /* Flush any child interfaces too */
642 list_for_each_entry(cpriv
, &priv
->child_intfs
, list
)
643 ipoib_ib_dev_flush(cpriv
->dev
);
645 mutex_unlock(&priv
->vlan_mutex
);
648 void ipoib_ib_dev_cleanup(struct net_device
*dev
)
650 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
652 ipoib_dbg(priv
, "cleaning up ib_dev\n");
654 ipoib_mcast_stop_thread(dev
, 1);
655 ipoib_mcast_dev_flush(dev
);
657 ipoib_transport_dev_cleanup(dev
);
661 * Delayed P_Key Assigment Interim Support
663 * The following is initial implementation of delayed P_Key assigment
664 * mechanism. It is using the same approach implemented for the multicast
665 * group join. The single goal of this implementation is to quickly address
666 * Bug #2507. This implementation will probably be removed when the P_Key
667 * change async notification is available.
670 void ipoib_pkey_poll(void *dev_ptr
)
672 struct net_device
*dev
= dev_ptr
;
673 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
675 ipoib_pkey_dev_check_presence(dev
);
677 if (test_bit(IPOIB_PKEY_ASSIGNED
, &priv
->flags
))
680 mutex_lock(&pkey_mutex
);
681 if (!test_bit(IPOIB_PKEY_STOP
, &priv
->flags
))
682 queue_delayed_work(ipoib_workqueue
,
685 mutex_unlock(&pkey_mutex
);
689 int ipoib_pkey_dev_delay_open(struct net_device
*dev
)
691 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
693 /* Look for the interface pkey value in the IB Port P_Key table and */
694 /* set the interface pkey assigment flag */
695 ipoib_pkey_dev_check_presence(dev
);
697 /* P_Key value not assigned yet - start polling */
698 if (!test_bit(IPOIB_PKEY_ASSIGNED
, &priv
->flags
)) {
699 mutex_lock(&pkey_mutex
);
700 clear_bit(IPOIB_PKEY_STOP
, &priv
->flags
);
701 queue_delayed_work(ipoib_workqueue
,
704 mutex_unlock(&pkey_mutex
);