2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5 * Copyright (c) 2004, 2005 Voltaire, Inc. All rights reserved.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 * $Id: ipoib_ib.c 1386 2004-12-27 16:23:17Z roland $
38 #include <linux/delay.h>
39 #include <linux/dma-mapping.h>
41 #include <rdma/ib_cache.h>
45 #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG_DATA
46 static int data_debug_level
;
48 module_param(data_debug_level
, int, 0644);
49 MODULE_PARM_DESC(data_debug_level
,
50 "Enable data path debug tracing if > 0");
53 #define IPOIB_OP_RECV (1ul << 31)
55 static DECLARE_MUTEX(pkey_sem
);
57 struct ipoib_ah
*ipoib_create_ah(struct net_device
*dev
,
58 struct ib_pd
*pd
, struct ib_ah_attr
*attr
)
62 ah
= kmalloc(sizeof *ah
, GFP_KERNEL
);
70 ah
->ah
= ib_create_ah(pd
, attr
);
75 ipoib_dbg(netdev_priv(dev
), "Created ah %p\n", ah
->ah
);
80 void ipoib_free_ah(struct kref
*kref
)
82 struct ipoib_ah
*ah
= container_of(kref
, struct ipoib_ah
, ref
);
83 struct ipoib_dev_priv
*priv
= netdev_priv(ah
->dev
);
87 if ((int) priv
->tx_tail
- (int) ah
->last_send
>= 0) {
88 ipoib_dbg(priv
, "Freeing ah %p\n", ah
->ah
);
89 ib_destroy_ah(ah
->ah
);
92 spin_lock_irqsave(&priv
->lock
, flags
);
93 list_add_tail(&ah
->list
, &priv
->dead_ahs
);
94 spin_unlock_irqrestore(&priv
->lock
, flags
);
98 static int ipoib_ib_post_receive(struct net_device
*dev
, int id
)
100 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
102 struct ib_recv_wr param
;
103 struct ib_recv_wr
*bad_wr
;
106 list
.addr
= priv
->rx_ring
[id
].mapping
;
107 list
.length
= IPOIB_BUF_SIZE
;
108 list
.lkey
= priv
->mr
->lkey
;
111 param
.wr_id
= id
| IPOIB_OP_RECV
;
112 param
.sg_list
= &list
;
115 ret
= ib_post_recv(priv
->qp
, ¶m
, &bad_wr
);
117 ipoib_warn(priv
, "receive failed for buf %d (%d)\n", id
, ret
);
118 dma_unmap_single(priv
->ca
->dma_device
,
119 priv
->rx_ring
[id
].mapping
,
120 IPOIB_BUF_SIZE
, DMA_FROM_DEVICE
);
121 dev_kfree_skb_any(priv
->rx_ring
[id
].skb
);
122 priv
->rx_ring
[id
].skb
= NULL
;
128 static int ipoib_alloc_rx_skb(struct net_device
*dev
, int id
)
130 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
134 skb
= dev_alloc_skb(IPOIB_BUF_SIZE
+ 4);
139 * IB will leave a 40 byte gap for a GRH and IPoIB adds a 4 byte
140 * header. So we need 4 more bytes to get to 48 and align the
141 * IP header to a multiple of 16.
145 addr
= dma_map_single(priv
->ca
->dma_device
,
146 skb
->data
, IPOIB_BUF_SIZE
,
148 if (unlikely(dma_mapping_error(addr
))) {
149 dev_kfree_skb_any(skb
);
153 priv
->rx_ring
[id
].skb
= skb
;
154 priv
->rx_ring
[id
].mapping
= addr
;
159 static int ipoib_ib_post_receives(struct net_device
*dev
)
161 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
164 for (i
= 0; i
< IPOIB_RX_RING_SIZE
; ++i
) {
165 if (ipoib_alloc_rx_skb(dev
, i
)) {
166 ipoib_warn(priv
, "failed to allocate receive buffer %d\n", i
);
169 if (ipoib_ib_post_receive(dev
, i
)) {
170 ipoib_warn(priv
, "ipoib_ib_post_receive failed for buf %d\n", i
);
178 static void ipoib_ib_handle_wc(struct net_device
*dev
,
181 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
182 unsigned int wr_id
= wc
->wr_id
;
184 ipoib_dbg_data(priv
, "called: id %d, op %d, status: %d\n",
185 wr_id
, wc
->opcode
, wc
->status
);
187 if (wr_id
& IPOIB_OP_RECV
) {
188 wr_id
&= ~IPOIB_OP_RECV
;
190 if (wr_id
< IPOIB_RX_RING_SIZE
) {
191 struct sk_buff
*skb
= priv
->rx_ring
[wr_id
].skb
;
192 dma_addr_t addr
= priv
->rx_ring
[wr_id
].mapping
;
194 if (unlikely(wc
->status
!= IB_WC_SUCCESS
)) {
195 if (wc
->status
!= IB_WC_WR_FLUSH_ERR
)
196 ipoib_warn(priv
, "failed recv event "
197 "(status=%d, wrid=%d vend_err %x)\n",
198 wc
->status
, wr_id
, wc
->vendor_err
);
199 dma_unmap_single(priv
->ca
->dma_device
, addr
,
200 IPOIB_BUF_SIZE
, DMA_FROM_DEVICE
);
201 dev_kfree_skb_any(skb
);
202 priv
->rx_ring
[wr_id
].skb
= NULL
;
207 * If we can't allocate a new RX buffer, dump
208 * this packet and reuse the old buffer.
210 if (unlikely(ipoib_alloc_rx_skb(dev
, wr_id
))) {
211 ++priv
->stats
.rx_dropped
;
215 ipoib_dbg_data(priv
, "received %d bytes, SLID 0x%04x\n",
216 wc
->byte_len
, wc
->slid
);
218 dma_unmap_single(priv
->ca
->dma_device
, addr
,
219 IPOIB_BUF_SIZE
, DMA_FROM_DEVICE
);
221 skb_put(skb
, wc
->byte_len
);
222 skb_pull(skb
, IB_GRH_BYTES
);
224 if (wc
->slid
!= priv
->local_lid
||
225 wc
->src_qp
!= priv
->qp
->qp_num
) {
226 skb
->protocol
= ((struct ipoib_header
*) skb
->data
)->proto
;
227 skb
->mac
.raw
= skb
->data
;
228 skb_pull(skb
, IPOIB_ENCAP_LEN
);
230 dev
->last_rx
= jiffies
;
231 ++priv
->stats
.rx_packets
;
232 priv
->stats
.rx_bytes
+= skb
->len
;
235 /* XXX get correct PACKET_ type here */
236 skb
->pkt_type
= PACKET_HOST
;
239 ipoib_dbg_data(priv
, "dropping loopback packet\n");
240 dev_kfree_skb_any(skb
);
244 if (unlikely(ipoib_ib_post_receive(dev
, wr_id
)))
245 ipoib_warn(priv
, "ipoib_ib_post_receive failed "
246 "for buf %d\n", wr_id
);
248 ipoib_warn(priv
, "completion event with wrid %d\n",
252 struct ipoib_tx_buf
*tx_req
;
255 if (wr_id
>= IPOIB_TX_RING_SIZE
) {
256 ipoib_warn(priv
, "completion event with wrid %d (> %d)\n",
257 wr_id
, IPOIB_TX_RING_SIZE
);
261 ipoib_dbg_data(priv
, "send complete, wrid %d\n", wr_id
);
263 tx_req
= &priv
->tx_ring
[wr_id
];
265 dma_unmap_single(priv
->ca
->dma_device
,
266 pci_unmap_addr(tx_req
, mapping
),
270 ++priv
->stats
.tx_packets
;
271 priv
->stats
.tx_bytes
+= tx_req
->skb
->len
;
273 dev_kfree_skb_any(tx_req
->skb
);
275 spin_lock_irqsave(&priv
->tx_lock
, flags
);
277 if (netif_queue_stopped(dev
) &&
278 priv
->tx_head
- priv
->tx_tail
<= IPOIB_TX_RING_SIZE
/ 2)
279 netif_wake_queue(dev
);
280 spin_unlock_irqrestore(&priv
->tx_lock
, flags
);
282 if (wc
->status
!= IB_WC_SUCCESS
&&
283 wc
->status
!= IB_WC_WR_FLUSH_ERR
)
284 ipoib_warn(priv
, "failed send event "
285 "(status=%d, wrid=%d vend_err %x)\n",
286 wc
->status
, wr_id
, wc
->vendor_err
);
290 void ipoib_ib_completion(struct ib_cq
*cq
, void *dev_ptr
)
292 struct net_device
*dev
= (struct net_device
*) dev_ptr
;
293 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
296 ib_req_notify_cq(cq
, IB_CQ_NEXT_COMP
);
298 n
= ib_poll_cq(cq
, IPOIB_NUM_WC
, priv
->ibwc
);
299 for (i
= 0; i
< n
; ++i
)
300 ipoib_ib_handle_wc(dev
, priv
->ibwc
+ i
);
301 } while (n
== IPOIB_NUM_WC
);
304 static inline int post_send(struct ipoib_dev_priv
*priv
,
306 struct ib_ah
*address
, u32 qpn
,
307 dma_addr_t addr
, int len
)
309 struct ib_send_wr
*bad_wr
;
311 priv
->tx_sge
.addr
= addr
;
312 priv
->tx_sge
.length
= len
;
314 priv
->tx_wr
.wr_id
= wr_id
;
315 priv
->tx_wr
.wr
.ud
.remote_qpn
= qpn
;
316 priv
->tx_wr
.wr
.ud
.ah
= address
;
318 return ib_post_send(priv
->qp
, &priv
->tx_wr
, &bad_wr
);
321 void ipoib_send(struct net_device
*dev
, struct sk_buff
*skb
,
322 struct ipoib_ah
*address
, u32 qpn
)
324 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
325 struct ipoib_tx_buf
*tx_req
;
328 if (skb
->len
> dev
->mtu
+ INFINIBAND_ALEN
) {
329 ipoib_warn(priv
, "packet len %d (> %d) too long to send, dropping\n",
330 skb
->len
, dev
->mtu
+ INFINIBAND_ALEN
);
331 ++priv
->stats
.tx_dropped
;
332 ++priv
->stats
.tx_errors
;
333 dev_kfree_skb_any(skb
);
337 ipoib_dbg_data(priv
, "sending packet, length=%d address=%p qpn=0x%06x\n",
338 skb
->len
, address
, qpn
);
341 * We put the skb into the tx_ring _before_ we call post_send()
342 * because it's entirely possible that the completion handler will
343 * run before we execute anything after the post_send(). That
344 * means we have to make sure everything is properly recorded and
345 * our state is consistent before we call post_send().
347 tx_req
= &priv
->tx_ring
[priv
->tx_head
& (IPOIB_TX_RING_SIZE
- 1)];
349 addr
= dma_map_single(priv
->ca
->dma_device
, skb
->data
, skb
->len
,
351 pci_unmap_addr_set(tx_req
, mapping
, addr
);
353 if (unlikely(post_send(priv
, priv
->tx_head
& (IPOIB_TX_RING_SIZE
- 1),
354 address
->ah
, qpn
, addr
, skb
->len
))) {
355 ipoib_warn(priv
, "post_send failed\n");
356 ++priv
->stats
.tx_errors
;
357 dma_unmap_single(priv
->ca
->dma_device
, addr
, skb
->len
,
359 dev_kfree_skb_any(skb
);
361 dev
->trans_start
= jiffies
;
363 address
->last_send
= priv
->tx_head
;
366 if (priv
->tx_head
- priv
->tx_tail
== IPOIB_TX_RING_SIZE
) {
367 ipoib_dbg(priv
, "TX ring full, stopping kernel net queue\n");
368 netif_stop_queue(dev
);
373 static void __ipoib_reap_ah(struct net_device
*dev
)
375 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
376 struct ipoib_ah
*ah
, *tah
;
377 LIST_HEAD(remove_list
);
379 spin_lock_irq(&priv
->lock
);
380 list_for_each_entry_safe(ah
, tah
, &priv
->dead_ahs
, list
)
381 if ((int) priv
->tx_tail
- (int) ah
->last_send
>= 0) {
383 list_add_tail(&ah
->list
, &remove_list
);
385 spin_unlock_irq(&priv
->lock
);
387 list_for_each_entry_safe(ah
, tah
, &remove_list
, list
) {
388 ipoib_dbg(priv
, "Reaping ah %p\n", ah
->ah
);
389 ib_destroy_ah(ah
->ah
);
394 void ipoib_reap_ah(void *dev_ptr
)
396 struct net_device
*dev
= dev_ptr
;
397 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
399 __ipoib_reap_ah(dev
);
401 if (!test_bit(IPOIB_STOP_REAPER
, &priv
->flags
))
402 queue_delayed_work(ipoib_workqueue
, &priv
->ah_reap_task
, HZ
);
405 int ipoib_ib_dev_open(struct net_device
*dev
)
407 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
410 ret
= ipoib_init_qp(dev
);
412 ipoib_warn(priv
, "ipoib_init_qp returned %d\n", ret
);
416 ret
= ipoib_ib_post_receives(dev
);
418 ipoib_warn(priv
, "ipoib_ib_post_receives returned %d\n", ret
);
422 clear_bit(IPOIB_STOP_REAPER
, &priv
->flags
);
423 queue_delayed_work(ipoib_workqueue
, &priv
->ah_reap_task
, HZ
);
428 int ipoib_ib_dev_up(struct net_device
*dev
)
430 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
432 set_bit(IPOIB_FLAG_OPER_UP
, &priv
->flags
);
434 return ipoib_mcast_start_thread(dev
);
437 int ipoib_ib_dev_down(struct net_device
*dev
)
439 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
441 ipoib_dbg(priv
, "downing ib_dev\n");
443 clear_bit(IPOIB_FLAG_OPER_UP
, &priv
->flags
);
444 netif_carrier_off(dev
);
446 /* Shutdown the P_Key thread if still active */
447 if (!test_bit(IPOIB_PKEY_ASSIGNED
, &priv
->flags
)) {
449 set_bit(IPOIB_PKEY_STOP
, &priv
->flags
);
450 cancel_delayed_work(&priv
->pkey_task
);
452 flush_workqueue(ipoib_workqueue
);
455 ipoib_mcast_stop_thread(dev
, 1);
458 * Flush the multicast groups first so we stop any multicast joins. The
459 * completion thread may have already died and we may deadlock waiting
460 * for the completion thread to finish some multicast joins.
462 ipoib_mcast_dev_flush(dev
);
464 /* Delete broadcast and local addresses since they will be recreated */
465 ipoib_mcast_dev_down(dev
);
467 ipoib_flush_paths(dev
);
472 static int recvs_pending(struct net_device
*dev
)
474 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
478 for (i
= 0; i
< IPOIB_RX_RING_SIZE
; ++i
)
479 if (priv
->rx_ring
[i
].skb
)
485 int ipoib_ib_dev_stop(struct net_device
*dev
)
487 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
488 struct ib_qp_attr qp_attr
;
490 struct ipoib_tx_buf
*tx_req
;
494 * Move our QP to the error state and then reinitialize in
495 * when all work requests have completed or have been flushed.
497 qp_attr
.qp_state
= IB_QPS_ERR
;
498 if (ib_modify_qp(priv
->qp
, &qp_attr
, IB_QP_STATE
))
499 ipoib_warn(priv
, "Failed to modify QP to ERROR state\n");
501 /* Wait for all sends and receives to complete */
504 while (priv
->tx_head
!= priv
->tx_tail
|| recvs_pending(dev
)) {
505 if (time_after(jiffies
, begin
+ 5 * HZ
)) {
506 ipoib_warn(priv
, "timing out; %d sends %d receives not completed\n",
507 priv
->tx_head
- priv
->tx_tail
, recvs_pending(dev
));
510 * assume the HW is wedged and just free up
511 * all our pending work requests.
513 while ((int) priv
->tx_tail
- (int) priv
->tx_head
< 0) {
514 tx_req
= &priv
->tx_ring
[priv
->tx_tail
&
515 (IPOIB_TX_RING_SIZE
- 1)];
516 dma_unmap_single(priv
->ca
->dma_device
,
517 pci_unmap_addr(tx_req
, mapping
),
520 dev_kfree_skb_any(tx_req
->skb
);
524 for (i
= 0; i
< IPOIB_RX_RING_SIZE
; ++i
)
525 if (priv
->rx_ring
[i
].skb
) {
526 dma_unmap_single(priv
->ca
->dma_device
,
527 pci_unmap_addr(&priv
->rx_ring
[i
],
531 dev_kfree_skb_any(priv
->rx_ring
[i
].skb
);
532 priv
->rx_ring
[i
].skb
= NULL
;
541 ipoib_dbg(priv
, "All sends and receives done.\n");
544 qp_attr
.qp_state
= IB_QPS_RESET
;
545 if (ib_modify_qp(priv
->qp
, &qp_attr
, IB_QP_STATE
))
546 ipoib_warn(priv
, "Failed to modify QP to RESET state\n");
548 /* Wait for all AHs to be reaped */
549 set_bit(IPOIB_STOP_REAPER
, &priv
->flags
);
550 cancel_delayed_work(&priv
->ah_reap_task
);
551 flush_workqueue(ipoib_workqueue
);
555 while (!list_empty(&priv
->dead_ahs
)) {
556 __ipoib_reap_ah(dev
);
558 if (time_after(jiffies
, begin
+ HZ
)) {
559 ipoib_warn(priv
, "timing out; will leak address handles\n");
569 int ipoib_ib_dev_init(struct net_device
*dev
, struct ib_device
*ca
, int port
)
571 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
577 if (ipoib_transport_dev_init(dev
, ca
)) {
578 printk(KERN_WARNING
"%s: ipoib_transport_dev_init failed\n", ca
->name
);
582 if (dev
->flags
& IFF_UP
) {
583 if (ipoib_ib_dev_open(dev
)) {
584 ipoib_transport_dev_cleanup(dev
);
592 void ipoib_ib_dev_flush(void *_dev
)
594 struct net_device
*dev
= (struct net_device
*)_dev
;
595 struct ipoib_dev_priv
*priv
= netdev_priv(dev
), *cpriv
;
597 if (!test_bit(IPOIB_FLAG_ADMIN_UP
, &priv
->flags
))
600 ipoib_dbg(priv
, "flushing\n");
602 ipoib_ib_dev_down(dev
);
605 * The device could have been brought down between the start and when
606 * we get here, don't bring it back up if it's not configured up
608 if (test_bit(IPOIB_FLAG_ADMIN_UP
, &priv
->flags
))
609 ipoib_ib_dev_up(dev
);
611 /* Flush any child interfaces too */
612 list_for_each_entry(cpriv
, &priv
->child_intfs
, list
)
613 ipoib_ib_dev_flush(&cpriv
->dev
);
616 void ipoib_ib_dev_cleanup(struct net_device
*dev
)
618 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
620 ipoib_dbg(priv
, "cleaning up ib_dev\n");
622 ipoib_mcast_stop_thread(dev
, 1);
624 /* Delete the broadcast address and the local address */
625 ipoib_mcast_dev_down(dev
);
627 ipoib_transport_dev_cleanup(dev
);
631 * Delayed P_Key Assigment Interim Support
633 * The following is initial implementation of delayed P_Key assigment
634 * mechanism. It is using the same approach implemented for the multicast
635 * group join. The single goal of this implementation is to quickly address
636 * Bug #2507. This implementation will probably be removed when the P_Key
637 * change async notification is available.
639 int ipoib_open(struct net_device
*dev
);
641 static void ipoib_pkey_dev_check_presence(struct net_device
*dev
)
643 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
646 if (ib_find_cached_pkey(priv
->ca
, priv
->port
, priv
->pkey
, &pkey_index
))
647 clear_bit(IPOIB_PKEY_ASSIGNED
, &priv
->flags
);
649 set_bit(IPOIB_PKEY_ASSIGNED
, &priv
->flags
);
652 void ipoib_pkey_poll(void *dev_ptr
)
654 struct net_device
*dev
= dev_ptr
;
655 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
657 ipoib_pkey_dev_check_presence(dev
);
659 if (test_bit(IPOIB_PKEY_ASSIGNED
, &priv
->flags
))
663 if (!test_bit(IPOIB_PKEY_STOP
, &priv
->flags
))
664 queue_delayed_work(ipoib_workqueue
,
671 int ipoib_pkey_dev_delay_open(struct net_device
*dev
)
673 struct ipoib_dev_priv
*priv
= netdev_priv(dev
);
675 /* Look for the interface pkey value in the IB Port P_Key table and */
676 /* set the interface pkey assigment flag */
677 ipoib_pkey_dev_check_presence(dev
);
679 /* P_Key value not assigned yet - start polling */
680 if (!test_bit(IPOIB_PKEY_ASSIGNED
, &priv
->flags
)) {
682 clear_bit(IPOIB_PKEY_STOP
, &priv
->flags
);
683 queue_delayed_work(ipoib_workqueue
,