2 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/mlx4/cq.h>
35 #include <linux/mlx4/qp.h>
36 #include <linux/skbuff.h>
37 #include <linux/if_ether.h>
38 #include <linux/if_vlan.h>
39 #include <linux/vmalloc.h>
43 static void *get_wqe(struct mlx4_en_rx_ring
*ring
, int n
)
45 int offset
= n
<< ring
->srq
.wqe_shift
;
46 return ring
->buf
+ offset
;
49 static void mlx4_en_srq_event(struct mlx4_srq
*srq
, enum mlx4_event type
)
54 static int mlx4_en_get_frag_header(struct skb_frag_struct
*frags
, void **mac_hdr
,
55 void **ip_hdr
, void **tcpudp_hdr
,
56 u64
*hdr_flags
, void *priv
)
58 *mac_hdr
= page_address(frags
->page
) + frags
->page_offset
;
59 *ip_hdr
= *mac_hdr
+ ETH_HLEN
;
60 *tcpudp_hdr
= (struct tcphdr
*)(*ip_hdr
+ sizeof(struct iphdr
));
61 *hdr_flags
= LRO_IPV4
| LRO_TCP
;
66 static int mlx4_en_alloc_frag(struct mlx4_en_priv
*priv
,
67 struct mlx4_en_rx_desc
*rx_desc
,
68 struct skb_frag_struct
*skb_frags
,
69 struct mlx4_en_rx_alloc
*ring_alloc
,
72 struct mlx4_en_dev
*mdev
= priv
->mdev
;
73 struct mlx4_en_frag_info
*frag_info
= &priv
->frag_info
[i
];
74 struct mlx4_en_rx_alloc
*page_alloc
= &ring_alloc
[i
];
78 if (page_alloc
->offset
== frag_info
->last_offset
) {
79 /* Allocate new page */
80 page
= alloc_pages(GFP_ATOMIC
| __GFP_COMP
, MLX4_EN_ALLOC_ORDER
);
84 skb_frags
[i
].page
= page_alloc
->page
;
85 skb_frags
[i
].page_offset
= page_alloc
->offset
;
86 page_alloc
->page
= page
;
87 page_alloc
->offset
= frag_info
->frag_align
;
89 page
= page_alloc
->page
;
92 skb_frags
[i
].page
= page
;
93 skb_frags
[i
].page_offset
= page_alloc
->offset
;
94 page_alloc
->offset
+= frag_info
->frag_stride
;
96 dma
= pci_map_single(mdev
->pdev
, page_address(skb_frags
[i
].page
) +
97 skb_frags
[i
].page_offset
, frag_info
->frag_size
,
99 rx_desc
->data
[i
].addr
= cpu_to_be64(dma
);
103 static int mlx4_en_init_allocator(struct mlx4_en_priv
*priv
,
104 struct mlx4_en_rx_ring
*ring
)
106 struct mlx4_en_rx_alloc
*page_alloc
;
109 for (i
= 0; i
< priv
->num_frags
; i
++) {
110 page_alloc
= &ring
->page_alloc
[i
];
111 page_alloc
->page
= alloc_pages(GFP_ATOMIC
| __GFP_COMP
,
112 MLX4_EN_ALLOC_ORDER
);
113 if (!page_alloc
->page
)
116 page_alloc
->offset
= priv
->frag_info
[i
].frag_align
;
117 mlx4_dbg(DRV
, priv
, "Initialized allocator:%d with page:%p\n",
118 i
, page_alloc
->page
);
124 page_alloc
= &ring
->page_alloc
[i
];
125 put_page(page_alloc
->page
);
126 page_alloc
->page
= NULL
;
131 static void mlx4_en_destroy_allocator(struct mlx4_en_priv
*priv
,
132 struct mlx4_en_rx_ring
*ring
)
134 struct mlx4_en_rx_alloc
*page_alloc
;
137 for (i
= 0; i
< priv
->num_frags
; i
++) {
138 page_alloc
= &ring
->page_alloc
[i
];
139 mlx4_dbg(DRV
, priv
, "Freeing allocator:%d count:%d\n",
140 i
, page_count(page_alloc
->page
));
142 put_page(page_alloc
->page
);
143 page_alloc
->page
= NULL
;
148 static void mlx4_en_init_rx_desc(struct mlx4_en_priv
*priv
,
149 struct mlx4_en_rx_ring
*ring
, int index
)
151 struct mlx4_en_rx_desc
*rx_desc
= ring
->buf
+ ring
->stride
* index
;
152 struct skb_frag_struct
*skb_frags
= ring
->rx_info
+
153 (index
<< priv
->log_rx_info
);
157 /* Pre-link descriptor */
158 rx_desc
->next
.next_wqe_index
= cpu_to_be16((index
+ 1) & ring
->size_mask
);
160 /* Set size and memtype fields */
161 for (i
= 0; i
< priv
->num_frags
; i
++) {
162 skb_frags
[i
].size
= priv
->frag_info
[i
].frag_size
;
163 rx_desc
->data
[i
].byte_count
=
164 cpu_to_be32(priv
->frag_info
[i
].frag_size
);
165 rx_desc
->data
[i
].lkey
= cpu_to_be32(priv
->mdev
->mr
.key
);
168 /* If the number of used fragments does not fill up the ring stride,
169 * remaining (unused) fragments must be padded with null address/size
170 * and a special memory key */
171 possible_frags
= (ring
->stride
- sizeof(struct mlx4_en_rx_desc
)) / DS_SIZE
;
172 for (i
= priv
->num_frags
; i
< possible_frags
; i
++) {
173 rx_desc
->data
[i
].byte_count
= 0;
174 rx_desc
->data
[i
].lkey
= cpu_to_be32(MLX4_EN_MEMTYPE_PAD
);
175 rx_desc
->data
[i
].addr
= 0;
180 static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv
*priv
,
181 struct mlx4_en_rx_ring
*ring
, int index
)
183 struct mlx4_en_rx_desc
*rx_desc
= ring
->buf
+ (index
* ring
->stride
);
184 struct skb_frag_struct
*skb_frags
= ring
->rx_info
+
185 (index
<< priv
->log_rx_info
);
188 for (i
= 0; i
< priv
->num_frags
; i
++)
189 if (mlx4_en_alloc_frag(priv
, rx_desc
, skb_frags
, ring
->page_alloc
, i
))
196 put_page(skb_frags
[i
].page
);
200 static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring
*ring
)
202 *ring
->wqres
.db
.db
= cpu_to_be32(ring
->prod
& 0xffff);
205 static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv
*priv
)
207 struct mlx4_en_dev
*mdev
= priv
->mdev
;
208 struct mlx4_en_rx_ring
*ring
;
212 for (buf_ind
= 0; buf_ind
< priv
->prof
->rx_ring_size
; buf_ind
++) {
213 for (ring_ind
= 0; ring_ind
< priv
->rx_ring_num
; ring_ind
++) {
214 ring
= &priv
->rx_ring
[ring_ind
];
216 if (mlx4_en_prepare_rx_desc(priv
, ring
,
217 ring
->actual_size
)) {
218 if (ring
->actual_size
< MLX4_EN_MIN_RX_SIZE
) {
219 mlx4_err(mdev
, "Failed to allocate "
220 "enough rx buffers\n");
223 if (netif_msg_rx_err(priv
))
225 "Only %d buffers allocated\n",
238 static int mlx4_en_fill_rx_buf(struct net_device
*dev
,
239 struct mlx4_en_rx_ring
*ring
)
241 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
245 while ((u32
) (ring
->prod
- ring
->cons
) < ring
->actual_size
) {
246 err
= mlx4_en_prepare_rx_desc(priv
, ring
, ring
->prod
&
249 if (netif_msg_rx_err(priv
))
250 mlx4_warn(priv
->mdev
,
251 "Failed preparing rx descriptor\n");
252 priv
->port_stats
.rx_alloc_failed
++;
258 if ((u32
) (ring
->prod
- ring
->cons
) == ring
->size
)
264 static void mlx4_en_free_rx_buf(struct mlx4_en_priv
*priv
,
265 struct mlx4_en_rx_ring
*ring
)
267 struct mlx4_en_dev
*mdev
= priv
->mdev
;
268 struct skb_frag_struct
*skb_frags
;
269 struct mlx4_en_rx_desc
*rx_desc
;
274 mlx4_dbg(DRV
, priv
, "Freeing Rx buf - cons:%d prod:%d\n",
275 ring
->cons
, ring
->prod
);
277 /* Unmap and free Rx buffers */
278 BUG_ON((u32
) (ring
->prod
- ring
->cons
) > ring
->size
);
279 while (ring
->cons
!= ring
->prod
) {
280 index
= ring
->cons
& ring
->size_mask
;
281 rx_desc
= ring
->buf
+ (index
<< ring
->log_stride
);
282 skb_frags
= ring
->rx_info
+ (index
<< priv
->log_rx_info
);
283 mlx4_dbg(DRV
, priv
, "Processing descriptor:%d\n", index
);
285 for (nr
= 0; nr
< priv
->num_frags
; nr
++) {
286 mlx4_dbg(DRV
, priv
, "Freeing fragment:%d\n", nr
);
287 dma
= be64_to_cpu(rx_desc
->data
[nr
].addr
);
289 mlx4_dbg(DRV
, priv
, "Unmaping buffer at dma:0x%llx\n", (u64
) dma
);
290 pci_unmap_single(mdev
->pdev
, dma
, skb_frags
[nr
].size
,
292 put_page(skb_frags
[nr
].page
);
299 void mlx4_en_rx_refill(struct work_struct
*work
)
301 struct delayed_work
*delay
= to_delayed_work(work
);
302 struct mlx4_en_priv
*priv
= container_of(delay
, struct mlx4_en_priv
,
304 struct mlx4_en_dev
*mdev
= priv
->mdev
;
305 struct net_device
*dev
= priv
->dev
;
306 struct mlx4_en_rx_ring
*ring
;
310 mutex_lock(&mdev
->state_lock
);
311 if (!mdev
->device_up
|| !priv
->port_up
)
314 /* We only get here if there are no receive buffers, so we can't race
315 * with Rx interrupts while filling buffers */
316 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
317 ring
= &priv
->rx_ring
[i
];
318 if (ring
->need_refill
) {
319 if (mlx4_en_fill_rx_buf(dev
, ring
)) {
320 ring
->need_refill
= 0;
321 mlx4_en_update_rx_prod_db(ring
);
327 queue_delayed_work(mdev
->workqueue
, &priv
->refill_task
, HZ
);
330 mutex_unlock(&mdev
->state_lock
);
334 int mlx4_en_create_rx_ring(struct mlx4_en_priv
*priv
,
335 struct mlx4_en_rx_ring
*ring
, u32 size
, u16 stride
)
337 struct mlx4_en_dev
*mdev
= priv
->mdev
;
341 /* Sanity check SRQ size before proceeding */
342 if (size
>= mdev
->dev
->caps
.max_srq_wqes
)
348 ring
->size_mask
= size
- 1;
349 ring
->stride
= stride
;
350 ring
->log_stride
= ffs(ring
->stride
) - 1;
351 ring
->buf_size
= ring
->size
* ring
->stride
;
353 tmp
= size
* roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS
*
354 sizeof(struct skb_frag_struct
));
355 ring
->rx_info
= vmalloc(tmp
);
356 if (!ring
->rx_info
) {
357 mlx4_err(mdev
, "Failed allocating rx_info ring\n");
360 mlx4_dbg(DRV
, priv
, "Allocated rx_info ring at addr:%p size:%d\n",
363 err
= mlx4_alloc_hwq_res(mdev
->dev
, &ring
->wqres
,
364 ring
->buf_size
, 2 * PAGE_SIZE
);
368 err
= mlx4_en_map_buffer(&ring
->wqres
.buf
);
370 mlx4_err(mdev
, "Failed to map RX buffer\n");
373 ring
->buf
= ring
->wqres
.buf
.direct
.buf
;
375 /* Configure lro mngr */
376 memset(&ring
->lro
, 0, sizeof(struct net_lro_mgr
));
377 ring
->lro
.dev
= priv
->dev
;
378 ring
->lro
.features
= LRO_F_NAPI
;
379 ring
->lro
.frag_align_pad
= NET_IP_ALIGN
;
380 ring
->lro
.ip_summed
= CHECKSUM_UNNECESSARY
;
381 ring
->lro
.ip_summed_aggr
= CHECKSUM_UNNECESSARY
;
382 ring
->lro
.max_desc
= mdev
->profile
.num_lro
;
383 ring
->lro
.max_aggr
= MAX_SKB_FRAGS
;
384 ring
->lro
.lro_arr
= kzalloc(mdev
->profile
.num_lro
*
385 sizeof(struct net_lro_desc
),
387 if (!ring
->lro
.lro_arr
) {
388 mlx4_err(mdev
, "Failed to allocate lro array\n");
391 ring
->lro
.get_frag_header
= mlx4_en_get_frag_header
;
396 mlx4_en_unmap_buffer(&ring
->wqres
.buf
);
398 mlx4_free_hwq_res(mdev
->dev
, &ring
->wqres
, ring
->buf_size
);
400 vfree(ring
->rx_info
);
401 ring
->rx_info
= NULL
;
405 int mlx4_en_activate_rx_rings(struct mlx4_en_priv
*priv
)
407 struct mlx4_en_dev
*mdev
= priv
->mdev
;
408 struct mlx4_wqe_srq_next_seg
*next
;
409 struct mlx4_en_rx_ring
*ring
;
413 int stride
= roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc
) +
414 DS_SIZE
* priv
->num_frags
);
415 int max_gs
= (stride
- sizeof(struct mlx4_wqe_srq_next_seg
)) / DS_SIZE
;
417 for (ring_ind
= 0; ring_ind
< priv
->rx_ring_num
; ring_ind
++) {
418 ring
= &priv
->rx_ring
[ring_ind
];
422 ring
->actual_size
= 0;
423 ring
->cqn
= priv
->rx_cq
[ring_ind
].mcq
.cqn
;
425 ring
->stride
= stride
;
426 ring
->log_stride
= ffs(ring
->stride
) - 1;
427 ring
->buf_size
= ring
->size
* ring
->stride
;
429 memset(ring
->buf
, 0, ring
->buf_size
);
430 mlx4_en_update_rx_prod_db(ring
);
432 /* Initailize all descriptors */
433 for (i
= 0; i
< ring
->size
; i
++)
434 mlx4_en_init_rx_desc(priv
, ring
, i
);
436 /* Initialize page allocators */
437 err
= mlx4_en_init_allocator(priv
, ring
);
439 mlx4_err(mdev
, "Failed initializing ring allocator\n");
444 /* Fill Rx buffers */
447 err
= mlx4_en_fill_rx_buffers(priv
);
451 for (ring_ind
= 0; ring_ind
< priv
->rx_ring_num
; ring_ind
++) {
452 ring
= &priv
->rx_ring
[ring_ind
];
454 mlx4_en_update_rx_prod_db(ring
);
456 /* Configure SRQ representing the ring */
457 ring
->srq
.max
= ring
->size
;
458 ring
->srq
.max_gs
= max_gs
;
459 ring
->srq
.wqe_shift
= ilog2(ring
->stride
);
461 for (i
= 0; i
< ring
->srq
.max
; ++i
) {
462 next
= get_wqe(ring
, i
);
463 next
->next_wqe_index
=
464 cpu_to_be16((i
+ 1) & (ring
->srq
.max
- 1));
467 err
= mlx4_srq_alloc(mdev
->dev
, mdev
->priv_pdn
, &ring
->wqres
.mtt
,
468 ring
->wqres
.db
.dma
, &ring
->srq
);
470 mlx4_err(mdev
, "Failed to allocate srq\n");
474 ring
->srq
.event
= mlx4_en_srq_event
;
480 while (ring_ind
>= 0) {
481 ring
= &priv
->rx_ring
[ring_ind
];
482 mlx4_srq_free(mdev
->dev
, &ring
->srq
);
487 for (ring_ind
= 0; ring_ind
< priv
->rx_ring_num
; ring_ind
++)
488 mlx4_en_free_rx_buf(priv
, &priv
->rx_ring
[ring_ind
]);
490 ring_ind
= priv
->rx_ring_num
- 1;
492 while (ring_ind
>= 0) {
493 mlx4_en_destroy_allocator(priv
, &priv
->rx_ring
[ring_ind
]);
499 void mlx4_en_destroy_rx_ring(struct mlx4_en_priv
*priv
,
500 struct mlx4_en_rx_ring
*ring
)
502 struct mlx4_en_dev
*mdev
= priv
->mdev
;
504 kfree(ring
->lro
.lro_arr
);
505 mlx4_en_unmap_buffer(&ring
->wqres
.buf
);
506 mlx4_free_hwq_res(mdev
->dev
, &ring
->wqres
, ring
->buf_size
);
507 vfree(ring
->rx_info
);
508 ring
->rx_info
= NULL
;
511 void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv
*priv
,
512 struct mlx4_en_rx_ring
*ring
)
514 struct mlx4_en_dev
*mdev
= priv
->mdev
;
516 mlx4_srq_free(mdev
->dev
, &ring
->srq
);
517 mlx4_en_free_rx_buf(priv
, ring
);
518 mlx4_en_destroy_allocator(priv
, ring
);
522 /* Unmap a completed descriptor and free unused pages */
523 static int mlx4_en_complete_rx_desc(struct mlx4_en_priv
*priv
,
524 struct mlx4_en_rx_desc
*rx_desc
,
525 struct skb_frag_struct
*skb_frags
,
526 struct skb_frag_struct
*skb_frags_rx
,
527 struct mlx4_en_rx_alloc
*page_alloc
,
530 struct mlx4_en_dev
*mdev
= priv
->mdev
;
531 struct mlx4_en_frag_info
*frag_info
;
535 /* Collect used fragments while replacing them in the HW descirptors */
536 for (nr
= 0; nr
< priv
->num_frags
; nr
++) {
537 frag_info
= &priv
->frag_info
[nr
];
538 if (length
<= frag_info
->frag_prefix_size
)
541 /* Save page reference in skb */
542 skb_frags_rx
[nr
].page
= skb_frags
[nr
].page
;
543 skb_frags_rx
[nr
].size
= skb_frags
[nr
].size
;
544 skb_frags_rx
[nr
].page_offset
= skb_frags
[nr
].page_offset
;
545 dma
= be64_to_cpu(rx_desc
->data
[nr
].addr
);
547 /* Allocate a replacement page */
548 if (mlx4_en_alloc_frag(priv
, rx_desc
, skb_frags
, page_alloc
, nr
))
552 pci_unmap_single(mdev
->pdev
, dma
, skb_frags
[nr
].size
,
555 /* Adjust size of last fragment to match actual length */
556 skb_frags_rx
[nr
- 1].size
= length
-
557 priv
->frag_info
[nr
- 1].frag_prefix_size
;
561 /* Drop all accumulated fragments (which have already been replaced in
562 * the descriptor) of this packet; remaining fragments are reused... */
565 put_page(skb_frags_rx
[nr
].page
);
571 static struct sk_buff
*mlx4_en_rx_skb(struct mlx4_en_priv
*priv
,
572 struct mlx4_en_rx_desc
*rx_desc
,
573 struct skb_frag_struct
*skb_frags
,
574 struct mlx4_en_rx_alloc
*page_alloc
,
577 struct mlx4_en_dev
*mdev
= priv
->mdev
;
583 skb
= dev_alloc_skb(SMALL_PACKET_SIZE
+ NET_IP_ALIGN
);
585 mlx4_dbg(RX_ERR
, priv
, "Failed allocating skb\n");
588 skb
->dev
= priv
->dev
;
589 skb_reserve(skb
, NET_IP_ALIGN
);
591 skb
->truesize
= length
+ sizeof(struct sk_buff
);
593 /* Get pointer to first fragment so we could copy the headers into the
594 * (linear part of the) skb */
595 va
= page_address(skb_frags
[0].page
) + skb_frags
[0].page_offset
;
597 if (length
<= SMALL_PACKET_SIZE
) {
598 /* We are copying all relevant data to the skb - temporarily
599 * synch buffers for the copy */
600 dma
= be64_to_cpu(rx_desc
->data
[0].addr
);
601 dma_sync_single_range_for_cpu(&mdev
->pdev
->dev
, dma
, 0,
602 length
, DMA_FROM_DEVICE
);
603 skb_copy_to_linear_data(skb
, va
, length
);
604 dma_sync_single_range_for_device(&mdev
->pdev
->dev
, dma
, 0,
605 length
, DMA_FROM_DEVICE
);
609 /* Move relevant fragments to skb */
610 used_frags
= mlx4_en_complete_rx_desc(priv
, rx_desc
, skb_frags
,
611 skb_shinfo(skb
)->frags
,
613 skb_shinfo(skb
)->nr_frags
= used_frags
;
615 /* Copy headers into the skb linear buffer */
616 memcpy(skb
->data
, va
, HEADER_COPY_SIZE
);
617 skb
->tail
+= HEADER_COPY_SIZE
;
619 /* Skip headers in first fragment */
620 skb_shinfo(skb
)->frags
[0].page_offset
+= HEADER_COPY_SIZE
;
622 /* Adjust size of first fragment */
623 skb_shinfo(skb
)->frags
[0].size
-= HEADER_COPY_SIZE
;
624 skb
->data_len
= length
- HEADER_COPY_SIZE
;
629 static void mlx4_en_copy_desc(struct mlx4_en_priv
*priv
,
630 struct mlx4_en_rx_ring
*ring
,
631 int from
, int to
, int num
)
633 struct skb_frag_struct
*skb_frags_from
;
634 struct skb_frag_struct
*skb_frags_to
;
635 struct mlx4_en_rx_desc
*rx_desc_from
;
636 struct mlx4_en_rx_desc
*rx_desc_to
;
637 int from_index
, to_index
;
640 for (i
= 0; i
< num
; i
++) {
641 from_index
= (from
+ i
) & ring
->size_mask
;
642 to_index
= (to
+ i
) & ring
->size_mask
;
643 skb_frags_from
= ring
->rx_info
+ (from_index
<< priv
->log_rx_info
);
644 skb_frags_to
= ring
->rx_info
+ (to_index
<< priv
->log_rx_info
);
645 rx_desc_from
= ring
->buf
+ (from_index
<< ring
->log_stride
);
646 rx_desc_to
= ring
->buf
+ (to_index
<< ring
->log_stride
);
648 for (nr
= 0; nr
< priv
->num_frags
; nr
++) {
649 skb_frags_to
[nr
].page
= skb_frags_from
[nr
].page
;
650 skb_frags_to
[nr
].page_offset
= skb_frags_from
[nr
].page_offset
;
651 rx_desc_to
->data
[nr
].addr
= rx_desc_from
->data
[nr
].addr
;
657 int mlx4_en_process_rx_cq(struct net_device
*dev
, struct mlx4_en_cq
*cq
, int budget
)
659 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
660 struct mlx4_en_dev
*mdev
= priv
->mdev
;
661 struct mlx4_cqe
*cqe
;
662 struct mlx4_en_rx_ring
*ring
= &priv
->rx_ring
[cq
->ring
];
663 struct skb_frag_struct
*skb_frags
;
664 struct skb_frag_struct lro_frags
[MLX4_EN_MAX_RX_FRAGS
];
665 struct mlx4_en_rx_desc
*rx_desc
;
676 /* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx
677 * descriptor offset can be deduced from the CQE index instead of
678 * reading 'cqe->index' */
679 index
= cq
->mcq
.cons_index
& ring
->size_mask
;
680 cqe
= &cq
->buf
[index
];
682 /* Process all completed CQEs */
683 while (XNOR(cqe
->owner_sr_opcode
& MLX4_CQE_OWNER_MASK
,
684 cq
->mcq
.cons_index
& cq
->size
)) {
686 skb_frags
= ring
->rx_info
+ (index
<< priv
->log_rx_info
);
687 rx_desc
= ring
->buf
+ (index
<< ring
->log_stride
);
690 * make sure we read the CQE after we read the ownership bit
694 /* Drop packet on bad receive or bad checksum */
695 if (unlikely((cqe
->owner_sr_opcode
& MLX4_CQE_OPCODE_MASK
) ==
696 MLX4_CQE_OPCODE_ERROR
)) {
697 mlx4_err(mdev
, "CQE completed in error - vendor "
698 "syndrom:%d syndrom:%d\n",
699 ((struct mlx4_err_cqe
*) cqe
)->vendor_err_syndrome
,
700 ((struct mlx4_err_cqe
*) cqe
)->syndrome
);
703 if (unlikely(cqe
->badfcs_enc
& MLX4_CQE_BAD_FCS
)) {
704 mlx4_dbg(RX_ERR
, priv
, "Accepted frame with bad FCS\n");
709 * Packet is OK - process it.
711 length
= be32_to_cpu(cqe
->byte_cnt
);
712 ring
->bytes
+= length
;
715 if (likely(priv
->rx_csum
)) {
716 if ((cqe
->status
& cpu_to_be16(MLX4_CQE_STATUS_IPOK
)) &&
717 (cqe
->checksum
== cpu_to_be16(0xffff))) {
718 priv
->port_stats
.rx_chksum_good
++;
719 /* This packet is eligible for LRO if it is:
720 * - DIX Ethernet (type interpretation)
722 * - without IP options
723 * - not an IP fragment */
724 if (mlx4_en_can_lro(cqe
->status
) &&
725 dev
->features
& NETIF_F_LRO
) {
727 nr
= mlx4_en_complete_rx_desc(
729 skb_frags
, lro_frags
,
730 ring
->page_alloc
, length
);
734 if (priv
->vlgrp
&& (cqe
->vlan_my_qpn
&
735 cpu_to_be32(MLX4_CQE_VLAN_PRESENT_MASK
))) {
736 lro_vlan_hwaccel_receive_frags(
737 &ring
->lro
, lro_frags
,
740 be16_to_cpu(cqe
->sl_vid
),
743 lro_receive_frags(&ring
->lro
,
752 /* LRO not possible, complete processing here */
753 ip_summed
= CHECKSUM_UNNECESSARY
;
754 INC_PERF_COUNTER(priv
->pstats
.lro_misses
);
756 ip_summed
= CHECKSUM_NONE
;
757 priv
->port_stats
.rx_chksum_none
++;
760 ip_summed
= CHECKSUM_NONE
;
761 priv
->port_stats
.rx_chksum_none
++;
764 skb
= mlx4_en_rx_skb(priv
, rx_desc
, skb_frags
,
765 ring
->page_alloc
, length
);
767 priv
->stats
.rx_dropped
++;
771 skb
->ip_summed
= ip_summed
;
772 skb
->protocol
= eth_type_trans(skb
, dev
);
773 skb_record_rx_queue(skb
, cq
->ring
);
775 /* Push it up the stack */
776 if (priv
->vlgrp
&& (be32_to_cpu(cqe
->vlan_my_qpn
) &
777 MLX4_CQE_VLAN_PRESENT_MASK
)) {
778 vlan_hwaccel_receive_skb(skb
, priv
->vlgrp
,
779 be16_to_cpu(cqe
->sl_vid
));
781 netif_receive_skb(skb
);
784 ++cq
->mcq
.cons_index
;
785 index
= (cq
->mcq
.cons_index
) & ring
->size_mask
;
786 cqe
= &cq
->buf
[index
];
787 if (++polled
== budget
) {
788 /* We are here because we reached the NAPI budget -
789 * flush only pending LRO sessions */
790 lro_flush_all(&ring
->lro
);
795 /* If CQ is empty flush all LRO sessions unconditionally */
796 lro_flush_all(&ring
->lro
);
799 AVG_PERF_COUNTER(priv
->pstats
.rx_coal_avg
, polled
);
800 mlx4_cq_set_ci(&cq
->mcq
);
801 wmb(); /* ensure HW sees CQ consumer before we post new buffers */
802 ring
->cons
= cq
->mcq
.cons_index
;
803 ring
->prod
+= polled
; /* Polled descriptors were realocated in place */
804 if (unlikely(!ring
->full
)) {
805 mlx4_en_copy_desc(priv
, ring
, ring
->cons
- polled
,
806 ring
->prod
- polled
, polled
);
807 mlx4_en_fill_rx_buf(dev
, ring
);
809 mlx4_en_update_rx_prod_db(ring
);
814 void mlx4_en_rx_irq(struct mlx4_cq
*mcq
)
816 struct mlx4_en_cq
*cq
= container_of(mcq
, struct mlx4_en_cq
, mcq
);
817 struct mlx4_en_priv
*priv
= netdev_priv(cq
->dev
);
820 napi_schedule(&cq
->napi
);
822 mlx4_en_arm_cq(priv
, cq
);
825 /* Rx CQ polling - called by NAPI */
826 int mlx4_en_poll_rx_cq(struct napi_struct
*napi
, int budget
)
828 struct mlx4_en_cq
*cq
= container_of(napi
, struct mlx4_en_cq
, napi
);
829 struct net_device
*dev
= cq
->dev
;
830 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
833 done
= mlx4_en_process_rx_cq(dev
, cq
, budget
);
835 /* If we used up all the quota - we're probably not done yet... */
837 INC_PERF_COUNTER(priv
->pstats
.napi_quota
);
841 mlx4_en_arm_cq(priv
, cq
);
847 /* Calculate the last offset position that accomodates a full fragment
848 * (assuming fagment size = stride-align) */
849 static int mlx4_en_last_alloc_offset(struct mlx4_en_priv
*priv
, u16 stride
, u16 align
)
851 u16 res
= MLX4_EN_ALLOC_SIZE
% stride
;
852 u16 offset
= MLX4_EN_ALLOC_SIZE
- stride
- res
+ align
;
854 mlx4_dbg(DRV
, priv
, "Calculated last offset for stride:%d align:%d "
855 "res:%d offset:%d\n", stride
, align
, res
, offset
);
860 static int frag_sizes
[] = {
867 void mlx4_en_calc_rx_buf(struct net_device
*dev
)
869 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
870 int eff_mtu
= dev
->mtu
+ ETH_HLEN
+ VLAN_HLEN
+ ETH_LLC_SNAP_SIZE
;
874 while (buf_size
< eff_mtu
) {
875 priv
->frag_info
[i
].frag_size
=
876 (eff_mtu
> buf_size
+ frag_sizes
[i
]) ?
877 frag_sizes
[i
] : eff_mtu
- buf_size
;
878 priv
->frag_info
[i
].frag_prefix_size
= buf_size
;
880 priv
->frag_info
[i
].frag_align
= NET_IP_ALIGN
;
881 priv
->frag_info
[i
].frag_stride
=
882 ALIGN(frag_sizes
[i
] + NET_IP_ALIGN
, SMP_CACHE_BYTES
);
884 priv
->frag_info
[i
].frag_align
= 0;
885 priv
->frag_info
[i
].frag_stride
=
886 ALIGN(frag_sizes
[i
], SMP_CACHE_BYTES
);
888 priv
->frag_info
[i
].last_offset
= mlx4_en_last_alloc_offset(
889 priv
, priv
->frag_info
[i
].frag_stride
,
890 priv
->frag_info
[i
].frag_align
);
891 buf_size
+= priv
->frag_info
[i
].frag_size
;
896 priv
->rx_skb_size
= eff_mtu
;
897 priv
->log_rx_info
= ROUNDUP_LOG2(i
* sizeof(struct skb_frag_struct
));
899 mlx4_dbg(DRV
, priv
, "Rx buffer scatter-list (effective-mtu:%d "
900 "num_frags:%d):\n", eff_mtu
, priv
->num_frags
);
901 for (i
= 0; i
< priv
->num_frags
; i
++) {
902 mlx4_dbg(DRV
, priv
, " frag:%d - size:%d prefix:%d align:%d "
903 "stride:%d last_offset:%d\n", i
,
904 priv
->frag_info
[i
].frag_size
,
905 priv
->frag_info
[i
].frag_prefix_size
,
906 priv
->frag_info
[i
].frag_align
,
907 priv
->frag_info
[i
].frag_stride
,
908 priv
->frag_info
[i
].last_offset
);
912 /* RSS related functions */
914 /* Calculate rss size and map each entry in rss table to rx ring */
915 void mlx4_en_set_default_rss_map(struct mlx4_en_priv
*priv
,
916 struct mlx4_en_rss_map
*rss_map
,
917 int num_entries
, int num_rings
)
921 rss_map
->size
= roundup_pow_of_two(num_entries
);
922 mlx4_dbg(DRV
, priv
, "Setting default RSS map of %d entires\n",
925 for (i
= 0; i
< rss_map
->size
; i
++) {
926 rss_map
->map
[i
] = i
% num_rings
;
927 mlx4_dbg(DRV
, priv
, "Entry %d ---> ring %d\n", i
, rss_map
->map
[i
]);
931 static int mlx4_en_config_rss_qp(struct mlx4_en_priv
*priv
,
932 int qpn
, int srqn
, int cqn
,
933 enum mlx4_qp_state
*state
,
936 struct mlx4_en_dev
*mdev
= priv
->mdev
;
937 struct mlx4_qp_context
*context
;
940 context
= kmalloc(sizeof *context
, GFP_KERNEL
);
942 mlx4_err(mdev
, "Failed to allocate qp context\n");
946 err
= mlx4_qp_alloc(mdev
->dev
, qpn
, qp
);
948 mlx4_err(mdev
, "Failed to allocate qp #%d\n", qpn
);
952 qp
->event
= mlx4_en_sqp_event
;
954 memset(context
, 0, sizeof *context
);
955 mlx4_en_fill_qp_context(priv
, 0, 0, 0, 0, qpn
, cqn
, srqn
, context
);
957 err
= mlx4_qp_to_ready(mdev
->dev
, &priv
->res
.mtt
, context
, qp
, state
);
959 mlx4_qp_remove(mdev
->dev
, qp
);
960 mlx4_qp_free(mdev
->dev
, qp
);
967 /* Allocate rx qp's and configure them according to rss map */
968 int mlx4_en_config_rss_steer(struct mlx4_en_priv
*priv
)
970 struct mlx4_en_dev
*mdev
= priv
->mdev
;
971 struct mlx4_en_rss_map
*rss_map
= &priv
->rss_map
;
972 struct mlx4_qp_context context
;
973 struct mlx4_en_rss_context
*rss_context
;
975 int rss_xor
= mdev
->profile
.rss_xor
;
976 u8 rss_mask
= mdev
->profile
.rss_mask
;
977 int i
, srqn
, qpn
, cqn
;
981 mlx4_dbg(DRV
, priv
, "Configuring rss steering for port %u\n", priv
->port
);
982 err
= mlx4_qp_reserve_range(mdev
->dev
, rss_map
->size
,
983 rss_map
->size
, &rss_map
->base_qpn
);
985 mlx4_err(mdev
, "Failed reserving %d qps for port %u\n",
986 rss_map
->size
, priv
->port
);
990 for (i
= 0; i
< rss_map
->size
; i
++) {
991 cqn
= priv
->rx_ring
[rss_map
->map
[i
]].cqn
;
992 srqn
= priv
->rx_ring
[rss_map
->map
[i
]].srq
.srqn
;
993 qpn
= rss_map
->base_qpn
+ i
;
994 err
= mlx4_en_config_rss_qp(priv
, qpn
, srqn
, cqn
,
1003 /* Configure RSS indirection qp */
1004 err
= mlx4_qp_reserve_range(mdev
->dev
, 1, 1, &priv
->base_qpn
);
1006 mlx4_err(mdev
, "Failed to reserve range for RSS "
1007 "indirection qp\n");
1010 err
= mlx4_qp_alloc(mdev
->dev
, priv
->base_qpn
, &rss_map
->indir_qp
);
1012 mlx4_err(mdev
, "Failed to allocate RSS indirection QP\n");
1015 rss_map
->indir_qp
.event
= mlx4_en_sqp_event
;
1016 mlx4_en_fill_qp_context(priv
, 0, 0, 0, 1, priv
->base_qpn
,
1017 priv
->rx_ring
[0].cqn
, 0, &context
);
1019 ptr
= ((void *) &context
) + 0x3c;
1020 rss_context
= (struct mlx4_en_rss_context
*) ptr
;
1021 rss_context
->base_qpn
= cpu_to_be32(ilog2(rss_map
->size
) << 24 |
1022 (rss_map
->base_qpn
));
1023 rss_context
->default_qpn
= cpu_to_be32(rss_map
->base_qpn
);
1024 rss_context
->hash_fn
= rss_xor
& 0x3;
1025 rss_context
->flags
= rss_mask
<< 2;
1027 err
= mlx4_qp_to_ready(mdev
->dev
, &priv
->res
.mtt
, &context
,
1028 &rss_map
->indir_qp
, &rss_map
->indir_state
);
1035 mlx4_qp_modify(mdev
->dev
, NULL
, rss_map
->indir_state
,
1036 MLX4_QP_STATE_RST
, NULL
, 0, 0, &rss_map
->indir_qp
);
1037 mlx4_qp_remove(mdev
->dev
, &rss_map
->indir_qp
);
1038 mlx4_qp_free(mdev
->dev
, &rss_map
->indir_qp
);
1040 mlx4_qp_release_range(mdev
->dev
, priv
->base_qpn
, 1);
1042 for (i
= 0; i
< good_qps
; i
++) {
1043 mlx4_qp_modify(mdev
->dev
, NULL
, rss_map
->state
[i
],
1044 MLX4_QP_STATE_RST
, NULL
, 0, 0, &rss_map
->qps
[i
]);
1045 mlx4_qp_remove(mdev
->dev
, &rss_map
->qps
[i
]);
1046 mlx4_qp_free(mdev
->dev
, &rss_map
->qps
[i
]);
1048 mlx4_qp_release_range(mdev
->dev
, rss_map
->base_qpn
, rss_map
->size
);
1052 void mlx4_en_release_rss_steer(struct mlx4_en_priv
*priv
)
1054 struct mlx4_en_dev
*mdev
= priv
->mdev
;
1055 struct mlx4_en_rss_map
*rss_map
= &priv
->rss_map
;
1058 mlx4_qp_modify(mdev
->dev
, NULL
, rss_map
->indir_state
,
1059 MLX4_QP_STATE_RST
, NULL
, 0, 0, &rss_map
->indir_qp
);
1060 mlx4_qp_remove(mdev
->dev
, &rss_map
->indir_qp
);
1061 mlx4_qp_free(mdev
->dev
, &rss_map
->indir_qp
);
1062 mlx4_qp_release_range(mdev
->dev
, priv
->base_qpn
, 1);
1064 for (i
= 0; i
< rss_map
->size
; i
++) {
1065 mlx4_qp_modify(mdev
->dev
, NULL
, rss_map
->state
[i
],
1066 MLX4_QP_STATE_RST
, NULL
, 0, 0, &rss_map
->qps
[i
]);
1067 mlx4_qp_remove(mdev
->dev
, &rss_map
->qps
[i
]);
1068 mlx4_qp_free(mdev
->dev
, &rss_map
->qps
[i
]);
1070 mlx4_qp_release_range(mdev
->dev
, rss_map
->base_qpn
, rss_map
->size
);