2 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/mlx4/cq.h>
35 #include <linux/mlx4/qp.h>
36 #include <linux/skbuff.h>
37 #include <linux/if_ether.h>
38 #include <linux/if_vlan.h>
39 #include <linux/vmalloc.h>
43 static void *get_wqe(struct mlx4_en_rx_ring
*ring
, int n
)
45 int offset
= n
<< ring
->srq
.wqe_shift
;
46 return ring
->buf
+ offset
;
49 static void mlx4_en_srq_event(struct mlx4_srq
*srq
, enum mlx4_event type
)
54 static int mlx4_en_get_frag_header(struct skb_frag_struct
*frags
, void **mac_hdr
,
55 void **ip_hdr
, void **tcpudp_hdr
,
56 u64
*hdr_flags
, void *priv
)
58 *mac_hdr
= page_address(frags
->page
) + frags
->page_offset
;
59 *ip_hdr
= *mac_hdr
+ ETH_HLEN
;
60 *tcpudp_hdr
= (struct tcphdr
*)(*ip_hdr
+ sizeof(struct iphdr
));
61 *hdr_flags
= LRO_IPV4
| LRO_TCP
;
66 static int mlx4_en_alloc_frag(struct mlx4_en_priv
*priv
,
67 struct mlx4_en_rx_desc
*rx_desc
,
68 struct skb_frag_struct
*skb_frags
,
69 struct mlx4_en_rx_alloc
*ring_alloc
,
72 struct mlx4_en_dev
*mdev
= priv
->mdev
;
73 struct mlx4_en_frag_info
*frag_info
= &priv
->frag_info
[i
];
74 struct mlx4_en_rx_alloc
*page_alloc
= &ring_alloc
[i
];
78 if (page_alloc
->offset
== frag_info
->last_offset
) {
79 /* Allocate new page */
80 page
= alloc_pages(GFP_ATOMIC
| __GFP_COMP
, MLX4_EN_ALLOC_ORDER
);
84 skb_frags
[i
].page
= page_alloc
->page
;
85 skb_frags
[i
].page_offset
= page_alloc
->offset
;
86 page_alloc
->page
= page
;
87 page_alloc
->offset
= frag_info
->frag_align
;
89 page
= page_alloc
->page
;
92 skb_frags
[i
].page
= page
;
93 skb_frags
[i
].page_offset
= page_alloc
->offset
;
94 page_alloc
->offset
+= frag_info
->frag_stride
;
96 dma
= pci_map_single(mdev
->pdev
, page_address(skb_frags
[i
].page
) +
97 skb_frags
[i
].page_offset
, frag_info
->frag_size
,
99 rx_desc
->data
[i
].addr
= cpu_to_be64(dma
);
103 static int mlx4_en_init_allocator(struct mlx4_en_priv
*priv
,
104 struct mlx4_en_rx_ring
*ring
)
106 struct mlx4_en_rx_alloc
*page_alloc
;
109 for (i
= 0; i
< priv
->num_frags
; i
++) {
110 page_alloc
= &ring
->page_alloc
[i
];
111 page_alloc
->page
= alloc_pages(GFP_ATOMIC
| __GFP_COMP
,
112 MLX4_EN_ALLOC_ORDER
);
113 if (!page_alloc
->page
)
116 page_alloc
->offset
= priv
->frag_info
[i
].frag_align
;
117 en_dbg(DRV
, priv
, "Initialized allocator:%d with page:%p\n",
118 i
, page_alloc
->page
);
124 page_alloc
= &ring
->page_alloc
[i
];
125 put_page(page_alloc
->page
);
126 page_alloc
->page
= NULL
;
131 static void mlx4_en_destroy_allocator(struct mlx4_en_priv
*priv
,
132 struct mlx4_en_rx_ring
*ring
)
134 struct mlx4_en_rx_alloc
*page_alloc
;
137 for (i
= 0; i
< priv
->num_frags
; i
++) {
138 page_alloc
= &ring
->page_alloc
[i
];
139 en_dbg(DRV
, priv
, "Freeing allocator:%d count:%d\n",
140 i
, page_count(page_alloc
->page
));
142 put_page(page_alloc
->page
);
143 page_alloc
->page
= NULL
;
148 static void mlx4_en_init_rx_desc(struct mlx4_en_priv
*priv
,
149 struct mlx4_en_rx_ring
*ring
, int index
)
151 struct mlx4_en_rx_desc
*rx_desc
= ring
->buf
+ ring
->stride
* index
;
152 struct skb_frag_struct
*skb_frags
= ring
->rx_info
+
153 (index
<< priv
->log_rx_info
);
157 /* Pre-link descriptor */
158 rx_desc
->next
.next_wqe_index
= cpu_to_be16((index
+ 1) & ring
->size_mask
);
160 /* Set size and memtype fields */
161 for (i
= 0; i
< priv
->num_frags
; i
++) {
162 skb_frags
[i
].size
= priv
->frag_info
[i
].frag_size
;
163 rx_desc
->data
[i
].byte_count
=
164 cpu_to_be32(priv
->frag_info
[i
].frag_size
);
165 rx_desc
->data
[i
].lkey
= cpu_to_be32(priv
->mdev
->mr
.key
);
168 /* If the number of used fragments does not fill up the ring stride,
169 * remaining (unused) fragments must be padded with null address/size
170 * and a special memory key */
171 possible_frags
= (ring
->stride
- sizeof(struct mlx4_en_rx_desc
)) / DS_SIZE
;
172 for (i
= priv
->num_frags
; i
< possible_frags
; i
++) {
173 rx_desc
->data
[i
].byte_count
= 0;
174 rx_desc
->data
[i
].lkey
= cpu_to_be32(MLX4_EN_MEMTYPE_PAD
);
175 rx_desc
->data
[i
].addr
= 0;
180 static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv
*priv
,
181 struct mlx4_en_rx_ring
*ring
, int index
)
183 struct mlx4_en_rx_desc
*rx_desc
= ring
->buf
+ (index
* ring
->stride
);
184 struct skb_frag_struct
*skb_frags
= ring
->rx_info
+
185 (index
<< priv
->log_rx_info
);
188 for (i
= 0; i
< priv
->num_frags
; i
++)
189 if (mlx4_en_alloc_frag(priv
, rx_desc
, skb_frags
, ring
->page_alloc
, i
))
196 put_page(skb_frags
[i
].page
);
200 static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring
*ring
)
202 *ring
->wqres
.db
.db
= cpu_to_be32(ring
->prod
& 0xffff);
205 static void mlx4_en_free_rx_desc(struct mlx4_en_priv
*priv
,
206 struct mlx4_en_rx_ring
*ring
,
209 struct mlx4_en_dev
*mdev
= priv
->mdev
;
210 struct skb_frag_struct
*skb_frags
;
211 struct mlx4_en_rx_desc
*rx_desc
= ring
->buf
+ (index
<< ring
->log_stride
);
215 skb_frags
= ring
->rx_info
+ (index
<< priv
->log_rx_info
);
216 for (nr
= 0; nr
< priv
->num_frags
; nr
++) {
217 en_dbg(DRV
, priv
, "Freeing fragment:%d\n", nr
);
218 dma
= be64_to_cpu(rx_desc
->data
[nr
].addr
);
220 en_dbg(DRV
, priv
, "Unmaping buffer at dma:0x%llx\n", (u64
) dma
);
221 pci_unmap_single(mdev
->pdev
, dma
, skb_frags
[nr
].size
,
223 put_page(skb_frags
[nr
].page
);
227 static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv
*priv
)
229 struct mlx4_en_rx_ring
*ring
;
234 for (buf_ind
= 0; buf_ind
< priv
->prof
->rx_ring_size
; buf_ind
++) {
235 for (ring_ind
= 0; ring_ind
< priv
->rx_ring_num
; ring_ind
++) {
236 ring
= &priv
->rx_ring
[ring_ind
];
238 if (mlx4_en_prepare_rx_desc(priv
, ring
,
239 ring
->actual_size
)) {
240 if (ring
->actual_size
< MLX4_EN_MIN_RX_SIZE
) {
241 en_err(priv
, "Failed to allocate "
242 "enough rx buffers\n");
245 new_size
= rounddown_pow_of_two(ring
->actual_size
);
246 en_warn(priv
, "Only %d buffers allocated "
247 "reducing ring size to %d",
248 ring
->actual_size
, new_size
);
259 for (ring_ind
= 0; ring_ind
< priv
->rx_ring_num
; ring_ind
++) {
260 ring
= &priv
->rx_ring
[ring_ind
];
261 while (ring
->actual_size
> new_size
) {
264 mlx4_en_free_rx_desc(priv
, ring
, ring
->actual_size
);
266 ring
->size_mask
= ring
->actual_size
- 1;
272 static void mlx4_en_free_rx_buf(struct mlx4_en_priv
*priv
,
273 struct mlx4_en_rx_ring
*ring
)
277 en_dbg(DRV
, priv
, "Freeing Rx buf - cons:%d prod:%d\n",
278 ring
->cons
, ring
->prod
);
280 /* Unmap and free Rx buffers */
281 BUG_ON((u32
) (ring
->prod
- ring
->cons
) > ring
->actual_size
);
282 while (ring
->cons
!= ring
->prod
) {
283 index
= ring
->cons
& ring
->size_mask
;
284 en_dbg(DRV
, priv
, "Processing descriptor:%d\n", index
);
285 mlx4_en_free_rx_desc(priv
, ring
, index
);
290 int mlx4_en_create_rx_ring(struct mlx4_en_priv
*priv
,
291 struct mlx4_en_rx_ring
*ring
, u32 size
, u16 stride
)
293 struct mlx4_en_dev
*mdev
= priv
->mdev
;
297 /* Sanity check SRQ size before proceeding */
298 if (size
>= mdev
->dev
->caps
.max_srq_wqes
)
304 ring
->size_mask
= size
- 1;
305 ring
->stride
= stride
;
306 ring
->log_stride
= ffs(ring
->stride
) - 1;
307 ring
->buf_size
= ring
->size
* ring
->stride
;
309 tmp
= size
* roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS
*
310 sizeof(struct skb_frag_struct
));
311 ring
->rx_info
= vmalloc(tmp
);
312 if (!ring
->rx_info
) {
313 en_err(priv
, "Failed allocating rx_info ring\n");
316 en_dbg(DRV
, priv
, "Allocated rx_info ring at addr:%p size:%d\n",
319 err
= mlx4_alloc_hwq_res(mdev
->dev
, &ring
->wqres
,
320 ring
->buf_size
, 2 * PAGE_SIZE
);
324 err
= mlx4_en_map_buffer(&ring
->wqres
.buf
);
326 en_err(priv
, "Failed to map RX buffer\n");
329 ring
->buf
= ring
->wqres
.buf
.direct
.buf
;
331 /* Configure lro mngr */
332 memset(&ring
->lro
, 0, sizeof(struct net_lro_mgr
));
333 ring
->lro
.dev
= priv
->dev
;
334 ring
->lro
.features
= LRO_F_NAPI
;
335 ring
->lro
.frag_align_pad
= NET_IP_ALIGN
;
336 ring
->lro
.ip_summed
= CHECKSUM_UNNECESSARY
;
337 ring
->lro
.ip_summed_aggr
= CHECKSUM_UNNECESSARY
;
338 ring
->lro
.max_desc
= mdev
->profile
.num_lro
;
339 ring
->lro
.max_aggr
= MAX_SKB_FRAGS
;
340 ring
->lro
.lro_arr
= kzalloc(mdev
->profile
.num_lro
*
341 sizeof(struct net_lro_desc
),
343 if (!ring
->lro
.lro_arr
) {
344 en_err(priv
, "Failed to allocate lro array\n");
347 ring
->lro
.get_frag_header
= mlx4_en_get_frag_header
;
352 mlx4_en_unmap_buffer(&ring
->wqres
.buf
);
354 mlx4_free_hwq_res(mdev
->dev
, &ring
->wqres
, ring
->buf_size
);
356 vfree(ring
->rx_info
);
357 ring
->rx_info
= NULL
;
361 int mlx4_en_activate_rx_rings(struct mlx4_en_priv
*priv
)
363 struct mlx4_en_dev
*mdev
= priv
->mdev
;
364 struct mlx4_wqe_srq_next_seg
*next
;
365 struct mlx4_en_rx_ring
*ring
;
369 int stride
= roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc
) +
370 DS_SIZE
* priv
->num_frags
);
371 int max_gs
= (stride
- sizeof(struct mlx4_wqe_srq_next_seg
)) / DS_SIZE
;
373 for (ring_ind
= 0; ring_ind
< priv
->rx_ring_num
; ring_ind
++) {
374 ring
= &priv
->rx_ring
[ring_ind
];
378 ring
->actual_size
= 0;
379 ring
->cqn
= priv
->rx_cq
[ring_ind
].mcq
.cqn
;
381 ring
->stride
= stride
;
382 ring
->log_stride
= ffs(ring
->stride
) - 1;
383 ring
->buf_size
= ring
->size
* ring
->stride
;
385 memset(ring
->buf
, 0, ring
->buf_size
);
386 mlx4_en_update_rx_prod_db(ring
);
388 /* Initailize all descriptors */
389 for (i
= 0; i
< ring
->size
; i
++)
390 mlx4_en_init_rx_desc(priv
, ring
, i
);
392 /* Initialize page allocators */
393 err
= mlx4_en_init_allocator(priv
, ring
);
395 en_err(priv
, "Failed initializing ring allocator\n");
400 err
= mlx4_en_fill_rx_buffers(priv
);
404 for (ring_ind
= 0; ring_ind
< priv
->rx_ring_num
; ring_ind
++) {
405 ring
= &priv
->rx_ring
[ring_ind
];
407 mlx4_en_update_rx_prod_db(ring
);
409 /* Configure SRQ representing the ring */
410 ring
->srq
.max
= ring
->actual_size
;
411 ring
->srq
.max_gs
= max_gs
;
412 ring
->srq
.wqe_shift
= ilog2(ring
->stride
);
414 for (i
= 0; i
< ring
->srq
.max
; ++i
) {
415 next
= get_wqe(ring
, i
);
416 next
->next_wqe_index
=
417 cpu_to_be16((i
+ 1) & (ring
->srq
.max
- 1));
420 err
= mlx4_srq_alloc(mdev
->dev
, mdev
->priv_pdn
, &ring
->wqres
.mtt
,
421 ring
->wqres
.db
.dma
, &ring
->srq
);
423 en_err(priv
, "Failed to allocate srq\n");
427 ring
->srq
.event
= mlx4_en_srq_event
;
433 while (ring_ind
>= 0) {
434 ring
= &priv
->rx_ring
[ring_ind
];
435 mlx4_srq_free(mdev
->dev
, &ring
->srq
);
440 for (ring_ind
= 0; ring_ind
< priv
->rx_ring_num
; ring_ind
++)
441 mlx4_en_free_rx_buf(priv
, &priv
->rx_ring
[ring_ind
]);
443 ring_ind
= priv
->rx_ring_num
- 1;
445 while (ring_ind
>= 0) {
446 mlx4_en_destroy_allocator(priv
, &priv
->rx_ring
[ring_ind
]);
452 void mlx4_en_destroy_rx_ring(struct mlx4_en_priv
*priv
,
453 struct mlx4_en_rx_ring
*ring
)
455 struct mlx4_en_dev
*mdev
= priv
->mdev
;
457 kfree(ring
->lro
.lro_arr
);
458 mlx4_en_unmap_buffer(&ring
->wqres
.buf
);
459 mlx4_free_hwq_res(mdev
->dev
, &ring
->wqres
, ring
->buf_size
);
460 vfree(ring
->rx_info
);
461 ring
->rx_info
= NULL
;
464 void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv
*priv
,
465 struct mlx4_en_rx_ring
*ring
)
467 struct mlx4_en_dev
*mdev
= priv
->mdev
;
469 mlx4_srq_free(mdev
->dev
, &ring
->srq
);
470 mlx4_en_free_rx_buf(priv
, ring
);
471 mlx4_en_destroy_allocator(priv
, ring
);
475 /* Unmap a completed descriptor and free unused pages */
476 static int mlx4_en_complete_rx_desc(struct mlx4_en_priv
*priv
,
477 struct mlx4_en_rx_desc
*rx_desc
,
478 struct skb_frag_struct
*skb_frags
,
479 struct skb_frag_struct
*skb_frags_rx
,
480 struct mlx4_en_rx_alloc
*page_alloc
,
483 struct mlx4_en_dev
*mdev
= priv
->mdev
;
484 struct mlx4_en_frag_info
*frag_info
;
488 /* Collect used fragments while replacing them in the HW descirptors */
489 for (nr
= 0; nr
< priv
->num_frags
; nr
++) {
490 frag_info
= &priv
->frag_info
[nr
];
491 if (length
<= frag_info
->frag_prefix_size
)
494 /* Save page reference in skb */
495 skb_frags_rx
[nr
].page
= skb_frags
[nr
].page
;
496 skb_frags_rx
[nr
].size
= skb_frags
[nr
].size
;
497 skb_frags_rx
[nr
].page_offset
= skb_frags
[nr
].page_offset
;
498 dma
= be64_to_cpu(rx_desc
->data
[nr
].addr
);
500 /* Allocate a replacement page */
501 if (mlx4_en_alloc_frag(priv
, rx_desc
, skb_frags
, page_alloc
, nr
))
505 pci_unmap_single(mdev
->pdev
, dma
, skb_frags
[nr
].size
,
508 /* Adjust size of last fragment to match actual length */
509 skb_frags_rx
[nr
- 1].size
= length
-
510 priv
->frag_info
[nr
- 1].frag_prefix_size
;
514 /* Drop all accumulated fragments (which have already been replaced in
515 * the descriptor) of this packet; remaining fragments are reused... */
518 put_page(skb_frags_rx
[nr
].page
);
524 static struct sk_buff
*mlx4_en_rx_skb(struct mlx4_en_priv
*priv
,
525 struct mlx4_en_rx_desc
*rx_desc
,
526 struct skb_frag_struct
*skb_frags
,
527 struct mlx4_en_rx_alloc
*page_alloc
,
530 struct mlx4_en_dev
*mdev
= priv
->mdev
;
536 skb
= dev_alloc_skb(SMALL_PACKET_SIZE
+ NET_IP_ALIGN
);
538 en_dbg(RX_ERR
, priv
, "Failed allocating skb\n");
541 skb
->dev
= priv
->dev
;
542 skb_reserve(skb
, NET_IP_ALIGN
);
544 skb
->truesize
= length
+ sizeof(struct sk_buff
);
546 /* Get pointer to first fragment so we could copy the headers into the
547 * (linear part of the) skb */
548 va
= page_address(skb_frags
[0].page
) + skb_frags
[0].page_offset
;
550 if (length
<= SMALL_PACKET_SIZE
) {
551 /* We are copying all relevant data to the skb - temporarily
552 * synch buffers for the copy */
553 dma
= be64_to_cpu(rx_desc
->data
[0].addr
);
554 dma_sync_single_range_for_cpu(&mdev
->pdev
->dev
, dma
, 0,
555 length
, DMA_FROM_DEVICE
);
556 skb_copy_to_linear_data(skb
, va
, length
);
557 dma_sync_single_range_for_device(&mdev
->pdev
->dev
, dma
, 0,
558 length
, DMA_FROM_DEVICE
);
562 /* Move relevant fragments to skb */
563 used_frags
= mlx4_en_complete_rx_desc(priv
, rx_desc
, skb_frags
,
564 skb_shinfo(skb
)->frags
,
566 if (unlikely(!used_frags
)) {
570 skb_shinfo(skb
)->nr_frags
= used_frags
;
572 /* Copy headers into the skb linear buffer */
573 memcpy(skb
->data
, va
, HEADER_COPY_SIZE
);
574 skb
->tail
+= HEADER_COPY_SIZE
;
576 /* Skip headers in first fragment */
577 skb_shinfo(skb
)->frags
[0].page_offset
+= HEADER_COPY_SIZE
;
579 /* Adjust size of first fragment */
580 skb_shinfo(skb
)->frags
[0].size
-= HEADER_COPY_SIZE
;
581 skb
->data_len
= length
- HEADER_COPY_SIZE
;
587 int mlx4_en_process_rx_cq(struct net_device
*dev
, struct mlx4_en_cq
*cq
, int budget
)
589 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
590 struct mlx4_cqe
*cqe
;
591 struct mlx4_en_rx_ring
*ring
= &priv
->rx_ring
[cq
->ring
];
592 struct skb_frag_struct
*skb_frags
;
593 struct skb_frag_struct lro_frags
[MLX4_EN_MAX_RX_FRAGS
];
594 struct mlx4_en_rx_desc
*rx_desc
;
605 /* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx
606 * descriptor offset can be deduced from the CQE index instead of
607 * reading 'cqe->index' */
608 index
= cq
->mcq
.cons_index
& ring
->size_mask
;
609 cqe
= &cq
->buf
[index
];
611 /* Process all completed CQEs */
612 while (XNOR(cqe
->owner_sr_opcode
& MLX4_CQE_OWNER_MASK
,
613 cq
->mcq
.cons_index
& cq
->size
)) {
615 skb_frags
= ring
->rx_info
+ (index
<< priv
->log_rx_info
);
616 rx_desc
= ring
->buf
+ (index
<< ring
->log_stride
);
619 * make sure we read the CQE after we read the ownership bit
623 /* Drop packet on bad receive or bad checksum */
624 if (unlikely((cqe
->owner_sr_opcode
& MLX4_CQE_OPCODE_MASK
) ==
625 MLX4_CQE_OPCODE_ERROR
)) {
626 en_err(priv
, "CQE completed in error - vendor "
627 "syndrom:%d syndrom:%d\n",
628 ((struct mlx4_err_cqe
*) cqe
)->vendor_err_syndrome
,
629 ((struct mlx4_err_cqe
*) cqe
)->syndrome
);
632 if (unlikely(cqe
->badfcs_enc
& MLX4_CQE_BAD_FCS
)) {
633 en_dbg(RX_ERR
, priv
, "Accepted frame with bad FCS\n");
638 * Packet is OK - process it.
640 length
= be32_to_cpu(cqe
->byte_cnt
);
641 ring
->bytes
+= length
;
644 if (likely(priv
->rx_csum
)) {
645 if ((cqe
->status
& cpu_to_be16(MLX4_CQE_STATUS_IPOK
)) &&
646 (cqe
->checksum
== cpu_to_be16(0xffff))) {
647 priv
->port_stats
.rx_chksum_good
++;
648 /* This packet is eligible for LRO if it is:
649 * - DIX Ethernet (type interpretation)
651 * - without IP options
652 * - not an IP fragment */
653 if (mlx4_en_can_lro(cqe
->status
) &&
654 dev
->features
& NETIF_F_LRO
) {
656 nr
= mlx4_en_complete_rx_desc(
658 skb_frags
, lro_frags
,
659 ring
->page_alloc
, length
);
663 if (priv
->vlgrp
&& (cqe
->vlan_my_qpn
&
664 cpu_to_be32(MLX4_CQE_VLAN_PRESENT_MASK
))) {
665 lro_vlan_hwaccel_receive_frags(
666 &ring
->lro
, lro_frags
,
669 be16_to_cpu(cqe
->sl_vid
),
672 lro_receive_frags(&ring
->lro
,
681 /* LRO not possible, complete processing here */
682 ip_summed
= CHECKSUM_UNNECESSARY
;
683 INC_PERF_COUNTER(priv
->pstats
.lro_misses
);
685 ip_summed
= CHECKSUM_NONE
;
686 priv
->port_stats
.rx_chksum_none
++;
689 ip_summed
= CHECKSUM_NONE
;
690 priv
->port_stats
.rx_chksum_none
++;
693 skb
= mlx4_en_rx_skb(priv
, rx_desc
, skb_frags
,
694 ring
->page_alloc
, length
);
696 priv
->stats
.rx_dropped
++;
700 skb
->ip_summed
= ip_summed
;
701 skb
->protocol
= eth_type_trans(skb
, dev
);
702 skb_record_rx_queue(skb
, cq
->ring
);
704 /* Push it up the stack */
705 if (priv
->vlgrp
&& (be32_to_cpu(cqe
->vlan_my_qpn
) &
706 MLX4_CQE_VLAN_PRESENT_MASK
)) {
707 vlan_hwaccel_receive_skb(skb
, priv
->vlgrp
,
708 be16_to_cpu(cqe
->sl_vid
));
710 netif_receive_skb(skb
);
713 ++cq
->mcq
.cons_index
;
714 index
= (cq
->mcq
.cons_index
) & ring
->size_mask
;
715 cqe
= &cq
->buf
[index
];
716 if (++polled
== budget
) {
717 /* We are here because we reached the NAPI budget -
718 * flush only pending LRO sessions */
719 lro_flush_all(&ring
->lro
);
724 /* If CQ is empty flush all LRO sessions unconditionally */
725 lro_flush_all(&ring
->lro
);
728 AVG_PERF_COUNTER(priv
->pstats
.rx_coal_avg
, polled
);
729 mlx4_cq_set_ci(&cq
->mcq
);
730 wmb(); /* ensure HW sees CQ consumer before we post new buffers */
731 ring
->cons
= cq
->mcq
.cons_index
;
732 ring
->prod
+= polled
; /* Polled descriptors were realocated in place */
733 mlx4_en_update_rx_prod_db(ring
);
738 void mlx4_en_rx_irq(struct mlx4_cq
*mcq
)
740 struct mlx4_en_cq
*cq
= container_of(mcq
, struct mlx4_en_cq
, mcq
);
741 struct mlx4_en_priv
*priv
= netdev_priv(cq
->dev
);
744 napi_schedule(&cq
->napi
);
746 mlx4_en_arm_cq(priv
, cq
);
749 /* Rx CQ polling - called by NAPI */
750 int mlx4_en_poll_rx_cq(struct napi_struct
*napi
, int budget
)
752 struct mlx4_en_cq
*cq
= container_of(napi
, struct mlx4_en_cq
, napi
);
753 struct net_device
*dev
= cq
->dev
;
754 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
757 done
= mlx4_en_process_rx_cq(dev
, cq
, budget
);
759 /* If we used up all the quota - we're probably not done yet... */
761 INC_PERF_COUNTER(priv
->pstats
.napi_quota
);
765 mlx4_en_arm_cq(priv
, cq
);
771 /* Calculate the last offset position that accomodates a full fragment
772 * (assuming fagment size = stride-align) */
773 static int mlx4_en_last_alloc_offset(struct mlx4_en_priv
*priv
, u16 stride
, u16 align
)
775 u16 res
= MLX4_EN_ALLOC_SIZE
% stride
;
776 u16 offset
= MLX4_EN_ALLOC_SIZE
- stride
- res
+ align
;
778 en_dbg(DRV
, priv
, "Calculated last offset for stride:%d align:%d "
779 "res:%d offset:%d\n", stride
, align
, res
, offset
);
784 static int frag_sizes
[] = {
791 void mlx4_en_calc_rx_buf(struct net_device
*dev
)
793 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
794 int eff_mtu
= dev
->mtu
+ ETH_HLEN
+ VLAN_HLEN
+ ETH_LLC_SNAP_SIZE
;
798 while (buf_size
< eff_mtu
) {
799 priv
->frag_info
[i
].frag_size
=
800 (eff_mtu
> buf_size
+ frag_sizes
[i
]) ?
801 frag_sizes
[i
] : eff_mtu
- buf_size
;
802 priv
->frag_info
[i
].frag_prefix_size
= buf_size
;
804 priv
->frag_info
[i
].frag_align
= NET_IP_ALIGN
;
805 priv
->frag_info
[i
].frag_stride
=
806 ALIGN(frag_sizes
[i
] + NET_IP_ALIGN
, SMP_CACHE_BYTES
);
808 priv
->frag_info
[i
].frag_align
= 0;
809 priv
->frag_info
[i
].frag_stride
=
810 ALIGN(frag_sizes
[i
], SMP_CACHE_BYTES
);
812 priv
->frag_info
[i
].last_offset
= mlx4_en_last_alloc_offset(
813 priv
, priv
->frag_info
[i
].frag_stride
,
814 priv
->frag_info
[i
].frag_align
);
815 buf_size
+= priv
->frag_info
[i
].frag_size
;
820 priv
->rx_skb_size
= eff_mtu
;
821 priv
->log_rx_info
= ROUNDUP_LOG2(i
* sizeof(struct skb_frag_struct
));
823 en_dbg(DRV
, priv
, "Rx buffer scatter-list (effective-mtu:%d "
824 "num_frags:%d):\n", eff_mtu
, priv
->num_frags
);
825 for (i
= 0; i
< priv
->num_frags
; i
++) {
826 en_dbg(DRV
, priv
, " frag:%d - size:%d prefix:%d align:%d "
827 "stride:%d last_offset:%d\n", i
,
828 priv
->frag_info
[i
].frag_size
,
829 priv
->frag_info
[i
].frag_prefix_size
,
830 priv
->frag_info
[i
].frag_align
,
831 priv
->frag_info
[i
].frag_stride
,
832 priv
->frag_info
[i
].last_offset
);
836 /* RSS related functions */
838 /* Calculate rss size and map each entry in rss table to rx ring */
839 void mlx4_en_set_default_rss_map(struct mlx4_en_priv
*priv
,
840 struct mlx4_en_rss_map
*rss_map
,
841 int num_entries
, int num_rings
)
845 rss_map
->size
= roundup_pow_of_two(num_entries
);
846 en_dbg(DRV
, priv
, "Setting default RSS map of %d entires\n",
849 for (i
= 0; i
< rss_map
->size
; i
++) {
850 rss_map
->map
[i
] = i
% num_rings
;
851 en_dbg(DRV
, priv
, "Entry %d ---> ring %d\n", i
, rss_map
->map
[i
]);
855 static int mlx4_en_config_rss_qp(struct mlx4_en_priv
*priv
,
856 int qpn
, int srqn
, int cqn
,
857 enum mlx4_qp_state
*state
,
860 struct mlx4_en_dev
*mdev
= priv
->mdev
;
861 struct mlx4_qp_context
*context
;
864 context
= kmalloc(sizeof *context
, GFP_KERNEL
);
866 en_err(priv
, "Failed to allocate qp context\n");
870 err
= mlx4_qp_alloc(mdev
->dev
, qpn
, qp
);
872 en_err(priv
, "Failed to allocate qp #%x\n", qpn
);
875 qp
->event
= mlx4_en_sqp_event
;
877 memset(context
, 0, sizeof *context
);
878 mlx4_en_fill_qp_context(priv
, 0, 0, 0, 0, qpn
, cqn
, srqn
, context
);
880 err
= mlx4_qp_to_ready(mdev
->dev
, &priv
->res
.mtt
, context
, qp
, state
);
882 mlx4_qp_remove(mdev
->dev
, qp
);
883 mlx4_qp_free(mdev
->dev
, qp
);
890 /* Allocate rx qp's and configure them according to rss map */
891 int mlx4_en_config_rss_steer(struct mlx4_en_priv
*priv
)
893 struct mlx4_en_dev
*mdev
= priv
->mdev
;
894 struct mlx4_en_rss_map
*rss_map
= &priv
->rss_map
;
895 struct mlx4_qp_context context
;
896 struct mlx4_en_rss_context
*rss_context
;
898 int rss_xor
= mdev
->profile
.rss_xor
;
899 u8 rss_mask
= mdev
->profile
.rss_mask
;
900 int i
, srqn
, qpn
, cqn
;
904 en_dbg(DRV
, priv
, "Configuring rss steering\n");
905 err
= mlx4_qp_reserve_range(mdev
->dev
, rss_map
->size
,
906 rss_map
->size
, &rss_map
->base_qpn
);
908 en_err(priv
, "Failed reserving %d qps\n", rss_map
->size
);
912 for (i
= 0; i
< rss_map
->size
; i
++) {
913 cqn
= priv
->rx_ring
[rss_map
->map
[i
]].cqn
;
914 srqn
= priv
->rx_ring
[rss_map
->map
[i
]].srq
.srqn
;
915 qpn
= rss_map
->base_qpn
+ i
;
916 err
= mlx4_en_config_rss_qp(priv
, qpn
, srqn
, cqn
,
925 /* Configure RSS indirection qp */
926 err
= mlx4_qp_reserve_range(mdev
->dev
, 1, 1, &priv
->base_qpn
);
928 en_err(priv
, "Failed to reserve range for RSS "
932 err
= mlx4_qp_alloc(mdev
->dev
, priv
->base_qpn
, &rss_map
->indir_qp
);
934 en_err(priv
, "Failed to allocate RSS indirection QP\n");
937 rss_map
->indir_qp
.event
= mlx4_en_sqp_event
;
938 mlx4_en_fill_qp_context(priv
, 0, 0, 0, 1, priv
->base_qpn
,
939 priv
->rx_ring
[0].cqn
, 0, &context
);
941 ptr
= ((void *) &context
) + 0x3c;
942 rss_context
= (struct mlx4_en_rss_context
*) ptr
;
943 rss_context
->base_qpn
= cpu_to_be32(ilog2(rss_map
->size
) << 24 |
944 (rss_map
->base_qpn
));
945 rss_context
->default_qpn
= cpu_to_be32(rss_map
->base_qpn
);
946 rss_context
->hash_fn
= rss_xor
& 0x3;
947 rss_context
->flags
= rss_mask
<< 2;
949 err
= mlx4_qp_to_ready(mdev
->dev
, &priv
->res
.mtt
, &context
,
950 &rss_map
->indir_qp
, &rss_map
->indir_state
);
957 mlx4_qp_modify(mdev
->dev
, NULL
, rss_map
->indir_state
,
958 MLX4_QP_STATE_RST
, NULL
, 0, 0, &rss_map
->indir_qp
);
959 mlx4_qp_remove(mdev
->dev
, &rss_map
->indir_qp
);
960 mlx4_qp_free(mdev
->dev
, &rss_map
->indir_qp
);
962 mlx4_qp_release_range(mdev
->dev
, priv
->base_qpn
, 1);
964 for (i
= 0; i
< good_qps
; i
++) {
965 mlx4_qp_modify(mdev
->dev
, NULL
, rss_map
->state
[i
],
966 MLX4_QP_STATE_RST
, NULL
, 0, 0, &rss_map
->qps
[i
]);
967 mlx4_qp_remove(mdev
->dev
, &rss_map
->qps
[i
]);
968 mlx4_qp_free(mdev
->dev
, &rss_map
->qps
[i
]);
970 mlx4_qp_release_range(mdev
->dev
, rss_map
->base_qpn
, rss_map
->size
);
974 void mlx4_en_release_rss_steer(struct mlx4_en_priv
*priv
)
976 struct mlx4_en_dev
*mdev
= priv
->mdev
;
977 struct mlx4_en_rss_map
*rss_map
= &priv
->rss_map
;
980 mlx4_qp_modify(mdev
->dev
, NULL
, rss_map
->indir_state
,
981 MLX4_QP_STATE_RST
, NULL
, 0, 0, &rss_map
->indir_qp
);
982 mlx4_qp_remove(mdev
->dev
, &rss_map
->indir_qp
);
983 mlx4_qp_free(mdev
->dev
, &rss_map
->indir_qp
);
984 mlx4_qp_release_range(mdev
->dev
, priv
->base_qpn
, 1);
986 for (i
= 0; i
< rss_map
->size
; i
++) {
987 mlx4_qp_modify(mdev
->dev
, NULL
, rss_map
->state
[i
],
988 MLX4_QP_STATE_RST
, NULL
, 0, 0, &rss_map
->qps
[i
]);
989 mlx4_qp_remove(mdev
->dev
, &rss_map
->qps
[i
]);
990 mlx4_qp_free(mdev
->dev
, &rss_map
->qps
[i
]);
992 mlx4_qp_release_range(mdev
->dev
, rss_map
->base_qpn
, rss_map
->size
);