?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
mlx4_en: UDP RSS support
[linux-2.6/x86.git] / drivers / net / mlx4 / en_rx.c
blobe2126c76d1dc7e26bb20228e95b2e8439420db03
1 /*
2 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33
34 #include <linux/mlx4/cq.h>
35 #include <linux/slab.h>
36 #include <linux/mlx4/qp.h>
37 #include <linux/skbuff.h>
38 #include <linux/if_ether.h>
39 #include <linux/if_vlan.h>
40 #include <linux/vmalloc.h>
41
42 #include "mlx4_en.h"
43
44
45 static int mlx4_en_get_frag_header(struct skb_frag_struct *frags, void **mac_hdr,
46 void **ip_hdr, void **tcpudp_hdr,
47 u64 *hdr_flags, void *priv)
48 {
49 *mac_hdr = page_address(frags->page) + frags->page_offset;
50 *ip_hdr = *mac_hdr + ETH_HLEN;
51 *tcpudp_hdr = (struct tcphdr *)(*ip_hdr + sizeof(struct iphdr));
52 *hdr_flags = LRO_IPV4 | LRO_TCP;
53
54 return 0;
55 }
56
57 static int mlx4_en_alloc_frag(struct mlx4_en_priv *priv,
58 struct mlx4_en_rx_desc *rx_desc,
59 struct skb_frag_struct *skb_frags,
60 struct mlx4_en_rx_alloc *ring_alloc,
61 int i)
62 {
63 struct mlx4_en_dev *mdev = priv->mdev;
64 struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];
65 struct mlx4_en_rx_alloc *page_alloc = &ring_alloc[i];
66 struct page *page;
67 dma_addr_t dma;
68
69 if (page_alloc->offset == frag_info->last_offset) {
70 /* Allocate new page */
71 page = alloc_pages(GFP_ATOMIC | __GFP_COMP, MLX4_EN_ALLOC_ORDER);
72 if (!page)
73 return -ENOMEM;
74
75 skb_frags[i].page = page_alloc->page;
76 skb_frags[i].page_offset = page_alloc->offset;
77 page_alloc->page = page;
78 page_alloc->offset = frag_info->frag_align;
79 } else {
80 page = page_alloc->page;
81 get_page(page);
82
83 skb_frags[i].page = page;
84 skb_frags[i].page_offset = page_alloc->offset;
85 page_alloc->offset += frag_info->frag_stride;
86 }
87 dma = pci_map_single(mdev->pdev, page_address(skb_frags[i].page) +
88 skb_frags[i].page_offset, frag_info->frag_size,
89 PCI_DMA_FROMDEVICE);
90 rx_desc->data[i].addr = cpu_to_be64(dma);
91 return 0;
92 }
93
94 static int mlx4_en_init_allocator(struct mlx4_en_priv *priv,
95 struct mlx4_en_rx_ring *ring)
96 {
97 struct mlx4_en_rx_alloc *page_alloc;
98 int i;
99
100 for (i = 0; i < priv->num_frags; i++) {
101 page_alloc = &ring->page_alloc[i];
102 page_alloc->page = alloc_pages(GFP_ATOMIC | __GFP_COMP,
103 MLX4_EN_ALLOC_ORDER);
104 if (!page_alloc->page)
105 goto out;
106
107 page_alloc->offset = priv->frag_info[i].frag_align;
108 en_dbg(DRV, priv, "Initialized allocator:%d with page:%p\n",
109 i, page_alloc->page);
110 }
111 return 0;
112
113 out:
114 while (i--) {
115 page_alloc = &ring->page_alloc[i];
116 put_page(page_alloc->page);
117 page_alloc->page = NULL;
118 }
119 return -ENOMEM;
120 }
121
122 static void mlx4_en_destroy_allocator(struct mlx4_en_priv *priv,
123 struct mlx4_en_rx_ring *ring)
124 {
125 struct mlx4_en_rx_alloc *page_alloc;
126 int i;
127
128 for (i = 0; i < priv->num_frags; i++) {
129 page_alloc = &ring->page_alloc[i];
130 en_dbg(DRV, priv, "Freeing allocator:%d count:%d\n",
131 i, page_count(page_alloc->page));
132
133 put_page(page_alloc->page);
134 page_alloc->page = NULL;
135 }
136 }
137
138
139 static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv,
140 struct mlx4_en_rx_ring *ring, int index)
141 {
142 struct mlx4_en_rx_desc *rx_desc = ring->buf + ring->stride * index;
143 struct skb_frag_struct *skb_frags = ring->rx_info +
144 (index << priv->log_rx_info);
145 int possible_frags;
146 int i;
147
148 /* Set size and memtype fields */
149 for (i = 0; i < priv->num_frags; i++) {
150 skb_frags[i].size = priv->frag_info[i].frag_size;
151 rx_desc->data[i].byte_count =
152 cpu_to_be32(priv->frag_info[i].frag_size);
153 rx_desc->data[i].lkey = cpu_to_be32(priv->mdev->mr.key);
154 }
155
156 /* If the number of used fragments does not fill up the ring stride,
157 * remaining (unused) fragments must be padded with null address/size
158 * and a special memory key */
159 possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE;
160 for (i = priv->num_frags; i < possible_frags; i++) {
161 rx_desc->data[i].byte_count = 0;
162 rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD);
163 rx_desc->data[i].addr = 0;
164 }
165 }
166
167
168 static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv,
169 struct mlx4_en_rx_ring *ring, int index)
170 {
171 struct mlx4_en_rx_desc *rx_desc = ring->buf + (index * ring->stride);
172 struct skb_frag_struct *skb_frags = ring->rx_info +
173 (index << priv->log_rx_info);
174 int i;
175
176 for (i = 0; i < priv->num_frags; i++)
177 if (mlx4_en_alloc_frag(priv, rx_desc, skb_frags, ring->page_alloc, i))
178 goto err;
179
180 return 0;
181
182 err:
183 while (i--)
184 put_page(skb_frags[i].page);
185 return -ENOMEM;
186 }
187
188 static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring)
189 {
190 *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff);
191 }
192
193 static void mlx4_en_free_rx_desc(struct mlx4_en_priv *priv,
194 struct mlx4_en_rx_ring *ring,
195 int index)
196 {
197 struct mlx4_en_dev *mdev = priv->mdev;
198 struct skb_frag_struct *skb_frags;
199 struct mlx4_en_rx_desc *rx_desc = ring->buf + (index << ring->log_stride);
200 dma_addr_t dma;
201 int nr;
202
203 skb_frags = ring->rx_info + (index << priv->log_rx_info);
204 for (nr = 0; nr < priv->num_frags; nr++) {
205 en_dbg(DRV, priv, "Freeing fragment:%d\n", nr);
206 dma = be64_to_cpu(rx_desc->data[nr].addr);
207
208 en_dbg(DRV, priv, "Unmapping buffer at dma:0x%llx\n", (u64) dma);
209 pci_unmap_single(mdev->pdev, dma, skb_frags[nr].size,
210 PCI_DMA_FROMDEVICE);
211 put_page(skb_frags[nr].page);
212 }
213 }
214
215 static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv)
216 {
217 struct mlx4_en_rx_ring *ring;
218 int ring_ind;
219 int buf_ind;
220 int new_size;
221
222 for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) {
223 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
224 ring = &priv->rx_ring[ring_ind];
225
226 if (mlx4_en_prepare_rx_desc(priv, ring,
227 ring->actual_size)) {
228 if (ring->actual_size < MLX4_EN_MIN_RX_SIZE) {
229 en_err(priv, "Failed to allocate "
230 "enough rx buffers\n");
231 return -ENOMEM;
232 } else {
233 new_size = rounddown_pow_of_two(ring->actual_size);
234 en_warn(priv, "Only %d buffers allocated "
235 "reducing ring size to %d",
236 ring->actual_size, new_size);
237 goto reduce_rings;
238 }
239 }
240 ring->actual_size++;
241 ring->prod++;
242 }
243 }
244 return 0;
245
246 reduce_rings:
247 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
248 ring = &priv->rx_ring[ring_ind];
249 while (ring->actual_size > new_size) {
250 ring->actual_size--;
251 ring->prod--;
252 mlx4_en_free_rx_desc(priv, ring, ring->actual_size);
253 }
254 }
255
256 return 0;
257 }
258
259 static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv,
260 struct mlx4_en_rx_ring *ring)
261 {
262 int index;
263
264 en_dbg(DRV, priv, "Freeing Rx buf - cons:%d prod:%d\n",
265 ring->cons, ring->prod);
266
267 /* Unmap and free Rx buffers */
268 BUG_ON((u32) (ring->prod - ring->cons) > ring->actual_size);
269 while (ring->cons != ring->prod) {
270 index = ring->cons & ring->size_mask;
271 en_dbg(DRV, priv, "Processing descriptor:%d\n", index);
272 mlx4_en_free_rx_desc(priv, ring, index);
273 ++ring->cons;
274 }
275 }
276
277 int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
278 struct mlx4_en_rx_ring *ring, u32 size, u16 stride)
279 {
280 struct mlx4_en_dev *mdev = priv->mdev;
281 int err;
282 int tmp;
283
284
285 ring->prod = 0;
286 ring->cons = 0;
287 ring->size = size;
288 ring->size_mask = size - 1;
289 ring->stride = stride;
290 ring->log_stride = ffs(ring->stride) - 1;
291 ring->buf_size = ring->size * ring->stride + TXBB_SIZE;
292
293 tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS *
294 sizeof(struct skb_frag_struct));
295 ring->rx_info = vmalloc(tmp);
296 if (!ring->rx_info) {
297 en_err(priv, "Failed allocating rx_info ring\n");
298 return -ENOMEM;
299 }
300 en_dbg(DRV, priv, "Allocated rx_info ring at addr:%p size:%d\n",
301 ring->rx_info, tmp);
302
303 err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres,
304 ring->buf_size, 2 * PAGE_SIZE);
305 if (err)
306 goto err_ring;
307
308 err = mlx4_en_map_buffer(&ring->wqres.buf);
309 if (err) {
310 en_err(priv, "Failed to map RX buffer\n");
311 goto err_hwq;
312 }
313 ring->buf = ring->wqres.buf.direct.buf;
314
315 /* Configure lro mngr */
316 memset(&ring->lro, 0, sizeof(struct net_lro_mgr));
317 ring->lro.dev = priv->dev;
318 ring->lro.features = LRO_F_NAPI;
319 ring->lro.frag_align_pad = NET_IP_ALIGN;
320 ring->lro.ip_summed = CHECKSUM_UNNECESSARY;
321 ring->lro.ip_summed_aggr = CHECKSUM_UNNECESSARY;
322 ring->lro.max_desc = MLX4_EN_MAX_LRO_DESCRIPTORS;
323 ring->lro.max_aggr = MAX_SKB_FRAGS;
324 ring->lro.lro_arr = kcalloc(MLX4_EN_MAX_LRO_DESCRIPTORS,
325 sizeof(struct net_lro_desc),
326 GFP_KERNEL);
327 if (!ring->lro.lro_arr) {
328 en_err(priv, "Failed to allocate lro array\n");
329 goto err_map;
330 }
331 ring->lro.get_frag_header = mlx4_en_get_frag_header;
332
333 return 0;
334
335 err_map:
336 mlx4_en_unmap_buffer(&ring->wqres.buf);
337 err_hwq:
338 mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
339 err_ring:
340 vfree(ring->rx_info);
341 ring->rx_info = NULL;
342 return err;
343 }
344
345 int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv)
346 {
347 struct mlx4_en_rx_ring *ring;
348 int i;
349 int ring_ind;
350 int err;
351 int stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) +
352 DS_SIZE * priv->num_frags);
353
354 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
355 ring = &priv->rx_ring[ring_ind];
356
357 ring->prod = 0;
358 ring->cons = 0;
359 ring->actual_size = 0;
360 ring->cqn = priv->rx_cq[ring_ind].mcq.cqn;
361
362 ring->stride = stride;
363 if (ring->stride <= TXBB_SIZE)
364 ring->buf += TXBB_SIZE;
365
366 ring->log_stride = ffs(ring->stride) - 1;
367 ring->buf_size = ring->size * ring->stride;
368
369 memset(ring->buf, 0, ring->buf_size);
370 mlx4_en_update_rx_prod_db(ring);
371
372 /* Initailize all descriptors */
373 for (i = 0; i < ring->size; i++)
374 mlx4_en_init_rx_desc(priv, ring, i);
375
376 /* Initialize page allocators */
377 err = mlx4_en_init_allocator(priv, ring);
378 if (err) {
379 en_err(priv, "Failed initializing ring allocator\n");
380 ring_ind--;
381 goto err_allocator;
382 }
383 }
384 err = mlx4_en_fill_rx_buffers(priv);
385 if (err)
386 goto err_buffers;
387
388 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
389 ring = &priv->rx_ring[ring_ind];
390
391 ring->size_mask = ring->actual_size - 1;
392 mlx4_en_update_rx_prod_db(ring);
393 }
394
395 return 0;
396
397 err_buffers:
398 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++)
399 mlx4_en_free_rx_buf(priv, &priv->rx_ring[ring_ind]);
400
401 ring_ind = priv->rx_ring_num - 1;
402 err_allocator:
403 while (ring_ind >= 0) {
404 mlx4_en_destroy_allocator(priv, &priv->rx_ring[ring_ind]);
405 ring_ind--;
406 }
407 return err;
408 }
409
410 void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
411 struct mlx4_en_rx_ring *ring)
412 {
413 struct mlx4_en_dev *mdev = priv->mdev;
414
415 kfree(ring->lro.lro_arr);
416 mlx4_en_unmap_buffer(&ring->wqres.buf);
417 mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size + TXBB_SIZE);
418 vfree(ring->rx_info);
419 ring->rx_info = NULL;
420 }
421
422 void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv,
423 struct mlx4_en_rx_ring *ring)
424 {
425 mlx4_en_free_rx_buf(priv, ring);
426 if (ring->stride <= TXBB_SIZE)
427 ring->buf -= TXBB_SIZE;
428 mlx4_en_destroy_allocator(priv, ring);
429 }
430
431
432 /* Unmap a completed descriptor and free unused pages */
433 static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv,
434 struct mlx4_en_rx_desc *rx_desc,
435 struct skb_frag_struct *skb_frags,
436 struct skb_frag_struct *skb_frags_rx,
437 struct mlx4_en_rx_alloc *page_alloc,
438 int length)
439 {
440 struct mlx4_en_dev *mdev = priv->mdev;
441 struct mlx4_en_frag_info *frag_info;
442 int nr;
443 dma_addr_t dma;
444
445 /* Collect used fragments while replacing them in the HW descirptors */
446 for (nr = 0; nr < priv->num_frags; nr++) {
447 frag_info = &priv->frag_info[nr];
448 if (length <= frag_info->frag_prefix_size)
449 break;
450
451 /* Save page reference in skb */
452 skb_frags_rx[nr].page = skb_frags[nr].page;
453 skb_frags_rx[nr].size = skb_frags[nr].size;
454 skb_frags_rx[nr].page_offset = skb_frags[nr].page_offset;
455 dma = be64_to_cpu(rx_desc->data[nr].addr);
456
457 /* Allocate a replacement page */
458 if (mlx4_en_alloc_frag(priv, rx_desc, skb_frags, page_alloc, nr))
459 goto fail;
460
461 /* Unmap buffer */
462 pci_unmap_single(mdev->pdev, dma, skb_frags_rx[nr].size,
463 PCI_DMA_FROMDEVICE);
464 }
465 /* Adjust size of last fragment to match actual length */
466 if (nr > 0)
467 skb_frags_rx[nr - 1].size = length -
468 priv->frag_info[nr - 1].frag_prefix_size;
469 return nr;
470
471 fail:
472 /* Drop all accumulated fragments (which have already been replaced in
473 * the descriptor) of this packet; remaining fragments are reused... */
474 while (nr > 0) {
475 nr--;
476 put_page(skb_frags_rx[nr].page);
477 }
478 return 0;
479 }
480
481
482 static struct sk_buff *mlx4_en_rx_skb(struct mlx4_en_priv *priv,
483 struct mlx4_en_rx_desc *rx_desc,
484 struct skb_frag_struct *skb_frags,
485 struct mlx4_en_rx_alloc *page_alloc,
486 unsigned int length)
487 {
488 struct mlx4_en_dev *mdev = priv->mdev;
489 struct sk_buff *skb;
490 void *va;
491 int used_frags;
492 dma_addr_t dma;
493
494 skb = dev_alloc_skb(SMALL_PACKET_SIZE + NET_IP_ALIGN);
495 if (!skb) {
496 en_dbg(RX_ERR, priv, "Failed allocating skb\n");
497 return NULL;
498 }
499 skb->dev = priv->dev;
500 skb_reserve(skb, NET_IP_ALIGN);
501 skb->len = length;
502 skb->truesize = length + sizeof(struct sk_buff);
503
504 /* Get pointer to first fragment so we could copy the headers into the
505 * (linear part of the) skb */
506 va = page_address(skb_frags[0].page) + skb_frags[0].page_offset;
507
508 if (length <= SMALL_PACKET_SIZE) {
509 /* We are copying all relevant data to the skb - temporarily
510 * synch buffers for the copy */
511 dma = be64_to_cpu(rx_desc->data[0].addr);
512 dma_sync_single_for_cpu(&mdev->pdev->dev, dma, length,
513 DMA_FROM_DEVICE);
514 skb_copy_to_linear_data(skb, va, length);
515 dma_sync_single_for_device(&mdev->pdev->dev, dma, length,
516 DMA_FROM_DEVICE);
517 skb->tail += length;
518 } else {
519
520 /* Move relevant fragments to skb */
521 used_frags = mlx4_en_complete_rx_desc(priv, rx_desc, skb_frags,
522 skb_shinfo(skb)->frags,
523 page_alloc, length);
524 if (unlikely(!used_frags)) {
525 kfree_skb(skb);
526 return NULL;
527 }
528 skb_shinfo(skb)->nr_frags = used_frags;
529
530 /* Copy headers into the skb linear buffer */
531 memcpy(skb->data, va, HEADER_COPY_SIZE);
532 skb->tail += HEADER_COPY_SIZE;
533
534 /* Skip headers in first fragment */
535 skb_shinfo(skb)->frags[0].page_offset += HEADER_COPY_SIZE;
536
537 /* Adjust size of first fragment */
538 skb_shinfo(skb)->frags[0].size -= HEADER_COPY_SIZE;
539 skb->data_len = length - HEADER_COPY_SIZE;
540 }
541 return skb;
542 }
543
544 static void validate_loopback(struct mlx4_en_priv *priv, struct sk_buff *skb)
545 {
546 int i;
547 int offset = ETH_HLEN;
548
549 for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++, offset++) {
550 if (*(skb->data + offset) != (unsigned char) (i & 0xff))
551 goto out_loopback;
552 }
553 /* Loopback found */
554 priv->loopback_ok = 1;
555
556 out_loopback:
557 dev_kfree_skb_any(skb);
558 }
559
560 int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget)
561 {
562 struct mlx4_en_priv *priv = netdev_priv(dev);
563 struct mlx4_cqe *cqe;
564 struct mlx4_en_rx_ring *ring = &priv->rx_ring[cq->ring];
565 struct skb_frag_struct *skb_frags;
566 struct skb_frag_struct lro_frags[MLX4_EN_MAX_RX_FRAGS];
567 struct mlx4_en_rx_desc *rx_desc;
568 struct sk_buff *skb;
569 int index;
570 int nr;
571 unsigned int length;
572 int polled = 0;
573 int ip_summed;
574
575 if (!priv->port_up)
576 return 0;
577
578 /* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx
579 * descriptor offset can be deduced from the CQE index instead of
580 * reading 'cqe->index' */
581 index = cq->mcq.cons_index & ring->size_mask;
582 cqe = &cq->buf[index];
583
584 /* Process all completed CQEs */
585 while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
586 cq->mcq.cons_index & cq->size)) {
587
588 skb_frags = ring->rx_info + (index << priv->log_rx_info);
589 rx_desc = ring->buf + (index << ring->log_stride);
590
591 /*
592 * make sure we read the CQE after we read the ownership bit
593 */
594 rmb();
595
596 /* Drop packet on bad receive or bad checksum */
597 if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
598 MLX4_CQE_OPCODE_ERROR)) {
599 en_err(priv, "CQE completed in error - vendor "
600 "syndrom:%d syndrom:%d\n",
601 ((struct mlx4_err_cqe *) cqe)->vendor_err_syndrome,
602 ((struct mlx4_err_cqe *) cqe)->syndrome);
603 goto next;
604 }
605 if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) {
606 en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n");
607 goto next;
608 }
609
610 /*
611 * Packet is OK - process it.
612 */
613 length = be32_to_cpu(cqe->byte_cnt);
614 ring->bytes += length;
615 ring->packets++;
616
617 if (likely(priv->rx_csum)) {
618 if ((cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) &&
619 (cqe->checksum == cpu_to_be16(0xffff))) {
620 priv->port_stats.rx_chksum_good++;
621 /* This packet is eligible for LRO if it is:
622 * - DIX Ethernet (type interpretation)
623 * - TCP/IP (v4)
624 * - without IP options
625 * - not an IP fragment */
626 if (mlx4_en_can_lro(cqe->status) &&
627 dev->features & NETIF_F_LRO) {
628
629 nr = mlx4_en_complete_rx_desc(
630 priv, rx_desc,
631 skb_frags, lro_frags,
632 ring->page_alloc, length);
633 if (!nr)
634 goto next;
635
636 if (priv->vlgrp && (cqe->vlan_my_qpn &
637 cpu_to_be32(MLX4_CQE_VLAN_PRESENT_MASK))) {
638 lro_vlan_hwaccel_receive_frags(
639 &ring->lro, lro_frags,
640 length, length,
641 priv->vlgrp,
642 be16_to_cpu(cqe->sl_vid),
643 NULL, 0);
644 } else
645 lro_receive_frags(&ring->lro,
646 lro_frags,
647 length,
648 length,
649 NULL, 0);
650
651 goto next;
652 }
653
654 /* LRO not possible, complete processing here */
655 ip_summed = CHECKSUM_UNNECESSARY;
656 INC_PERF_COUNTER(priv->pstats.lro_misses);
657 } else {
658 ip_summed = CHECKSUM_NONE;
659 priv->port_stats.rx_chksum_none++;
660 }
661 } else {
662 ip_summed = CHECKSUM_NONE;
663 priv->port_stats.rx_chksum_none++;
664 }
665
666 skb = mlx4_en_rx_skb(priv, rx_desc, skb_frags,
667 ring->page_alloc, length);
668 if (!skb) {
669 priv->stats.rx_dropped++;
670 goto next;
671 }
672
673 if (unlikely(priv->validate_loopback)) {
674 validate_loopback(priv, skb);
675 goto next;
676 }
677
678 skb->ip_summed = ip_summed;
679 skb->protocol = eth_type_trans(skb, dev);
680 skb_record_rx_queue(skb, cq->ring);
681
682 /* Push it up the stack */
683 if (priv->vlgrp && (be32_to_cpu(cqe->vlan_my_qpn) &
684 MLX4_CQE_VLAN_PRESENT_MASK)) {
685 vlan_hwaccel_receive_skb(skb, priv->vlgrp,
686 be16_to_cpu(cqe->sl_vid));
687 } else
688 netif_receive_skb(skb);
689
690 next:
691 ++cq->mcq.cons_index;
692 index = (cq->mcq.cons_index) & ring->size_mask;
693 cqe = &cq->buf[index];
694 if (++polled == budget) {
695 /* We are here because we reached the NAPI budget -
696 * flush only pending LRO sessions */
697 lro_flush_all(&ring->lro);
698 goto out;
699 }
700 }
701
702 /* If CQ is empty flush all LRO sessions unconditionally */
703 lro_flush_all(&ring->lro);
704
705 out:
706 AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled);
707 mlx4_cq_set_ci(&cq->mcq);
708 wmb(); /* ensure HW sees CQ consumer before we post new buffers */
709 ring->cons = cq->mcq.cons_index;
710 ring->prod += polled; /* Polled descriptors were realocated in place */
711 mlx4_en_update_rx_prod_db(ring);
712 return polled;
713 }
714
715
716 void mlx4_en_rx_irq(struct mlx4_cq *mcq)
717 {
718 struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq);
719 struct mlx4_en_priv *priv = netdev_priv(cq->dev);
720
721 if (priv->port_up)
722 napi_schedule(&cq->napi);
723 else
724 mlx4_en_arm_cq(priv, cq);
725 }
726
727 /* Rx CQ polling - called by NAPI */
728 int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget)
729 {
730 struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi);
731 struct net_device *dev = cq->dev;
732 struct mlx4_en_priv *priv = netdev_priv(dev);
733 int done;
734
735 done = mlx4_en_process_rx_cq(dev, cq, budget);
736
737 /* If we used up all the quota - we're probably not done yet... */
738 if (done == budget)
739 INC_PERF_COUNTER(priv->pstats.napi_quota);
740 else {
741 /* Done for now */
742 napi_complete(napi);
743 mlx4_en_arm_cq(priv, cq);
744 }
745 return done;
746 }
747
748
749 /* Calculate the last offset position that accomodates a full fragment
750 * (assuming fagment size = stride-align) */
751 static int mlx4_en_last_alloc_offset(struct mlx4_en_priv *priv, u16 stride, u16 align)
752 {
753 u16 res = MLX4_EN_ALLOC_SIZE % stride;
754 u16 offset = MLX4_EN_ALLOC_SIZE - stride - res + align;
755
756 en_dbg(DRV, priv, "Calculated last offset for stride:%d align:%d "
757 "res:%d offset:%d\n", stride, align, res, offset);
758 return offset;
759 }
760
761
762 static int frag_sizes[] = {
763 FRAG_SZ0,
764 FRAG_SZ1,
765 FRAG_SZ2,
766 FRAG_SZ3
767 };
768
769 void mlx4_en_calc_rx_buf(struct net_device *dev)
770 {
771 struct mlx4_en_priv *priv = netdev_priv(dev);
772 int eff_mtu = dev->mtu + ETH_HLEN + VLAN_HLEN + ETH_LLC_SNAP_SIZE;
773 int buf_size = 0;
774 int i = 0;
775
776 while (buf_size < eff_mtu) {
777 priv->frag_info[i].frag_size =
778 (eff_mtu > buf_size + frag_sizes[i]) ?
779 frag_sizes[i] : eff_mtu - buf_size;
780 priv->frag_info[i].frag_prefix_size = buf_size;
781 if (!i) {
782 priv->frag_info[i].frag_align = NET_IP_ALIGN;
783 priv->frag_info[i].frag_stride =
784 ALIGN(frag_sizes[i] + NET_IP_ALIGN, SMP_CACHE_BYTES);
785 } else {
786 priv->frag_info[i].frag_align = 0;
787 priv->frag_info[i].frag_stride =
788 ALIGN(frag_sizes[i], SMP_CACHE_BYTES);
789 }
790 priv->frag_info[i].last_offset = mlx4_en_last_alloc_offset(
791 priv, priv->frag_info[i].frag_stride,
792 priv->frag_info[i].frag_align);
793 buf_size += priv->frag_info[i].frag_size;
794 i++;
795 }
796
797 priv->num_frags = i;
798 priv->rx_skb_size = eff_mtu;
799 priv->log_rx_info = ROUNDUP_LOG2(i * sizeof(struct skb_frag_struct));
800
801 en_dbg(DRV, priv, "Rx buffer scatter-list (effective-mtu:%d "
802 "num_frags:%d):\n", eff_mtu, priv->num_frags);
803 for (i = 0; i < priv->num_frags; i++) {
804 en_dbg(DRV, priv, " frag:%d - size:%d prefix:%d align:%d "
805 "stride:%d last_offset:%d\n", i,
806 priv->frag_info[i].frag_size,
807 priv->frag_info[i].frag_prefix_size,
808 priv->frag_info[i].frag_align,
809 priv->frag_info[i].frag_stride,
810 priv->frag_info[i].last_offset);
811 }
812 }
813
814 /* RSS related functions */
815
816 static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn,
817 struct mlx4_en_rx_ring *ring,
818 enum mlx4_qp_state *state,
819 struct mlx4_qp *qp)
820 {
821 struct mlx4_en_dev *mdev = priv->mdev;
822 struct mlx4_qp_context *context;
823 int err = 0;
824
825 context = kmalloc(sizeof *context , GFP_KERNEL);
826 if (!context) {
827 en_err(priv, "Failed to allocate qp context\n");
828 return -ENOMEM;
829 }
830
831 err = mlx4_qp_alloc(mdev->dev, qpn, qp);
832 if (err) {
833 en_err(priv, "Failed to allocate qp #%x\n", qpn);
834 goto out;
835 }
836 qp->event = mlx4_en_sqp_event;
837
838 memset(context, 0, sizeof *context);
839 mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0,
840 qpn, ring->cqn, context);
841 context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma);
842
843 err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state);
844 if (err) {
845 mlx4_qp_remove(mdev->dev, qp);
846 mlx4_qp_free(mdev->dev, qp);
847 }
848 mlx4_en_update_rx_prod_db(ring);
849 out:
850 kfree(context);
851 return err;
852 }
853
854 /* Allocate rx qp's and configure them according to rss map */
855 int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv)
856 {
857 struct mlx4_en_dev *mdev = priv->mdev;
858 struct mlx4_en_rss_map *rss_map = &priv->rss_map;
859 struct mlx4_qp_context context;
860 struct mlx4_en_rss_context *rss_context;
861 void *ptr;
862 u8 rss_mask = 0x3f;
863 int i, qpn;
864 int err = 0;
865 int good_qps = 0;
866
867 en_dbg(DRV, priv, "Configuring rss steering\n");
868 err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num,
869 priv->rx_ring_num,
870 &rss_map->base_qpn);
871 if (err) {
872 en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num);
873 return err;
874 }
875
876 for (i = 0; i < priv->rx_ring_num; i++) {
877 qpn = rss_map->base_qpn + i;
878 err = mlx4_en_config_rss_qp(priv, qpn, &priv->rx_ring[i],
879 &rss_map->state[i],
880 &rss_map->qps[i]);
881 if (err)
882 goto rss_err;
883
884 ++good_qps;
885 }
886
887 /* Configure RSS indirection qp */
888 err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &priv->base_qpn);
889 if (err) {
890 en_err(priv, "Failed to reserve range for RSS "
891 "indirection qp\n");
892 goto rss_err;
893 }
894 err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp);
895 if (err) {
896 en_err(priv, "Failed to allocate RSS indirection QP\n");
897 goto reserve_err;
898 }
899 rss_map->indir_qp.event = mlx4_en_sqp_event;
900 mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn,
901 priv->rx_ring[0].cqn, &context);
902
903 ptr = ((void *) &context) + 0x3c;
904 rss_context = (struct mlx4_en_rss_context *) ptr;
905 rss_context->base_qpn = cpu_to_be32(ilog2(priv->rx_ring_num) << 24 |
906 (rss_map->base_qpn));
907 rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn);
908 rss_context->flags = rss_mask;
909
910 if (priv->mdev->profile.udp_rss)
911 rss_context->base_qpn_udp = rss_context->default_qpn;
912 err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context,
913 &rss_map->indir_qp, &rss_map->indir_state);
914 if (err)
915 goto indir_err;
916
917 return 0;
918
919 indir_err:
920 mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
921 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
922 mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
923 mlx4_qp_free(mdev->dev, &rss_map->indir_qp);
924 reserve_err:
925 mlx4_qp_release_range(mdev->dev, priv->base_qpn, 1);
926 rss_err:
927 for (i = 0; i < good_qps; i++) {
928 mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
929 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
930 mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
931 mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
932 }
933 mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
934 return err;
935 }
936
937 void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv)
938 {
939 struct mlx4_en_dev *mdev = priv->mdev;
940 struct mlx4_en_rss_map *rss_map = &priv->rss_map;
941 int i;
942
943 mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
944 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
945 mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
946 mlx4_qp_free(mdev->dev, &rss_map->indir_qp);
947 mlx4_qp_release_range(mdev->dev, priv->base_qpn, 1);
948
949 for (i = 0; i < priv->rx_ring_num; i++) {
950 mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
951 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
952 mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
953 mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
954 }
955 mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
956 }
957
958
959
960
961
x86 "subsystem" repository