4 * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 * $FreeBSD: head/sys/dev/ena/ena.c 325593 2017-11-09 13:38:17Z mw $
33 #include <sys/param.h>
34 #include <sys/systm.h>
36 #include <sys/endian.h>
37 #include <sys/kernel.h>
38 #include <sys/kthread.h>
39 #include <sys/malloc.h>
41 #include <sys/module.h>
43 #include <sys/socket.h>
44 #include <sys/sockio.h>
45 #include <sys/sysctl.h>
46 #include <sys/taskqueue.h>
48 #include <sys/eventhandler.h>
52 #include <net/if_var.h>
53 #include <net/if_arp.h>
54 #include <net/if_dl.h>
55 #include <net/if_media.h>
56 #include <net/if_types.h>
57 #include <net/ifq_var.h>
58 #include <net/vlan/if_vlan_var.h>
60 #include <netinet/in_systm.h>
61 #include <netinet/in.h>
62 #include <netinet/if_ether.h>
63 #include <netinet/ip.h>
64 #include <netinet/ip6.h>
65 #include <netinet/tcp.h>
66 #include <netinet/udp.h>
68 #include <bus/pci/pcivar.h>
69 #include <bus/pci/pcireg.h>
72 #include "ena_sysctl.h"
74 /*********************************************************
76 *********************************************************/
77 static int ena_probe(device_t
);
78 static void ena_intr_msix_mgmnt(void *);
79 static int ena_allocate_pci_resources(struct ena_adapter
*);
80 static void ena_free_pci_resources(struct ena_adapter
*);
81 static int ena_change_mtu(if_t
, int);
82 #if 0 /* XXX swildner counters */
83 static inline void ena_alloc_counters(counter_u64_t
*, int);
84 static inline void ena_free_counters(counter_u64_t
*, int);
85 static inline void ena_reset_counters(counter_u64_t
*, int);
87 static void ena_init_io_rings_common(struct ena_adapter
*,
88 struct ena_ring
*, uint16_t);
89 static void ena_init_io_rings(struct ena_adapter
*);
90 static void ena_free_io_ring_resources(struct ena_adapter
*, unsigned int);
91 static void ena_free_all_io_rings_resources(struct ena_adapter
*);
92 static int ena_setup_tx_dma_tag(struct ena_adapter
*);
93 static int ena_free_tx_dma_tag(struct ena_adapter
*);
94 static int ena_setup_rx_dma_tag(struct ena_adapter
*);
95 static int ena_free_rx_dma_tag(struct ena_adapter
*);
96 static int ena_setup_tx_resources(struct ena_adapter
*, int);
97 static void ena_free_tx_resources(struct ena_adapter
*, int);
98 static int ena_setup_all_tx_resources(struct ena_adapter
*);
99 static void ena_free_all_tx_resources(struct ena_adapter
*);
100 static inline int validate_rx_req_id(struct ena_ring
*, uint16_t);
101 static int ena_setup_rx_resources(struct ena_adapter
*, unsigned int);
102 static void ena_free_rx_resources(struct ena_adapter
*, unsigned int);
103 static int ena_setup_all_rx_resources(struct ena_adapter
*);
104 static void ena_free_all_rx_resources(struct ena_adapter
*);
105 static inline int ena_alloc_rx_mbuf(struct ena_adapter
*, struct ena_ring
*,
106 struct ena_rx_buffer
*);
107 static void ena_free_rx_mbuf(struct ena_adapter
*, struct ena_ring
*,
108 struct ena_rx_buffer
*);
109 static int ena_refill_rx_bufs(struct ena_ring
*, uint32_t);
110 static void ena_free_rx_bufs(struct ena_adapter
*, unsigned int);
111 static void ena_refill_all_rx_bufs(struct ena_adapter
*);
112 static void ena_free_all_rx_bufs(struct ena_adapter
*);
113 static void ena_free_tx_bufs(struct ena_adapter
*, unsigned int);
114 static void ena_free_all_tx_bufs(struct ena_adapter
*);
115 static void ena_destroy_all_tx_queues(struct ena_adapter
*);
116 static void ena_destroy_all_rx_queues(struct ena_adapter
*);
117 static void ena_destroy_all_io_queues(struct ena_adapter
*);
118 static int ena_create_io_queues(struct ena_adapter
*);
119 static int ena_tx_cleanup(struct ena_ring
*);
120 static void ena_deferred_rx_cleanup(void *, int);
121 static int ena_rx_cleanup(struct ena_ring
*);
122 static inline int validate_tx_req_id(struct ena_ring
*, uint16_t);
123 static void ena_rx_hash_mbuf(struct ena_ring
*, struct ena_com_rx_ctx
*,
125 static struct mbuf
* ena_rx_mbuf(struct ena_ring
*, struct ena_com_rx_buf_info
*,
126 struct ena_com_rx_ctx
*, uint16_t *);
127 static inline void ena_rx_checksum(struct ena_ring
*, struct ena_com_rx_ctx
*,
129 static void ena_handle_msix(void *);
130 static int ena_enable_msix(struct ena_adapter
*);
131 static void ena_setup_mgmnt_intr(struct ena_adapter
*);
132 static void ena_setup_io_intr(struct ena_adapter
*);
133 static int ena_request_mgmnt_irq(struct ena_adapter
*);
134 static int ena_request_io_irq(struct ena_adapter
*);
135 static void ena_free_mgmnt_irq(struct ena_adapter
*);
136 static void ena_free_io_irq(struct ena_adapter
*);
137 static void ena_free_irqs(struct ena_adapter
*);
138 static void ena_disable_msix(struct ena_adapter
*);
139 static void ena_unmask_all_io_irqs(struct ena_adapter
*);
140 static int ena_rss_configure(struct ena_adapter
*);
141 static int ena_up_complete(struct ena_adapter
*);
142 static int ena_up(struct ena_adapter
*);
143 static void ena_down(struct ena_adapter
*);
144 #if 0 /* XXX swildner counters */
145 static uint64_t ena_get_counter(if_t
, ift_counter
);
147 static int ena_media_change(if_t
);
148 static void ena_media_status(if_t
, struct ifmediareq
*);
149 static void ena_init(void *);
150 static int ena_ioctl(if_t
, u_long
, caddr_t
, struct ucred
*);
151 static int ena_get_dev_offloads(struct ena_com_dev_get_features_ctx
*);
152 static void ena_update_host_info(struct ena_admin_host_info
*, if_t
);
153 static void ena_update_hwassist(struct ena_adapter
*);
154 static int ena_setup_ifnet(device_t
, struct ena_adapter
*,
155 struct ena_com_dev_get_features_ctx
*);
156 static void ena_tx_csum(struct ena_com_tx_ctx
*, struct mbuf
*);
157 static int ena_check_and_collapse_mbuf(struct ena_ring
*tx_ring
,
159 static int ena_xmit_mbuf(struct ena_ring
*, struct mbuf
**);
160 static void ena_start_xmit(struct ifnet
*, struct ifaltq_subque
*);
161 static int ena_calc_io_queue_num(struct ena_adapter
*,
162 struct ena_com_dev_get_features_ctx
*);
163 static int ena_calc_queue_size(struct ena_adapter
*, uint16_t *,
164 uint16_t *, struct ena_com_dev_get_features_ctx
*);
165 static int ena_rss_init_default(struct ena_adapter
*);
166 static void ena_rss_init_default_deferred(void *);
167 static void ena_config_host_info(struct ena_com_dev
*);
168 static int ena_attach(device_t
);
169 static int ena_detach(device_t
);
170 static int ena_device_init(struct ena_adapter
*, device_t
,
171 struct ena_com_dev_get_features_ctx
*, int *);
172 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter
*,
174 static void ena_update_on_link_change(void *, struct ena_admin_aenq_entry
*);
175 static void unimplemented_aenq_handler(void *,
176 struct ena_admin_aenq_entry
*);
177 static void ena_timer_service(void *);
179 static char ena_version
[] = DEVICE_NAME DRV_MODULE_NAME
" v" DRV_MODULE_VERSION
;
181 static SYSCTL_NODE(_hw
, OID_AUTO
, ena
, CTLFLAG_RD
, 0, "ENA driver parameters");
184 * Logging level for changing verbosity of the output
186 int ena_log_level
= ENA_ALERT
| ENA_WARNING
;
187 TUNABLE_INT("hw.ena.ena_log_level", &ena_log_level
);
188 SYSCTL_INT(_hw_ena
, OID_AUTO
, log_level
, CTLFLAG_RW
,
189 &ena_log_level
, 0, "Logging level indicating verbosity of the logs");
191 static ena_vendor_info_t ena_vendor_info_array
[] = {
192 { PCI_VENDOR_ID_AMAZON
, PCI_DEV_ID_ENA_PF
, 0},
193 { PCI_VENDOR_ID_AMAZON
, PCI_DEV_ID_ENA_LLQ_PF
, 0},
194 { PCI_VENDOR_ID_AMAZON
, PCI_DEV_ID_ENA_VF
, 0},
195 { PCI_VENDOR_ID_AMAZON
, PCI_DEV_ID_ENA_LLQ_VF
, 0},
201 * Contains pointers to event handlers, e.g. link state chage.
203 static struct ena_aenq_handlers aenq_handlers
;
206 ena_dmamap_callback(void *arg
, bus_dma_segment_t
*segs
, int nseg
, int error
)
210 *(bus_addr_t
*) arg
= segs
[0].ds_addr
;
214 ena_dma_alloc(device_t dmadev
, bus_size_t size
,
215 ena_mem_handle_t
*dma
, int mapflags
)
217 struct ena_adapter
* adapter
= device_get_softc(dmadev
);
219 uint64_t dma_space_addr
;
222 maxsize
= ((size
- 1) / PAGE_SIZE
+ 1) * PAGE_SIZE
;
224 dma_space_addr
= ENA_DMA_BIT_MASK(adapter
->dma_width
);
225 if (unlikely(dma_space_addr
== 0))
226 dma_space_addr
= BUS_SPACE_MAXADDR
;
228 error
= bus_dma_tag_create(bus_get_dma_tag(dmadev
), /* parent */
229 8, 0, /* alignment, bounds */
230 dma_space_addr
, /* lowaddr of exclusion window */
231 BUS_SPACE_MAXADDR
,/* highaddr of exclusion window */
232 maxsize
, /* maxsize */
234 maxsize
, /* maxsegsize */
235 BUS_DMA_ALLOCNOW
, /* flags */
237 if (unlikely(error
!= 0)) {
238 ena_trace(ENA_ALERT
, "bus_dma_tag_create failed: %d\n", error
);
242 error
= bus_dmamem_alloc(dma
->tag
, (void**) &dma
->vaddr
,
243 BUS_DMA_COHERENT
| BUS_DMA_ZERO
, &dma
->map
);
244 if (unlikely(error
!= 0)) {
245 ena_trace(ENA_ALERT
, "bus_dmamem_alloc(%ju) failed: %d\n",
246 (uintmax_t)size
, error
);
247 goto fail_map_create
;
251 error
= bus_dmamap_load(dma
->tag
, dma
->map
, dma
->vaddr
,
252 size
, ena_dmamap_callback
, &dma
->paddr
, mapflags
);
253 if (unlikely((error
!= 0) || (dma
->paddr
== 0))) {
254 ena_trace(ENA_ALERT
, ": bus_dmamap_load failed: %d\n", error
);
261 bus_dmamem_free(dma
->tag
, dma
->vaddr
, dma
->map
);
263 bus_dma_tag_destroy(dma
->tag
);
271 ena_allocate_pci_resources(struct ena_adapter
* adapter
)
273 device_t pdev
= adapter
->pdev
;
276 rid
= PCIR_BAR(ENA_REG_BAR
);
277 adapter
->memory
= NULL
;
278 adapter
->registers
= bus_alloc_resource_any(pdev
, SYS_RES_MEMORY
,
280 if (unlikely(adapter
->registers
== NULL
)) {
281 device_printf(pdev
, "Unable to allocate bus resource: "
290 ena_free_pci_resources(struct ena_adapter
*adapter
)
292 device_t pdev
= adapter
->pdev
;
294 if (adapter
->memory
!= NULL
) {
295 bus_release_resource(pdev
, SYS_RES_MEMORY
,
296 PCIR_BAR(ENA_MEM_BAR
), adapter
->memory
);
299 if (adapter
->registers
!= NULL
) {
300 bus_release_resource(pdev
, SYS_RES_MEMORY
,
301 PCIR_BAR(ENA_REG_BAR
), adapter
->registers
);
306 ena_probe(device_t dev
)
308 ena_vendor_info_t
*ent
;
309 char adapter_name
[60];
310 uint16_t pci_vendor_id
= 0;
311 uint16_t pci_device_id
= 0;
313 pci_vendor_id
= pci_get_vendor(dev
);
314 pci_device_id
= pci_get_device(dev
);
316 ent
= ena_vendor_info_array
;
317 while (ent
->vendor_id
!= 0) {
318 if ((pci_vendor_id
== ent
->vendor_id
) &&
319 (pci_device_id
== ent
->device_id
)) {
320 ena_trace(ENA_DBG
, "vendor=%x device=%x ",
321 pci_vendor_id
, pci_device_id
);
323 ksprintf(adapter_name
, DEVICE_DESC
);
324 device_set_desc_copy(dev
, adapter_name
);
325 return (BUS_PROBE_DEFAULT
);
336 ena_change_mtu(if_t ifp
, int new_mtu
)
338 struct ena_adapter
*adapter
= ifp
->if_softc
;
341 if ((new_mtu
> adapter
->max_mtu
) || (new_mtu
< ENA_MIN_MTU
)) {
342 device_printf(adapter
->pdev
, "Invalid MTU setting. "
343 "new_mtu: %d max mtu: %d min mtu: %d\n",
344 new_mtu
, adapter
->max_mtu
, ENA_MIN_MTU
);
348 rc
= ena_com_set_dev_mtu(adapter
->ena_dev
, new_mtu
);
349 if (likely(rc
== 0)) {
350 ena_trace(ENA_DBG
, "set MTU to %d\n", new_mtu
);
351 ifp
->if_mtu
= new_mtu
;
353 device_printf(adapter
->pdev
, "Failed to set MTU to %d\n",
360 #if 0 /* XXX swildner counters */
362 ena_alloc_counters(counter_u64_t
*begin
, int size
)
364 counter_u64_t
*end
= (counter_u64_t
*)((char *)begin
+ size
);
366 for (; begin
< end
; ++begin
)
367 *begin
= counter_u64_alloc(M_WAITOK
);
371 ena_free_counters(counter_u64_t
*begin
, int size
)
373 counter_u64_t
*end
= (counter_u64_t
*)((char *)begin
+ size
);
375 for (; begin
< end
; ++begin
)
376 counter_u64_free(*begin
);
380 ena_reset_counters(counter_u64_t
*begin
, int size
)
382 counter_u64_t
*end
= (counter_u64_t
*)((char *)begin
+ size
);
384 for (; begin
< end
; ++begin
)
385 counter_u64_zero(*begin
);
390 ena_init_io_rings_common(struct ena_adapter
*adapter
, struct ena_ring
*ring
,
395 ring
->adapter
= adapter
;
396 ring
->ena_dev
= adapter
->ena_dev
;
400 ena_init_io_rings(struct ena_adapter
*adapter
)
402 struct ena_com_dev
*ena_dev
;
403 struct ena_ring
*txr
, *rxr
;
407 ena_dev
= adapter
->ena_dev
;
409 for (i
= 0; i
< adapter
->num_queues
; i
++) {
410 txr
= &adapter
->tx_ring
[i
];
411 rxr
= &adapter
->rx_ring
[i
];
413 /* TX/RX common ring state */
414 ena_init_io_rings_common(adapter
, txr
, i
);
415 ena_init_io_rings_common(adapter
, rxr
, i
);
417 /* TX specific ring state */
418 txr
->ring_size
= adapter
->tx_ring_size
;
419 txr
->tx_max_header_size
= ena_dev
->tx_max_header_size
;
420 txr
->tx_mem_queue_type
= ena_dev
->tx_mem_queue_type
;
421 txr
->smoothed_interval
=
422 ena_com_get_nonadaptive_moderation_interval_tx(ena_dev
);
424 #if 0 /* XXX swildner counters */
425 /* Alloc TX statistics. */
426 ena_alloc_counters((counter_u64_t
*)&txr
->tx_stats
,
427 sizeof(txr
->tx_stats
));
430 /* RX specific ring state */
431 rxr
->ring_size
= adapter
->rx_ring_size
;
432 rxr
->smoothed_interval
=
433 ena_com_get_nonadaptive_moderation_interval_rx(ena_dev
);
435 #if 0 /* XXX swildner counters */
436 /* Alloc RX statistics. */
437 ena_alloc_counters((counter_u64_t
*)&rxr
->rx_stats
,
438 sizeof(rxr
->rx_stats
));
441 /* Initialize locks */
442 ksnprintf(txr
->lock_name
, nitems(txr
->lock_name
), "%s:tx(%d)",
443 device_get_nameunit(adapter
->pdev
), i
);
444 ksnprintf(rxr
->lock_name
, nitems(rxr
->lock_name
), "%s:rx(%d)",
445 device_get_nameunit(adapter
->pdev
), i
);
447 lockinit(&txr
->ring_lock
, txr
->lock_name
, 0, LK_CANRECURSE
);
448 lockinit(&rxr
->ring_lock
, rxr
->lock_name
, 0, LK_CANRECURSE
);
450 que
= &adapter
->que
[i
];
451 que
->adapter
= adapter
;
459 rxr
->empty_rx_queue
= 0;
464 ena_free_io_ring_resources(struct ena_adapter
*adapter
, unsigned int qid
)
466 struct ena_ring
*txr
= &adapter
->tx_ring
[qid
];
467 struct ena_ring
*rxr
= &adapter
->rx_ring
[qid
];
469 #if 0 /* XXX swildner counters */
470 ena_free_counters((counter_u64_t
*)&txr
->tx_stats
,
471 sizeof(txr
->tx_stats
));
472 ena_free_counters((counter_u64_t
*)&rxr
->rx_stats
,
473 sizeof(rxr
->rx_stats
));
476 lockuninit(&txr
->ring_lock
);
477 lockuninit(&rxr
->ring_lock
);
481 ena_free_all_io_rings_resources(struct ena_adapter
*adapter
)
485 for (i
= 0; i
< adapter
->num_queues
; i
++)
486 ena_free_io_ring_resources(adapter
, i
);
491 ena_setup_tx_dma_tag(struct ena_adapter
*adapter
)
495 /* Create DMA tag for Tx buffers */
496 ret
= bus_dma_tag_create(bus_get_dma_tag(adapter
->pdev
),
497 1, 0, /* alignment, bounds */
498 ENA_DMA_BIT_MASK(adapter
->dma_width
), /* lowaddr of excl window */
499 BUS_SPACE_MAXADDR
, /* highaddr of excl window */
500 ENA_TSO_MAXSIZE
, /* maxsize */
501 ENA_BUS_DMA_SEGS
, /* nsegments */
502 ENA_TSO_MAXSIZE
, /* maxsegsize */
504 &adapter
->tx_buf_tag
);
510 ena_free_tx_dma_tag(struct ena_adapter
*adapter
)
514 ret
= bus_dma_tag_destroy(adapter
->tx_buf_tag
);
516 if (likely(ret
== 0))
517 adapter
->tx_buf_tag
= NULL
;
523 ena_setup_rx_dma_tag(struct ena_adapter
*adapter
)
527 /* Create DMA tag for Rx buffers*/
528 ret
= bus_dma_tag_create(bus_get_dma_tag(adapter
->pdev
), /* parent */
529 1, 0, /* alignment, bounds */
530 ENA_DMA_BIT_MASK(adapter
->dma_width
), /* lowaddr of excl window */
531 BUS_SPACE_MAXADDR
, /* highaddr of excl window */
532 MJUM16BYTES
, /* maxsize */
533 adapter
->max_rx_sgl_size
, /* nsegments */
534 MJUM16BYTES
, /* maxsegsize */
536 &adapter
->rx_buf_tag
);
542 ena_free_rx_dma_tag(struct ena_adapter
*adapter
)
546 ret
= bus_dma_tag_destroy(adapter
->rx_buf_tag
);
548 if (likely(ret
== 0))
549 adapter
->rx_buf_tag
= NULL
;
555 * ena_setup_tx_resources - allocate Tx resources (Descriptors)
556 * @adapter: network interface device structure
559 * Returns 0 on success, otherwise on failure.
562 ena_setup_tx_resources(struct ena_adapter
*adapter
, int qid
)
564 struct ena_que
*que
= &adapter
->que
[qid
];
565 struct ena_ring
*tx_ring
= que
->tx_ring
;
571 size
= sizeof(struct ena_tx_buffer
) * tx_ring
->ring_size
;
573 tx_ring
->tx_buffer_info
= kmalloc(size
, M_DEVBUF
, M_NOWAIT
| M_ZERO
);
574 if (unlikely(tx_ring
->tx_buffer_info
== NULL
))
577 size
= sizeof(uint16_t) * tx_ring
->ring_size
;
578 tx_ring
->free_tx_ids
= kmalloc(size
, M_DEVBUF
, M_NOWAIT
| M_ZERO
);
579 if (unlikely(tx_ring
->free_tx_ids
== NULL
))
580 goto err_buf_info_free
;
582 /* Req id stack for TX OOO completions */
583 for (i
= 0; i
< tx_ring
->ring_size
; i
++)
584 tx_ring
->free_tx_ids
[i
] = i
;
586 #if 0 /* XXX swildner counters */
587 /* Reset TX statistics. */
588 ena_reset_counters((counter_u64_t
*)&tx_ring
->tx_stats
,
589 sizeof(tx_ring
->tx_stats
));
592 tx_ring
->next_to_use
= 0;
593 tx_ring
->next_to_clean
= 0;
595 /* ... and create the buffer DMA maps */
596 for (i
= 0; i
< tx_ring
->ring_size
; i
++) {
597 err
= bus_dmamap_create(adapter
->tx_buf_tag
, 0,
598 &tx_ring
->tx_buffer_info
[i
].map
);
599 if (unlikely(err
!= 0)) {
601 "Unable to create Tx DMA map for buffer %d\n", i
);
602 goto err_buf_info_unmap
;
610 bus_dmamap_destroy(adapter
->tx_buf_tag
,
611 tx_ring
->tx_buffer_info
[i
].map
);
613 kfree(tx_ring
->free_tx_ids
, M_DEVBUF
);
614 tx_ring
->free_tx_ids
= NULL
;
616 kfree(tx_ring
->tx_buffer_info
, M_DEVBUF
);
617 tx_ring
->tx_buffer_info
= NULL
;
623 * ena_free_tx_resources - Free Tx Resources per Queue
624 * @adapter: network interface device structure
627 * Free all transmit software resources
630 ena_free_tx_resources(struct ena_adapter
*adapter
, int qid
)
632 struct ena_ring
*tx_ring
= &adapter
->tx_ring
[qid
];
634 ENA_RING_MTX_LOCK(tx_ring
);
636 /* Free buffer DMA maps, */
637 for (int i
= 0; i
< tx_ring
->ring_size
; i
++) {
638 m_freem(tx_ring
->tx_buffer_info
[i
].mbuf
);
639 tx_ring
->tx_buffer_info
[i
].mbuf
= NULL
;
640 bus_dmamap_unload(adapter
->tx_buf_tag
,
641 tx_ring
->tx_buffer_info
[i
].map
);
642 bus_dmamap_destroy(adapter
->tx_buf_tag
,
643 tx_ring
->tx_buffer_info
[i
].map
);
645 ENA_RING_MTX_UNLOCK(tx_ring
);
647 /* And free allocated memory. */
648 kfree(tx_ring
->tx_buffer_info
, M_DEVBUF
);
649 tx_ring
->tx_buffer_info
= NULL
;
651 kfree(tx_ring
->free_tx_ids
, M_DEVBUF
);
652 tx_ring
->free_tx_ids
= NULL
;
656 * ena_setup_all_tx_resources - allocate all queues Tx resources
657 * @adapter: network interface device structure
659 * Returns 0 on success, otherwise on failure.
662 ena_setup_all_tx_resources(struct ena_adapter
*adapter
)
666 for (i
= 0; i
< adapter
->num_queues
; i
++) {
667 rc
= ena_setup_tx_resources(adapter
, i
);
669 device_printf(adapter
->pdev
,
670 "Allocation for Tx Queue %u failed\n", i
);
678 /* Rewind the index freeing the rings as we go */
680 ena_free_tx_resources(adapter
, i
);
685 * ena_free_all_tx_resources - Free Tx Resources for All Queues
686 * @adapter: network interface device structure
688 * Free all transmit software resources
691 ena_free_all_tx_resources(struct ena_adapter
*adapter
)
695 for (i
= 0; i
< adapter
->num_queues
; i
++)
696 ena_free_tx_resources(adapter
, i
);
700 validate_rx_req_id(struct ena_ring
*rx_ring
, uint16_t req_id
)
702 if (likely(req_id
< rx_ring
->ring_size
))
705 device_printf(rx_ring
->adapter
->pdev
, "Invalid rx req_id: %hu\n",
707 IFNET_STAT_INC(rx_ring
->adapter
->ifp
, ierrors
, 1);
708 #if 0 /* XXX swildner counters */
709 counter_u64_add(rx_ring
->rx_stats
.bad_req_id
, 1);
712 /* Trigger device reset */
713 rx_ring
->adapter
->reset_reason
= ENA_REGS_RESET_INV_RX_REQ_ID
;
714 rx_ring
->adapter
->trigger_reset
= true;
720 * ena_setup_rx_resources - allocate Rx resources (Descriptors)
721 * @adapter: network interface device structure
724 * Returns 0 on success, otherwise on failure.
727 ena_setup_rx_resources(struct ena_adapter
*adapter
, unsigned int qid
)
729 struct ena_que
*que
= &adapter
->que
[qid
];
730 struct ena_ring
*rx_ring
= que
->rx_ring
;
736 size
= sizeof(struct ena_rx_buffer
) * rx_ring
->ring_size
;
739 * Alloc extra element so in rx path
740 * we can always prefetch rx_info + 1
742 size
+= sizeof(struct ena_rx_buffer
);
744 rx_ring
->rx_buffer_info
= kmalloc(size
, M_DEVBUF
, M_WAITOK
| M_ZERO
);
746 size
= sizeof(uint16_t) * rx_ring
->ring_size
;
747 rx_ring
->free_rx_ids
= kmalloc(size
, M_DEVBUF
, M_WAITOK
);
749 for (i
= 0; i
< rx_ring
->ring_size
; i
++)
750 rx_ring
->free_rx_ids
[i
] = i
;
752 #if 0 /* XXX swildner counters */
753 /* Reset RX statistics. */
754 ena_reset_counters((counter_u64_t
*)&rx_ring
->rx_stats
,
755 sizeof(rx_ring
->rx_stats
));
758 rx_ring
->next_to_clean
= 0;
759 rx_ring
->next_to_use
= 0;
761 /* ... and create the buffer DMA maps */
762 for (i
= 0; i
< rx_ring
->ring_size
; i
++) {
763 err
= bus_dmamap_create(adapter
->rx_buf_tag
, 0,
764 &(rx_ring
->rx_buffer_info
[i
].map
));
767 "Unable to create Rx DMA map for buffer %d\n", i
);
768 goto err_buf_info_unmap
;
773 /* Create LRO for the ring */
774 if ((adapter
->ifp
->if_capenable
& IFCAP_LRO
) != 0) {
775 int err
= tcp_lro_init(&rx_ring
->lro
);
777 device_printf(adapter
->pdev
,
778 "LRO[%d] Initialization failed!\n", qid
);
781 "RX Soft LRO[%d] Initialized\n", qid
);
782 rx_ring
->lro
.ifp
= adapter
->ifp
;
787 /* Allocate taskqueues */
788 TASK_INIT(&rx_ring
->cmpl_task
, 0, ena_deferred_rx_cleanup
, rx_ring
);
789 rx_ring
->cmpl_tq
= taskqueue_create("ena RX completion", M_WAITOK
,
790 taskqueue_thread_enqueue
, &rx_ring
->cmpl_tq
);
792 /* RSS set cpu for thread */
794 CPU_SETOF(que
->cpu
, &cpu_mask
);
795 taskqueue_start_threads_cpuset(&rx_ring
->cmpl_tq
, 1, PI_NET
, &cpu_mask
,
796 "%s rx_ring cmpl (bucket %d)",
797 device_get_nameunit(adapter
->pdev
), que
->cpu
);
799 taskqueue_start_threads(&rx_ring
->cmpl_tq
, 1, TDPRI_KERN_DAEMON
, -1,
800 "%s rx_ring cmpl", device_get_nameunit(adapter
->pdev
));
807 bus_dmamap_destroy(adapter
->rx_buf_tag
,
808 rx_ring
->rx_buffer_info
[i
].map
);
811 kfree(rx_ring
->free_rx_ids
, M_DEVBUF
);
812 rx_ring
->free_rx_ids
= NULL
;
813 kfree(rx_ring
->rx_buffer_info
, M_DEVBUF
);
814 rx_ring
->rx_buffer_info
= NULL
;
819 * ena_free_rx_resources - Free Rx Resources
820 * @adapter: network interface device structure
823 * Free all receive software resources
826 ena_free_rx_resources(struct ena_adapter
*adapter
, unsigned int qid
)
828 struct ena_ring
*rx_ring
= &adapter
->rx_ring
[qid
];
830 while (taskqueue_cancel(rx_ring
->cmpl_tq
, &rx_ring
->cmpl_task
, NULL
) != 0)
831 taskqueue_drain(rx_ring
->cmpl_tq
, &rx_ring
->cmpl_task
);
833 taskqueue_free(rx_ring
->cmpl_tq
);
835 /* Free buffer DMA maps, */
836 for (int i
= 0; i
< rx_ring
->ring_size
; i
++) {
837 m_freem(rx_ring
->rx_buffer_info
[i
].mbuf
);
838 rx_ring
->rx_buffer_info
[i
].mbuf
= NULL
;
839 bus_dmamap_unload(adapter
->rx_buf_tag
,
840 rx_ring
->rx_buffer_info
[i
].map
);
841 bus_dmamap_destroy(adapter
->rx_buf_tag
,
842 rx_ring
->rx_buffer_info
[i
].map
);
846 /* free LRO resources, */
847 tcp_lro_free(&rx_ring
->lro
);
850 /* free allocated memory */
851 kfree(rx_ring
->rx_buffer_info
, M_DEVBUF
);
852 rx_ring
->rx_buffer_info
= NULL
;
854 kfree(rx_ring
->free_rx_ids
, M_DEVBUF
);
855 rx_ring
->free_rx_ids
= NULL
;
859 * ena_setup_all_rx_resources - allocate all queues Rx resources
860 * @adapter: network interface device structure
862 * Returns 0 on success, otherwise on failure.
865 ena_setup_all_rx_resources(struct ena_adapter
*adapter
)
869 for (i
= 0; i
< adapter
->num_queues
; i
++) {
870 rc
= ena_setup_rx_resources(adapter
, i
);
872 device_printf(adapter
->pdev
,
873 "Allocation for Rx Queue %u failed\n", i
);
880 /* rewind the index freeing the rings as we go */
882 ena_free_rx_resources(adapter
, i
);
887 * ena_free_all_rx_resources - Free Rx resources for all queues
888 * @adapter: network interface device structure
890 * Free all receive software resources
893 ena_free_all_rx_resources(struct ena_adapter
*adapter
)
897 for (i
= 0; i
< adapter
->num_queues
; i
++)
898 ena_free_rx_resources(adapter
, i
);
902 ena_alloc_rx_mbuf(struct ena_adapter
*adapter
,
903 struct ena_ring
*rx_ring
, struct ena_rx_buffer
*rx_info
)
905 struct ena_com_buf
*ena_buf
;
906 bus_dma_segment_t segs
[1];
910 /* if previous allocated frag is not used */
911 if (unlikely(rx_info
->mbuf
!= NULL
))
914 /* Get mbuf using UMA allocator */
915 rx_info
->mbuf
= m_getjcl(M_NOWAIT
, MT_DATA
, M_PKTHDR
, MJUMPAGESIZE
);
917 if (unlikely(rx_info
->mbuf
== NULL
)) {
918 #if 0 /* XXX swildner counters */
919 counter_u64_add(rx_ring
->rx_stats
.mjum_alloc_fail
, 1);
921 rx_info
->mbuf
= m_getcl(M_NOWAIT
, MT_DATA
, M_PKTHDR
);
922 if (unlikely(rx_info
->mbuf
== NULL
)) {
923 IFNET_STAT_INC(rx_ring
->adapter
->ifp
, ierrors
, 1);
924 #if 0 /* XXX swildner counters */
925 counter_u64_add(rx_ring
->rx_stats
.mbuf_alloc_fail
, 1);
934 rx_info
->mbuf
->m_pkthdr
.len
= rx_info
->mbuf
->m_len
= mlen
;
936 /* Map packets for DMA */
937 ena_trace(ENA_DBG
| ENA_RSC
| ENA_RXPTH
,
938 "Using tag %p for buffers' DMA mapping, mbuf %p len: %d",
939 adapter
->rx_buf_tag
,rx_info
->mbuf
, rx_info
->mbuf
->m_len
);
940 error
= bus_dmamap_load_mbuf_segment(adapter
->rx_buf_tag
, rx_info
->map
,
941 rx_info
->mbuf
, segs
, 1, &nsegs
, BUS_DMA_NOWAIT
);
942 if (unlikely((error
!= 0) || (nsegs
!= 1))) {
943 ena_trace(ENA_WARNING
, "failed to map mbuf, error: %d, "
944 "nsegs: %d\n", error
, nsegs
);
945 IFNET_STAT_INC(rx_ring
->adapter
->ifp
, ierrors
, 1);
946 #if 0 /* XXX swildner counters */
947 counter_u64_add(rx_ring
->rx_stats
.dma_mapping_err
, 1);
953 bus_dmamap_sync(adapter
->rx_buf_tag
, rx_info
->map
, BUS_DMASYNC_PREREAD
);
955 ena_buf
= &rx_info
->ena_buf
;
956 ena_buf
->paddr
= segs
[0].ds_addr
;
959 ena_trace(ENA_DBG
| ENA_RSC
| ENA_RXPTH
,
960 "ALLOC RX BUF: mbuf %p, rx_info %p, len %d, paddr %#jx\n",
961 rx_info
->mbuf
, rx_info
,ena_buf
->len
, (uintmax_t)ena_buf
->paddr
);
966 m_freem(rx_info
->mbuf
);
967 rx_info
->mbuf
= NULL
;
972 ena_free_rx_mbuf(struct ena_adapter
*adapter
, struct ena_ring
*rx_ring
,
973 struct ena_rx_buffer
*rx_info
)
976 if (rx_info
->mbuf
== NULL
) {
977 ena_trace(ENA_WARNING
, "Trying to free unallocated buffer\n");
981 bus_dmamap_unload(adapter
->rx_buf_tag
, rx_info
->map
);
982 m_freem(rx_info
->mbuf
);
983 rx_info
->mbuf
= NULL
;
987 * ena_refill_rx_bufs - Refills ring with descriptors
988 * @rx_ring: the ring which we want to feed with free descriptors
989 * @num: number of descriptors to refill
990 * Refills the ring with newly allocated DMA-mapped mbufs for receiving
993 ena_refill_rx_bufs(struct ena_ring
*rx_ring
, uint32_t num
)
995 struct ena_adapter
*adapter
= rx_ring
->adapter
;
996 uint16_t next_to_use
, req_id
;
1000 ena_trace(ENA_DBG
| ENA_RXPTH
| ENA_RSC
, "refill qid: %d",
1003 next_to_use
= rx_ring
->next_to_use
;
1005 for (i
= 0; i
< num
; i
++) {
1006 struct ena_rx_buffer
*rx_info
;
1008 ena_trace(ENA_DBG
| ENA_RXPTH
| ENA_RSC
,
1009 "RX buffer - next to use: %d", next_to_use
);
1011 req_id
= rx_ring
->free_rx_ids
[next_to_use
];
1012 rc
= validate_rx_req_id(rx_ring
, req_id
);
1013 if (unlikely(rc
!= 0))
1016 rx_info
= &rx_ring
->rx_buffer_info
[req_id
];
1018 rc
= ena_alloc_rx_mbuf(adapter
, rx_ring
, rx_info
);
1019 if (unlikely(rc
!= 0)) {
1020 ena_trace(ENA_WARNING
,
1021 "failed to alloc buffer for rx queue %d\n",
1025 rc
= ena_com_add_single_rx_desc(rx_ring
->ena_com_io_sq
,
1026 &rx_info
->ena_buf
, req_id
);
1027 if (unlikely(rc
!= 0)) {
1028 ena_trace(ENA_WARNING
,
1029 "failed to add buffer for rx queue %d\n",
1033 next_to_use
= ENA_RX_RING_IDX_NEXT(next_to_use
,
1034 rx_ring
->ring_size
);
1037 if (unlikely(i
< num
)) {
1038 IFNET_STAT_INC(rx_ring
->adapter
->ifp
, ierrors
, 1);
1039 #if 0 /* XXX swildner counters */
1040 counter_u64_add(rx_ring
->rx_stats
.refil_partial
, 1);
1042 ena_trace(ENA_WARNING
,
1043 "refilled rx qid %d with only %d mbufs (from %d)\n",
1044 rx_ring
->qid
, i
, num
);
1047 if (likely(i
!= 0)) {
1049 ena_com_write_sq_doorbell(rx_ring
->ena_com_io_sq
);
1051 rx_ring
->next_to_use
= next_to_use
;
1056 ena_free_rx_bufs(struct ena_adapter
*adapter
, unsigned int qid
)
1058 struct ena_ring
*rx_ring
= &adapter
->rx_ring
[qid
];
1061 for (i
= 0; i
< rx_ring
->ring_size
; i
++) {
1062 struct ena_rx_buffer
*rx_info
= &rx_ring
->rx_buffer_info
[i
];
1064 if (rx_info
->mbuf
!= NULL
)
1065 ena_free_rx_mbuf(adapter
, rx_ring
, rx_info
);
1070 * ena_refill_all_rx_bufs - allocate all queues Rx buffers
1071 * @adapter: network interface device structure
1075 ena_refill_all_rx_bufs(struct ena_adapter
*adapter
)
1077 struct ena_ring
*rx_ring
;
1078 int i
, rc
, bufs_num
;
1080 for (i
= 0; i
< adapter
->num_queues
; i
++) {
1081 rx_ring
= &adapter
->rx_ring
[i
];
1082 bufs_num
= rx_ring
->ring_size
- 1;
1083 rc
= ena_refill_rx_bufs(rx_ring
, bufs_num
);
1085 if (unlikely(rc
!= bufs_num
))
1086 ena_trace(ENA_WARNING
, "refilling Queue %d failed. "
1087 "Allocated %d buffers from: %d\n", i
, rc
, bufs_num
);
1092 ena_free_all_rx_bufs(struct ena_adapter
*adapter
)
1096 for (i
= 0; i
< adapter
->num_queues
; i
++)
1097 ena_free_rx_bufs(adapter
, i
);
1101 * ena_free_tx_bufs - Free Tx Buffers per Queue
1102 * @adapter: network interface device structure
1106 ena_free_tx_bufs(struct ena_adapter
*adapter
, unsigned int qid
)
1108 bool print_once
= true;
1109 struct ena_ring
*tx_ring
= &adapter
->tx_ring
[qid
];
1111 ENA_RING_MTX_LOCK(tx_ring
);
1112 for (int i
= 0; i
< tx_ring
->ring_size
; i
++) {
1113 struct ena_tx_buffer
*tx_info
= &tx_ring
->tx_buffer_info
[i
];
1115 if (tx_info
->mbuf
== NULL
)
1119 device_printf(adapter
->pdev
,
1120 "free uncompleted tx mbuf qid %d idx 0x%x",
1125 "free uncompleted tx mbuf qid %d idx 0x%x",
1129 bus_dmamap_unload(adapter
->tx_buf_tag
, tx_info
->map
);
1130 m_free(tx_info
->mbuf
);
1131 tx_info
->mbuf
= NULL
;
1133 ENA_RING_MTX_UNLOCK(tx_ring
);
1137 ena_free_all_tx_bufs(struct ena_adapter
*adapter
)
1140 for (int i
= 0; i
< adapter
->num_queues
; i
++)
1141 ena_free_tx_bufs(adapter
, i
);
1145 ena_destroy_all_tx_queues(struct ena_adapter
*adapter
)
1150 for (i
= 0; i
< adapter
->num_queues
; i
++) {
1151 ena_qid
= ENA_IO_TXQ_IDX(i
);
1152 ena_com_destroy_io_queue(adapter
->ena_dev
, ena_qid
);
1157 ena_destroy_all_rx_queues(struct ena_adapter
*adapter
)
1162 for (i
= 0; i
< adapter
->num_queues
; i
++) {
1163 ena_qid
= ENA_IO_RXQ_IDX(i
);
1164 ena_com_destroy_io_queue(adapter
->ena_dev
, ena_qid
);
1169 ena_destroy_all_io_queues(struct ena_adapter
*adapter
)
1171 ena_destroy_all_tx_queues(adapter
);
1172 ena_destroy_all_rx_queues(adapter
);
1176 validate_tx_req_id(struct ena_ring
*tx_ring
, uint16_t req_id
)
1178 struct ena_adapter
*adapter
= tx_ring
->adapter
;
1179 struct ena_tx_buffer
*tx_info
= NULL
;
1181 if (likely(req_id
< tx_ring
->ring_size
)) {
1182 tx_info
= &tx_ring
->tx_buffer_info
[req_id
];
1183 if (tx_info
->mbuf
!= NULL
)
1187 if (tx_info
->mbuf
== NULL
)
1188 device_printf(adapter
->pdev
,
1189 "tx_info doesn't have valid mbuf\n");
1191 device_printf(adapter
->pdev
, "Invalid req_id: %hu\n", req_id
);
1193 IFNET_STAT_INC(tx_ring
->adapter
->ifp
, oerrors
, 1);
1194 #if 0 /* XXX swildner counters */
1195 counter_u64_add(tx_ring
->tx_stats
.bad_req_id
, 1);
1202 ena_create_io_queues(struct ena_adapter
*adapter
)
1204 struct ena_com_dev
*ena_dev
= adapter
->ena_dev
;
1205 struct ena_com_create_io_ctx ctx
;
1206 struct ena_ring
*ring
;
1208 uint32_t msix_vector
;
1211 /* Create TX queues */
1212 for (i
= 0; i
< adapter
->num_queues
; i
++) {
1213 msix_vector
= ENA_IO_IRQ_IDX(i
);
1214 ena_qid
= ENA_IO_TXQ_IDX(i
);
1215 ctx
.mem_queue_type
= ena_dev
->tx_mem_queue_type
;
1216 ctx
.direction
= ENA_COM_IO_QUEUE_DIRECTION_TX
;
1217 ctx
.queue_size
= adapter
->tx_ring_size
;
1218 ctx
.msix_vector
= msix_vector
;
1220 rc
= ena_com_create_io_queue(ena_dev
, &ctx
);
1222 device_printf(adapter
->pdev
,
1223 "Failed to create io TX queue #%d rc: %d\n", i
, rc
);
1226 ring
= &adapter
->tx_ring
[i
];
1227 rc
= ena_com_get_io_handlers(ena_dev
, ena_qid
,
1228 &ring
->ena_com_io_sq
,
1229 &ring
->ena_com_io_cq
);
1231 device_printf(adapter
->pdev
,
1232 "Failed to get TX queue handlers. TX queue num"
1233 " %d rc: %d\n", i
, rc
);
1234 ena_com_destroy_io_queue(ena_dev
, ena_qid
);
1239 /* Create RX queues */
1240 for (i
= 0; i
< adapter
->num_queues
; i
++) {
1241 msix_vector
= ENA_IO_IRQ_IDX(i
);
1242 ena_qid
= ENA_IO_RXQ_IDX(i
);
1243 ctx
.mem_queue_type
= ENA_ADMIN_PLACEMENT_POLICY_HOST
;
1244 ctx
.direction
= ENA_COM_IO_QUEUE_DIRECTION_RX
;
1245 ctx
.queue_size
= adapter
->rx_ring_size
;
1246 ctx
.msix_vector
= msix_vector
;
1248 rc
= ena_com_create_io_queue(ena_dev
, &ctx
);
1249 if (unlikely(rc
!= 0)) {
1250 device_printf(adapter
->pdev
,
1251 "Failed to create io RX queue[%d] rc: %d\n", i
, rc
);
1255 ring
= &adapter
->rx_ring
[i
];
1256 rc
= ena_com_get_io_handlers(ena_dev
, ena_qid
,
1257 &ring
->ena_com_io_sq
,
1258 &ring
->ena_com_io_cq
);
1259 if (unlikely(rc
!= 0)) {
1260 device_printf(adapter
->pdev
,
1261 "Failed to get RX queue handlers. RX queue num"
1262 " %d rc: %d\n", i
, rc
);
1263 ena_com_destroy_io_queue(ena_dev
, ena_qid
);
1272 ena_com_destroy_io_queue(ena_dev
, ENA_IO_RXQ_IDX(i
));
1273 i
= adapter
->num_queues
;
1276 ena_com_destroy_io_queue(ena_dev
, ENA_IO_TXQ_IDX(i
));
1282 * ena_tx_cleanup - clear sent packets and corresponding descriptors
1283 * @tx_ring: ring for which we want to clean packets
1285 * Once packets are sent, we ask the device in a loop for no longer used
1286 * descriptors. We find the related mbuf chain in a map (index in an array)
1287 * and free it, then update ring state.
1288 * This is performed in "endless" loop, updating ring pointers every
1289 * TX_COMMIT. The first check of free descriptor is performed before the actual
1290 * loop, then repeated at the loop end.
1293 ena_tx_cleanup(struct ena_ring
*tx_ring
)
1295 struct ena_adapter
*adapter
;
1296 struct ena_com_io_cq
* io_cq
;
1297 uint16_t next_to_clean
;
1300 unsigned int total_done
= 0;
1302 int commit
= TX_COMMIT
;
1303 int budget
= TX_BUDGET
;
1306 adapter
= tx_ring
->que
->adapter
;
1307 ena_qid
= ENA_IO_TXQ_IDX(tx_ring
->que
->id
);
1308 io_cq
= &adapter
->ena_dev
->io_cq_queues
[ena_qid
];
1309 next_to_clean
= tx_ring
->next_to_clean
;
1312 struct ena_tx_buffer
*tx_info
;
1315 rc
= ena_com_tx_comp_req_id_get(io_cq
, &req_id
);
1316 if (unlikely(rc
!= 0))
1319 rc
= validate_tx_req_id(tx_ring
, req_id
);
1320 if (unlikely(rc
!= 0))
1323 tx_info
= &tx_ring
->tx_buffer_info
[req_id
];
1325 mbuf
= tx_info
->mbuf
;
1327 tx_info
->mbuf
= NULL
;
1328 timevalclear(&tx_info
->timestamp
);
1330 if (likely(tx_info
->num_of_bufs
!= 0)) {
1331 /* Map is no longer required */
1332 bus_dmamap_unload(adapter
->tx_buf_tag
, tx_info
->map
);
1335 ena_trace(ENA_DBG
| ENA_TXPTH
, "tx: q %d mbuf %p completed",
1336 tx_ring
->qid
, mbuf
);
1340 total_done
+= tx_info
->tx_descs
;
1342 tx_ring
->free_tx_ids
[next_to_clean
] = req_id
;
1343 next_to_clean
= ENA_TX_RING_IDX_NEXT(next_to_clean
,
1344 tx_ring
->ring_size
);
1346 if (unlikely(--commit
== 0)) {
1348 /* update ring state every TX_COMMIT descriptor */
1349 tx_ring
->next_to_clean
= next_to_clean
;
1351 &adapter
->ena_dev
->io_sq_queues
[ena_qid
],
1353 ena_com_update_dev_comp_head(io_cq
);
1356 } while (likely(--budget
));
1358 work_done
= TX_BUDGET
- budget
;
1360 ena_trace(ENA_DBG
| ENA_TXPTH
, "tx: q %d done. total pkts: %d",
1361 tx_ring
->qid
, work_done
);
1363 /* If there is still something to commit update ring state */
1364 if (likely(commit
!= TX_COMMIT
)) {
1365 tx_ring
->next_to_clean
= next_to_clean
;
1366 ena_com_comp_ack(&adapter
->ena_dev
->io_sq_queues
[ena_qid
],
1368 ena_com_update_dev_comp_head(io_cq
);
1375 ena_rx_hash_mbuf(struct ena_ring
*rx_ring
, struct ena_com_rx_ctx
*ena_rx_ctx
,
1378 struct ena_adapter
*adapter
= rx_ring
->adapter
;
1380 if (likely(adapter
->rss_support
)) {
1381 //mbuf->m_pkthdr.flowid = ena_rx_ctx->hash;
1382 m_sethash(mbuf
, ena_rx_ctx
->hash
);
1384 #if 0 /* XXX rsstype doesn't seem to be needed by the network stack, we will only supply the hash. */
1385 if (ena_rx_ctx
->frag
&&
1386 (ena_rx_ctx
->l3_proto
!= ENA_ETH_IO_L3_PROTO_UNKNOWN
)) {
1387 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_OPAQUE_HASH
);
1391 switch (ena_rx_ctx
->l3_proto
) {
1392 case ENA_ETH_IO_L3_PROTO_IPV4
:
1393 switch (ena_rx_ctx
->l4_proto
) {
1394 case ENA_ETH_IO_L4_PROTO_TCP
:
1395 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_RSS_TCP_IPV4
);
1397 case ENA_ETH_IO_L4_PROTO_UDP
:
1398 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_RSS_UDP_IPV4
);
1401 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_RSS_IPV4
);
1404 case ENA_ETH_IO_L3_PROTO_IPV6
:
1405 switch (ena_rx_ctx
->l4_proto
) {
1406 case ENA_ETH_IO_L4_PROTO_TCP
:
1407 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_RSS_TCP_IPV6
);
1409 case ENA_ETH_IO_L4_PROTO_UDP
:
1410 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_RSS_UDP_IPV6
);
1413 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_RSS_IPV6
);
1416 case ENA_ETH_IO_L3_PROTO_UNKNOWN
:
1417 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_NONE
);
1420 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_OPAQUE_HASH
);
1424 //mbuf->m_pkthdr.flowid = rx_ring->qid;
1425 //M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
1426 m_sethash(mbuf
, rx_ring
->qid
);
1431 * ena_rx_mbuf - assemble mbuf from descriptors
1432 * @rx_ring: ring for which we want to clean packets
1433 * @ena_bufs: buffer info
1434 * @ena_rx_ctx: metadata for this packet(s)
1435 * @next_to_clean: ring pointer, will be updated only upon success
1439 ena_rx_mbuf(struct ena_ring
*rx_ring
, struct ena_com_rx_buf_info
*ena_bufs
,
1440 struct ena_com_rx_ctx
*ena_rx_ctx
, uint16_t *next_to_clean
)
1443 struct ena_rx_buffer
*rx_info
;
1444 struct ena_adapter
*adapter
;
1445 unsigned int descs
= ena_rx_ctx
->descs
;
1446 uint16_t ntc
, len
, req_id
, buf
= 0;
1448 ntc
= *next_to_clean
;
1449 adapter
= rx_ring
->adapter
;
1450 rx_info
= &rx_ring
->rx_buffer_info
[ntc
];
1452 if (unlikely(rx_info
->mbuf
== NULL
)) {
1453 device_printf(adapter
->pdev
, "NULL mbuf in rx_info");
1457 len
= ena_bufs
[buf
].len
;
1458 req_id
= ena_bufs
[buf
].req_id
;
1459 rx_info
= &rx_ring
->rx_buffer_info
[req_id
];
1461 ena_trace(ENA_DBG
| ENA_RXPTH
, "rx_info %p, mbuf %p, paddr %jx",
1462 rx_info
, rx_info
->mbuf
, (uintmax_t)rx_info
->ena_buf
.paddr
);
1464 mbuf
= rx_info
->mbuf
;
1465 mbuf
->m_flags
|= M_PKTHDR
;
1466 mbuf
->m_pkthdr
.len
= len
;
1468 mbuf
->m_pkthdr
.rcvif
= rx_ring
->que
->adapter
->ifp
;
1470 /* Fill mbuf with hash key and it's interpretation for optimization */
1471 ena_rx_hash_mbuf(rx_ring
, ena_rx_ctx
, mbuf
);
1473 ena_trace(ENA_DBG
| ENA_RXPTH
, "rx mbuf 0x%p, flags=0x%x, len: %d",
1474 mbuf
, mbuf
->m_flags
, mbuf
->m_pkthdr
.len
);
1476 /* DMA address is not needed anymore, unmap it */
1477 bus_dmamap_unload(rx_ring
->adapter
->rx_buf_tag
, rx_info
->map
);
1479 rx_info
->mbuf
= NULL
;
1480 rx_ring
->free_rx_ids
[ntc
] = req_id
;
1481 ntc
= ENA_RX_RING_IDX_NEXT(ntc
, rx_ring
->ring_size
);
1484 * While we have more than 1 descriptors for one rcvd packet, append
1485 * other mbufs to the main one
1489 len
= ena_bufs
[buf
].len
;
1490 req_id
= ena_bufs
[buf
].req_id
;
1491 rx_info
= &rx_ring
->rx_buffer_info
[req_id
];
1493 if (unlikely(rx_info
->mbuf
== NULL
)) {
1494 device_printf(adapter
->pdev
, "NULL mbuf in rx_info");
1496 * If one of the required mbufs was not allocated yet,
1497 * we can break there.
1498 * All earlier used descriptors will be reallocated
1499 * later and not used mbufs can be reused.
1500 * The next_to_clean pointer will not be updated in case
1501 * of an error, so caller should advance it manually
1502 * in error handling routine to keep it up to date
1509 if (unlikely(m_append(mbuf
, len
, rx_info
->mbuf
->m_data
) == 0)) {
1510 IFNET_STAT_INC(rx_ring
->adapter
->ifp
, ierrors
, 1);
1511 #if 0 /* XXX swildner counters */
1512 counter_u64_add(rx_ring
->rx_stats
.mbuf_alloc_fail
, 1);
1514 ena_trace(ENA_WARNING
, "Failed to append Rx mbuf %p",
1518 ena_trace(ENA_DBG
| ENA_RXPTH
,
1519 "rx mbuf updated. len %d", mbuf
->m_pkthdr
.len
);
1521 /* Free already appended mbuf, it won't be useful anymore */
1522 bus_dmamap_unload(rx_ring
->adapter
->rx_buf_tag
, rx_info
->map
);
1523 m_freem(rx_info
->mbuf
);
1524 rx_info
->mbuf
= NULL
;
1526 rx_ring
->free_rx_ids
[ntc
] = req_id
;
1527 ntc
= ENA_RX_RING_IDX_NEXT(ntc
, rx_ring
->ring_size
);
1530 *next_to_clean
= ntc
;
1536 * ena_rx_checksum - indicate in mbuf if hw indicated a good cksum
1539 ena_rx_checksum(struct ena_ring
*rx_ring
, struct ena_com_rx_ctx
*ena_rx_ctx
,
1543 /* if IP and error */
1544 if (unlikely((ena_rx_ctx
->l3_proto
== ENA_ETH_IO_L3_PROTO_IPV4
) &&
1545 ena_rx_ctx
->l3_csum_err
)) {
1546 /* ipv4 checksum error */
1547 mbuf
->m_pkthdr
.csum_flags
= 0;
1548 IFNET_STAT_INC(rx_ring
->adapter
->ifp
, ierrors
, 1);
1549 #if 0 /* XXX swildner counters */
1550 counter_u64_add(rx_ring
->rx_stats
.bad_csum
, 1);
1552 ena_trace(ENA_DBG
, "RX IPv4 header checksum error");
1557 if ((ena_rx_ctx
->l4_proto
== ENA_ETH_IO_L4_PROTO_TCP
) ||
1558 (ena_rx_ctx
->l4_proto
== ENA_ETH_IO_L4_PROTO_UDP
)) {
1559 if (ena_rx_ctx
->l4_csum_err
) {
1560 /* TCP/UDP checksum error */
1561 mbuf
->m_pkthdr
.csum_flags
= 0;
1562 IFNET_STAT_INC(rx_ring
->adapter
->ifp
, ierrors
, 1);
1563 #if 0 /* XXX swildner counters */
1564 counter_u64_add(rx_ring
->rx_stats
.bad_csum
, 1);
1566 ena_trace(ENA_DBG
, "RX L4 checksum error");
1568 mbuf
->m_pkthdr
.csum_flags
= CSUM_IP_CHECKED
;
1569 mbuf
->m_pkthdr
.csum_flags
|= CSUM_IP_VALID
;
1575 ena_deferred_rx_cleanup(void *arg
, int pending
)
1577 struct ena_ring
*rx_ring
= arg
;
1578 int budget
= CLEAN_BUDGET
;
1580 ENA_RING_MTX_LOCK(rx_ring
);
1582 * If deferred task was executed, perform cleanup of all awaiting
1583 * descs (or until given budget is depleted to avoid infinite loop).
1585 while (likely(budget
--)) {
1586 if (ena_rx_cleanup(rx_ring
) == 0)
1589 ENA_RING_MTX_UNLOCK(rx_ring
);
1593 * ena_rx_cleanup - handle rx irq
1594 * @arg: ring for which irq is being handled
1597 ena_rx_cleanup(struct ena_ring
*rx_ring
)
1599 struct ena_adapter
*adapter
;
1601 struct ena_com_rx_ctx ena_rx_ctx
;
1602 struct ena_com_io_cq
* io_cq
;
1603 struct ena_com_io_sq
* io_sq
;
1606 uint16_t next_to_clean
;
1607 uint32_t refill_required
;
1608 uint32_t refill_threshold
;
1609 uint32_t do_if_input
= 0;
1612 int budget
= RX_BUDGET
;
1614 adapter
= rx_ring
->que
->adapter
;
1616 qid
= rx_ring
->que
->id
;
1617 ena_qid
= ENA_IO_RXQ_IDX(qid
);
1618 io_cq
= &adapter
->ena_dev
->io_cq_queues
[ena_qid
];
1619 io_sq
= &adapter
->ena_dev
->io_sq_queues
[ena_qid
];
1620 next_to_clean
= rx_ring
->next_to_clean
;
1622 ena_trace(ENA_DBG
, "rx: qid %d", qid
);
1625 ena_rx_ctx
.ena_bufs
= rx_ring
->ena_bufs
;
1626 ena_rx_ctx
.max_bufs
= adapter
->max_rx_sgl_size
;
1627 ena_rx_ctx
.descs
= 0;
1628 rc
= ena_com_rx_pkt(io_cq
, io_sq
, &ena_rx_ctx
);
1630 if (unlikely(rc
!= 0))
1633 if (unlikely(ena_rx_ctx
.descs
== 0))
1636 ena_trace(ENA_DBG
| ENA_RXPTH
, "rx: q %d got packet from ena. "
1637 "descs #: %d l3 proto %d l4 proto %d hash: %x",
1638 rx_ring
->qid
, ena_rx_ctx
.descs
, ena_rx_ctx
.l3_proto
,
1639 ena_rx_ctx
.l4_proto
, ena_rx_ctx
.hash
);
1641 /* Receive mbuf from the ring */
1642 mbuf
= ena_rx_mbuf(rx_ring
, rx_ring
->ena_bufs
,
1643 &ena_rx_ctx
, &next_to_clean
);
1645 /* Exit if we failed to retrieve a buffer */
1646 if (unlikely(mbuf
== NULL
)) {
1647 for (i
= 0; i
< ena_rx_ctx
.descs
; ++i
) {
1648 rx_ring
->free_rx_ids
[next_to_clean
] =
1649 rx_ring
->ena_bufs
[i
].req_id
;
1651 ENA_RX_RING_IDX_NEXT(next_to_clean
,
1652 rx_ring
->ring_size
);
1659 * XXX Removed IFCAP_RXCSUM_IPV6 check because DragonFly
1660 * does not seem to support it
1662 if ((ifp
->if_capenable
& IFCAP_RXCSUM
) != 0) {
1663 ena_rx_checksum(rx_ring
, &ena_rx_ctx
, mbuf
);
1666 #if 0 /* XXX swildner counters */
1668 counter_u64_add_protected(rx_ring
->rx_stats
.bytes
,
1669 mbuf
->m_pkthdr
.len
);
1670 counter_u64_add_protected(adapter
->hw_stats
.rx_bytes
,
1671 mbuf
->m_pkthdr
.len
);
1675 * LRO is only for IP/TCP packets and TCP checksum of the packet
1676 * should be computed by hardware.
1680 if (((ifp
->if_capenable
& IFCAP_LRO
) != 0) &&
1681 ((mbuf
->m_pkthdr
.csum_flags
& CSUM_IP_VALID
) != 0) &&
1682 (ena_rx_ctx
.l4_proto
== ENA_ETH_IO_L4_PROTO_TCP
)) {
1684 * Send to the stack if:
1685 * - LRO not enabled, or
1686 * - no LRO resources, or
1687 * - lro enqueue fails
1689 if ((rx_ring
->lro
.lro_cnt
!= 0) &&
1690 (tcp_lro_rx(&rx_ring
->lro
, mbuf
, 0) == 0))
1694 if (do_if_input
!= 0) {
1695 ena_trace(ENA_DBG
| ENA_RXPTH
,
1696 "calling if_input() with mbuf %p", mbuf
);
1697 ENA_RING_MTX_UNLOCK(rx_ring
);
1698 (*ifp
->if_input
)(ifp
, mbuf
, NULL
, -1);
1699 ENA_RING_MTX_LOCK(rx_ring
);
1702 IFNET_STAT_INC(ifp
, ipackets
, 1);
1703 #if 0 /* XXX swildner counters */
1705 counter_u64_add_protected(rx_ring
->rx_stats
.cnt
, 1);
1706 counter_u64_add_protected(adapter
->hw_stats
.rx_packets
, 1);
1711 rx_ring
->next_to_clean
= next_to_clean
;
1713 refill_required
= ena_com_free_desc(io_sq
);
1714 refill_threshold
= rx_ring
->ring_size
/ ENA_RX_REFILL_THRESH_DIVIDER
;
1716 if (refill_required
> refill_threshold
) {
1717 ena_com_update_dev_comp_head(rx_ring
->ena_com_io_cq
);
1718 ena_refill_rx_bufs(rx_ring
, refill_required
);
1722 tcp_lro_flush_all(&rx_ring
->lro
);
1725 return (RX_BUDGET
- budget
);
1728 IFNET_STAT_INC(rx_ring
->adapter
->ifp
, ierrors
, 1);
1729 #if 0 /* XXX swildner counters */
1730 counter_u64_add(rx_ring
->rx_stats
.bad_desc_num
, 1);
1732 return (RX_BUDGET
- budget
);
1735 /*********************************************************************
1737 * MSIX & Interrupt Service routine
1739 **********************************************************************/
1742 * ena_handle_msix - MSIX Interrupt Handler for admin/async queue
1743 * @arg: interrupt number
1746 ena_intr_msix_mgmnt(void *arg
)
1748 struct ena_adapter
*adapter
= (struct ena_adapter
*)arg
;
1750 ena_com_admin_q_comp_intr_handler(adapter
->ena_dev
);
1751 if (likely(adapter
->running
))
1752 ena_com_aenq_intr_handler(adapter
->ena_dev
, arg
);
1756 * ena_handle_msix - MSIX Interrupt Handler for Tx/Rx
1757 * @arg: interrupt number
1760 ena_handle_msix(void *arg
)
1762 struct ena_que
*que
= arg
;
1763 struct ena_adapter
*adapter
= que
->adapter
;
1764 if_t ifp
= adapter
->ifp
;
1765 struct ena_ring
*tx_ring
;
1766 struct ena_ring
*rx_ring
;
1767 struct ena_com_io_cq
* io_cq
;
1768 struct ena_eth_io_intr_reg intr_reg
;
1772 if (unlikely((ifp
->if_flags
& IFF_RUNNING
) == 0))
1775 ena_trace(ENA_DBG
, "MSI-X TX/RX routine");
1777 tx_ring
= que
->tx_ring
;
1778 rx_ring
= que
->rx_ring
;
1780 ena_qid
= ENA_IO_TXQ_IDX(qid
);
1781 io_cq
= &adapter
->ena_dev
->io_cq_queues
[ena_qid
];
1783 for (i
= 0; i
< CLEAN_BUDGET
; ++i
) {
1785 * If lock cannot be acquired, then deferred cleanup task was
1786 * being executed and rx ring is being cleaned up in
1789 if (likely(ENA_RING_MTX_TRYLOCK(rx_ring
) != 0)) {
1790 rxc
= ena_rx_cleanup(rx_ring
);
1791 ENA_RING_MTX_UNLOCK(rx_ring
);
1796 /* Protection from calling ena_tx_cleanup from ena_start_xmit */
1797 ENA_RING_MTX_LOCK(tx_ring
);
1798 txc
= ena_tx_cleanup(tx_ring
);
1799 ENA_RING_MTX_UNLOCK(tx_ring
);
1801 if (unlikely((ifp
->if_flags
& IFF_RUNNING
) == 0))
1804 if ((txc
!= TX_BUDGET
) && (rxc
!= RX_BUDGET
))
1808 /* Signal that work is done and unmask interrupt */
1809 ena_com_update_intr_reg(&intr_reg
,
1813 ena_com_unmask_intr(io_cq
, &intr_reg
);
1817 ena_enable_msix(struct ena_adapter
*adapter
)
1819 device_t dev
= adapter
->pdev
;
1821 int error
, i
, rc
= 0;
1823 /* Reserved the max msix vectors we might need */
1824 msix_vecs
= ENA_MAX_MSIX_VEC(adapter
->num_queues
);
1826 adapter
->msix_entries
= kmalloc(msix_vecs
* sizeof(struct msix_entry
),
1827 M_DEVBUF
, M_WAITOK
| M_ZERO
);
1829 ena_trace(ENA_DBG
, "trying to enable MSI-X, vectors: %d", msix_vecs
);
1831 for (i
= 0; i
< msix_vecs
; i
++) {
1832 adapter
->msix_entries
[i
].entry
= i
;
1833 /* Vectors must start from 1 */
1834 adapter
->msix_entries
[i
].vector
= i
+ 1;
1837 error
= pci_setup_msix(dev
);
1839 device_printf(dev
, "pci_setup_msix() failed\n");
1843 adapter
->msix_vecs
= msix_vecs
;
1844 adapter
->msix_enabled
= true;
1849 kfree(adapter
->msix_entries
, M_DEVBUF
);
1850 adapter
->msix_entries
= NULL
;
1856 ena_setup_mgmnt_intr(struct ena_adapter
*adapter
)
1859 ksnprintf(adapter
->irq_tbl
[ENA_MGMNT_IRQ_IDX
].name
,
1860 ENA_IRQNAME_SIZE
, "ena-mgmnt@pci:%s",
1861 device_get_nameunit(adapter
->pdev
));
1863 * Handler is NULL on purpose, it will be set
1864 * when mgmnt interrupt is acquired
1866 adapter
->irq_tbl
[ENA_MGMNT_IRQ_IDX
].handler
= NULL
;
1867 adapter
->irq_tbl
[ENA_MGMNT_IRQ_IDX
].data
= adapter
;
1868 adapter
->irq_tbl
[ENA_MGMNT_IRQ_IDX
].vector
=
1869 adapter
->msix_entries
[ENA_MGMNT_IRQ_IDX
].vector
;
1873 ena_setup_io_intr(struct ena_adapter
*adapter
)
1875 static int last_bind_cpu
= -1;
1878 for (int i
= 0; i
< adapter
->num_queues
; i
++) {
1879 irq_idx
= ENA_IO_IRQ_IDX(i
);
1881 ksnprintf(adapter
->irq_tbl
[irq_idx
].name
, ENA_IRQNAME_SIZE
,
1882 "%s-TxRx-%d", device_get_nameunit(adapter
->pdev
), i
);
1883 adapter
->irq_tbl
[irq_idx
].handler
= ena_handle_msix
;
1884 adapter
->irq_tbl
[irq_idx
].data
= &adapter
->que
[i
];
1885 adapter
->irq_tbl
[irq_idx
].vector
=
1886 adapter
->msix_entries
[irq_idx
].vector
;
1887 ena_trace(ENA_INFO
| ENA_IOQ
, "ena_setup_io_intr vector: %d\n",
1888 adapter
->msix_entries
[irq_idx
].vector
);
1890 adapter
->que
[i
].cpu
= adapter
->irq_tbl
[irq_idx
].cpu
=
1891 rss_getcpu(i
% rss_getnumbuckets());
1894 * We still want to bind rings to the corresponding cpu
1895 * using something similar to the RSS round-robin technique.
1897 * XXX It seems that this can be removed since DragonFly has
1898 * native support for RSS. DragonFly also does not have
1899 * support for CPU_FIRST or CPU_NEXT.
1902 if (last_bind_cpu
< 0)
1903 last_bind_cpu
= (last_bind_cpu
+ 1) % ncpus
;
1904 adapter
->que
[i
].cpu
= adapter
->irq_tbl
[irq_idx
].cpu
=
1906 last_bind_cpu
= (last_bind_cpu
+ 1) % ncpus
;
1912 ena_request_mgmnt_irq(struct ena_adapter
*adapter
)
1914 struct ena_irq
*irq
;
1915 unsigned long flags
;
1918 flags
= RF_ACTIVE
| RF_SHAREABLE
;
1920 irq
= &adapter
->irq_tbl
[ENA_MGMNT_IRQ_IDX
];
1922 error
= pci_alloc_msix_vector(adapter
->pdev
, 0, &irq
->vector
, 0);
1924 device_printf(adapter
->pdev
, "Could not initialize MGMNT MSI-X Vector on cpu0\n");
1928 irq
->res
= bus_alloc_resource_any(adapter
->pdev
, SYS_RES_IRQ
,
1929 &irq
->vector
, flags
);
1931 if (unlikely(irq
->res
== NULL
)) {
1932 device_printf(adapter
->pdev
, "could not allocate "
1933 "irq vector: %d\n", irq
->vector
);
1934 pci_release_msix_vector(adapter
->pdev
, irq
->vector
);
1938 rc
= bus_activate_resource(adapter
->pdev
, SYS_RES_IRQ
,
1939 irq
->vector
, irq
->res
);
1940 if (unlikely(rc
!= 0)) {
1941 device_printf(adapter
->pdev
, "could not activate "
1942 "irq vector: %d\n", irq
->vector
);
1946 rc
= bus_setup_intr(adapter
->pdev
, irq
->res
,
1947 INTR_MPSAFE
, ena_intr_msix_mgmnt
,
1948 irq
->data
, &irq
->cookie
, NULL
);
1949 if (unlikely(rc
!= 0)) {
1950 device_printf(adapter
->pdev
, "failed to register "
1951 "interrupt handler for irq %ju: %d\n",
1952 rman_get_start(irq
->res
), rc
);
1955 irq
->requested
= true;
1960 ena_trace(ENA_INFO
| ENA_ADMQ
, "releasing resource for irq %d\n",
1962 rcc
= bus_release_resource(adapter
->pdev
, SYS_RES_IRQ
,
1963 irq
->vector
, irq
->res
);
1964 pci_release_msix_vector(adapter
->pdev
, irq
->vector
);
1965 if (unlikely(rcc
!= 0))
1966 device_printf(adapter
->pdev
, "dev has no parent while "
1967 "releasing res for irq: %d\n", irq
->vector
);
1974 ena_request_io_irq(struct ena_adapter
*adapter
)
1976 struct ena_irq
*irq
;
1977 unsigned long flags
= 0;
1978 int rc
= 0, i
, rcc
, error
;
1980 if (unlikely(adapter
->msix_enabled
== 0)) {
1981 device_printf(adapter
->pdev
,
1982 "failed to request I/O IRQ: MSI-X is not enabled\n");
1985 flags
= RF_ACTIVE
| RF_SHAREABLE
;
1988 for (i
= ENA_IO_IRQ_FIRST_IDX
; i
< adapter
->msix_vecs
; i
++) {
1989 irq
= &adapter
->irq_tbl
[i
];
1991 if (unlikely(irq
->requested
))
1994 error
= pci_alloc_msix_vector(adapter
->pdev
, i
, &irq
->vector
, irq
->cpu
);
1996 device_printf(adapter
->pdev
, "Unable to allocated MSI-X %d on cpu%d\n", i
, irq
->cpu
);
2000 irq
->res
= bus_alloc_resource_any(adapter
->pdev
, SYS_RES_IRQ
,
2001 &irq
->vector
, flags
);
2002 if (unlikely(irq
->res
== NULL
)) {
2003 device_printf(adapter
->pdev
, "could not allocate "
2004 "irq vector: %d\n", irq
->vector
);
2010 * TODO: Might need to setup desc and use irq->name as the
2013 rc
= bus_setup_intr(adapter
->pdev
, irq
->res
,
2015 irq
->handler
, irq
->data
, &irq
->cookie
, NULL
);
2016 if (unlikely(rc
!= 0)) {
2017 device_printf(adapter
->pdev
, "failed to register "
2018 "interrupt handler for irq %ju: %d\n",
2019 rman_get_start(irq
->res
), rc
);
2022 irq
->requested
= true;
2025 ena_trace(ENA_INFO
, "queue %d - RSS bucket %d\n",
2026 i
- ENA_IO_IRQ_FIRST_IDX
, irq
->cpu
);
2028 ena_trace(ENA_INFO
, "queue %d - cpu %d\n",
2029 i
- ENA_IO_IRQ_FIRST_IDX
, irq
->cpu
);
2037 for (; i
>= ENA_IO_IRQ_FIRST_IDX
; i
--) {
2038 irq
= &adapter
->irq_tbl
[i
];
2041 /* Once we entered err: section and irq->requested is true we
2042 free both intr and resources */
2044 rcc
= bus_teardown_intr(adapter
->pdev
, irq
->res
, irq
->cookie
);
2045 if (unlikely(rcc
!= 0))
2046 device_printf(adapter
->pdev
, "could not release"
2047 " irq: %d, error: %d\n", irq
->vector
, rcc
);
2049 /* If we entred err: section without irq->requested set we know
2050 it was bus_alloc_resource_any() that needs cleanup, provided
2051 res is not NULL. In case res is NULL no work in needed in
2054 if (irq
->res
!= NULL
) {
2055 rcc
= bus_release_resource(adapter
->pdev
, SYS_RES_IRQ
,
2056 irq
->vector
, irq
->res
);
2057 pci_release_msix_vector(adapter
->pdev
, irq
->vector
);
2059 if (unlikely(rcc
!= 0))
2060 device_printf(adapter
->pdev
, "dev has no parent while "
2061 "releasing res for irq: %d\n", irq
->vector
);
2062 irq
->requested
= false;
2070 ena_free_mgmnt_irq(struct ena_adapter
*adapter
)
2072 struct ena_irq
*irq
;
2075 irq
= &adapter
->irq_tbl
[ENA_MGMNT_IRQ_IDX
];
2076 if (irq
->requested
) {
2077 ena_trace(ENA_INFO
| ENA_ADMQ
, "tear down irq: %d\n",
2079 rc
= bus_teardown_intr(adapter
->pdev
, irq
->res
, irq
->cookie
);
2080 if (unlikely(rc
!= 0))
2081 device_printf(adapter
->pdev
, "failed to tear "
2082 "down irq: %d\n", irq
->vector
);
2086 if (irq
->res
!= NULL
) {
2087 ena_trace(ENA_INFO
| ENA_ADMQ
, "release resource irq: %d\n",
2089 rc
= bus_release_resource(adapter
->pdev
, SYS_RES_IRQ
,
2090 irq
->vector
, irq
->res
);
2091 pci_release_msix_vector(adapter
->pdev
, irq
->vector
);
2093 if (unlikely(rc
!= 0))
2094 device_printf(adapter
->pdev
, "dev has no parent while "
2095 "releasing res for irq: %d\n", irq
->vector
);
2100 ena_free_io_irq(struct ena_adapter
*adapter
)
2102 struct ena_irq
*irq
;
2105 for (int i
= ENA_IO_IRQ_FIRST_IDX
; i
< adapter
->msix_vecs
; i
++) {
2106 irq
= &adapter
->irq_tbl
[i
];
2107 if (irq
->requested
) {
2108 ena_trace(ENA_INFO
| ENA_IOQ
, "tear down irq: %d\n",
2110 rc
= bus_teardown_intr(adapter
->pdev
, irq
->res
,
2112 if (unlikely(rc
!= 0)) {
2113 device_printf(adapter
->pdev
, "failed to tear "
2114 "down irq: %d\n", irq
->vector
);
2119 if (irq
->res
!= NULL
) {
2120 ena_trace(ENA_INFO
| ENA_IOQ
, "release resource irq: %d\n",
2122 rc
= bus_release_resource(adapter
->pdev
, SYS_RES_IRQ
,
2123 irq
->vector
, irq
->res
);
2124 pci_release_msix_vector(adapter
->pdev
, irq
->vector
);
2126 if (unlikely(rc
!= 0)) {
2127 device_printf(adapter
->pdev
, "dev has no parent"
2128 " while releasing res for irq: %d\n",
2136 ena_free_irqs(struct ena_adapter
* adapter
)
2139 ena_free_io_irq(adapter
);
2140 ena_free_mgmnt_irq(adapter
);
2141 ena_disable_msix(adapter
);
2145 ena_disable_msix(struct ena_adapter
*adapter
)
2148 pci_release_msi(adapter
->pdev
);
2150 adapter
->msix_vecs
= 0;
2151 kfree(adapter
->msix_entries
, M_DEVBUF
);
2152 adapter
->msix_entries
= NULL
;
2156 ena_unmask_all_io_irqs(struct ena_adapter
*adapter
)
2158 struct ena_com_io_cq
* io_cq
;
2159 struct ena_eth_io_intr_reg intr_reg
;
2163 /* Unmask interrupts for all queues */
2164 for (i
= 0; i
< adapter
->num_queues
; i
++) {
2165 ena_qid
= ENA_IO_TXQ_IDX(i
);
2166 io_cq
= &adapter
->ena_dev
->io_cq_queues
[ena_qid
];
2167 ena_com_update_intr_reg(&intr_reg
, 0, 0, true);
2168 ena_com_unmask_intr(io_cq
, &intr_reg
);
2172 /* Configure the Rx forwarding */
2174 ena_rss_configure(struct ena_adapter
*adapter
)
2176 struct ena_com_dev
*ena_dev
= adapter
->ena_dev
;
2179 /* Set indirect table */
2180 rc
= ena_com_indirect_table_set(ena_dev
);
2181 if (unlikely((rc
!= 0) && (rc
!= EOPNOTSUPP
)))
2184 /* Configure hash function (if supported) */
2185 rc
= ena_com_set_hash_function(ena_dev
);
2186 if (unlikely((rc
!= 0) && (rc
!= EOPNOTSUPP
)))
2189 /* Configure hash inputs (if supported) */
2190 rc
= ena_com_set_hash_ctrl(ena_dev
);
2191 if (unlikely((rc
!= 0) && (rc
!= EOPNOTSUPP
)))
2198 ena_up_complete(struct ena_adapter
*adapter
)
2202 if (likely(adapter
->rss_support
)) {
2203 rc
= ena_rss_configure(adapter
);
2208 rc
= ena_change_mtu(adapter
->ifp
, adapter
->ifp
->if_mtu
);
2209 if (unlikely(rc
!= 0))
2212 ena_refill_all_rx_bufs(adapter
);
2213 #if 0 /* XXX swildner counters */
2214 ena_reset_counters((counter_u64_t
*)&adapter
->hw_stats
,
2215 sizeof(adapter
->hw_stats
));
2222 ena_up(struct ena_adapter
*adapter
)
2226 if (unlikely(device_is_attached(adapter
->pdev
) == 0)) {
2227 device_printf(adapter
->pdev
, "device is not attached!\n");
2231 if (unlikely(!adapter
->running
)) {
2232 device_printf(adapter
->pdev
, "device is not running!\n");
2237 device_printf(adapter
->pdev
, "device is going UP\n");
2239 /* setup interrupts for IO queues */
2240 ena_setup_io_intr(adapter
);
2241 rc
= ena_request_io_irq(adapter
);
2242 if (unlikely(rc
!= 0)) {
2243 ena_trace(ENA_ALERT
, "err_req_irq");
2247 /* allocate transmit descriptors */
2248 rc
= ena_setup_all_tx_resources(adapter
);
2249 if (unlikely(rc
!= 0)) {
2250 ena_trace(ENA_ALERT
, "err_setup_tx");
2254 /* allocate receive descriptors */
2255 rc
= ena_setup_all_rx_resources(adapter
);
2256 if (unlikely(rc
!= 0)) {
2257 ena_trace(ENA_ALERT
, "err_setup_rx");
2261 /* create IO queues for Rx & Tx */
2262 rc
= ena_create_io_queues(adapter
);
2263 if (unlikely(rc
!= 0)) {
2264 ena_trace(ENA_ALERT
,
2265 "create IO queues failed");
2269 if (unlikely(adapter
->link_status
)) {
2270 adapter
->ifp
->if_link_state
= LINK_STATE_UP
;
2271 if_link_state_change(adapter
->ifp
);
2274 rc
= ena_up_complete(adapter
);
2275 if (unlikely(rc
!= 0))
2276 goto err_up_complete
;
2278 #if 0 /* XXX swildner counters */
2279 counter_u64_add(adapter
->dev_stats
.interface_up
, 1);
2282 ena_update_hwassist(adapter
);
2284 adapter
->ifp
->if_flags
|= IFF_RUNNING
;
2285 ifq_clr_oactive(&adapter
->ifp
->if_snd
);
2287 callout_reset(&adapter
->timer_service
, hz
,
2288 ena_timer_service
, (void *)adapter
);
2292 ena_unmask_all_io_irqs(adapter
);
2298 ena_destroy_all_io_queues(adapter
);
2300 ena_free_all_rx_resources(adapter
);
2302 ena_free_all_tx_resources(adapter
);
2304 ena_free_io_irq(adapter
);
2309 #if 0 /* XXX swildner counters */
2311 ena_get_counter(if_t ifp
, ift_counter cnt
)
2313 struct ena_adapter
*adapter
;
2314 struct ena_hw_stats
*stats
;
2316 adapter
= ifp
->if_softc
;
2317 stats
= &adapter
->hw_stats
;
2320 case IFCOUNTER_IPACKETS
:
2321 return (counter_u64_fetch(stats
->rx_packets
));
2322 case IFCOUNTER_OPACKETS
:
2323 return (counter_u64_fetch(stats
->tx_packets
));
2324 case IFCOUNTER_IBYTES
:
2325 return (counter_u64_fetch(stats
->rx_bytes
));
2326 case IFCOUNTER_OBYTES
:
2327 return (counter_u64_fetch(stats
->tx_bytes
));
2328 case IFCOUNTER_IQDROPS
:
2329 return (counter_u64_fetch(stats
->rx_drops
));
2331 return (if_get_counter_default(ifp
, cnt
));
2337 ena_media_change(if_t ifp
)
2339 /* Media Change is not supported by firmware */
2344 ena_media_status(if_t ifp
, struct ifmediareq
*ifmr
)
2346 struct ena_adapter
*adapter
= ifp
->if_softc
;
2347 ena_trace(ENA_DBG
, "enter");
2349 lockmgr(&adapter
->global_lock
, LK_EXCLUSIVE
);
2351 ifmr
->ifm_status
= IFM_AVALID
;
2352 ifmr
->ifm_active
= IFM_ETHER
;
2354 if (!adapter
->link_status
) {
2355 lockmgr(&adapter
->global_lock
, LK_RELEASE
);
2356 ena_trace(ENA_INFO
, "link_status = false");
2360 ifmr
->ifm_status
|= IFM_ACTIVE
;
2361 ifmr
->ifm_active
|= IFM_10G_T
| IFM_FDX
;
2363 lockmgr(&adapter
->global_lock
, LK_RELEASE
);
2369 struct ena_adapter
*adapter
= (struct ena_adapter
*)arg
;
2372 lockmgr(&adapter
->ioctl_lock
, LK_EXCLUSIVE
);
2374 lockmgr(&adapter
->ioctl_lock
, LK_RELEASE
);
2379 ena_ioctl(if_t ifp
, u_long command
, caddr_t data
, struct ucred
*cred
)
2381 struct ena_adapter
*adapter
;
2385 adapter
= ifp
->if_softc
;
2386 ifr
= (struct ifreq
*)data
;
2389 * Acquiring lock to prevent from running up and down routines parallel.
2394 lockmgr(&adapter
->ioctl_lock
, LK_EXCLUSIVE
);
2397 ena_change_mtu(ifp
, ifr
->ifr_mtu
);
2399 rc
= ena_up(adapter
);
2400 lockmgr(&adapter
->ioctl_lock
, LK_RELEASE
);
2404 if ((ifp
->if_flags
& IFF_UP
) != 0) {
2405 if ((ifp
->if_flags
& IFF_RUNNING
) != 0) {
2406 if ((ifp
->if_flags
& (IFF_PROMISC
|
2407 IFF_ALLMULTI
)) != 0) {
2408 device_printf(adapter
->pdev
,
2409 "ioctl promisc/allmulti\n");
2412 lockmgr(&adapter
->ioctl_lock
, LK_EXCLUSIVE
);
2413 rc
= ena_up(adapter
);
2414 lockmgr(&adapter
->ioctl_lock
, LK_RELEASE
);
2417 if ((ifp
->if_flags
& IFF_RUNNING
) != 0) {
2418 lockmgr(&adapter
->ioctl_lock
, LK_EXCLUSIVE
);
2420 lockmgr(&adapter
->ioctl_lock
, LK_RELEASE
);
2431 rc
= ifmedia_ioctl(ifp
, ifr
, &adapter
->media
, command
);
2438 if (ifr
->ifr_reqcap
!= ifp
->if_capenable
) {
2439 ifp
->if_capenable
= ifr
->ifr_reqcap
;
2443 if ((reinit
!= 0) &&
2444 ((ifp
->if_flags
& IFF_RUNNING
) != 0)) {
2445 lockmgr(&adapter
->ioctl_lock
, LK_EXCLUSIVE
);
2447 rc
= ena_up(adapter
);
2448 lockmgr(&adapter
->ioctl_lock
, LK_RELEASE
);
2454 rc
= ether_ioctl(ifp
, command
, data
);
2462 ena_get_dev_offloads(struct ena_com_dev_get_features_ctx
*feat
)
2466 if ((feat
->offload
.tx
&
2467 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK
|
2468 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK
|
2469 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK
)) != 0)
2470 caps
|= IFCAP_TXCSUM
;
2472 if ((feat
->offload
.tx
&
2473 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK
|
2474 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK
)) != 0)
2475 caps
|= IFCAP_TXCSUM
;
2477 if ((feat
->offload
.tx
&
2478 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK
) != 0)
2481 if ((feat
->offload
.tx
&
2482 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK
) != 0)
2485 if ((feat
->offload
.rx_supported
&
2486 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK
|
2487 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK
)) != 0)
2488 caps
|= IFCAP_RXCSUM
;
2491 if ((feat
->offload
.rx_supported
&
2492 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK
) != 0)
2493 caps
|= IFCAP_RXCSUM_IPV6
;
2498 caps
|= IFCAP_JUMBO_MTU
;
2504 ena_update_host_info(struct ena_admin_host_info
*host_info
, if_t ifp
)
2507 host_info
->supported_network_features
[0] =
2508 (uint32_t)ifp
->if_capabilities
;
2512 ena_update_hwassist(struct ena_adapter
*adapter
)
2514 if_t ifp
= adapter
->ifp
;
2515 uint32_t feat
= adapter
->tx_offload_cap
;
2516 int cap
= ifp
->if_capenable
;
2519 ifp
->if_hwassist
= 0;
2521 if ((cap
& IFCAP_TXCSUM
) != 0) {
2523 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK
) != 0)
2526 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK
|
2527 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK
)) != 0)
2528 flags
|= CSUM_UDP
| CSUM_TCP
;
2532 if ((cap
& IFCAP_TXCSUM_IPV6
) != 0)
2533 flags
|= CSUM_IP6_UDP
| CSUM_IP6_TCP
;
2536 if ((cap
& IFCAP_TSO4
) != 0 || (cap
& IFCAP_TSO6
) != 0)
2539 ifp
->if_hwassist
|= flags
;
2543 ena_setup_ifnet(device_t pdev
, struct ena_adapter
*adapter
,
2544 struct ena_com_dev_get_features_ctx
*feat
)
2549 ifp
= adapter
->ifp
= if_alloc(IFT_ETHER
);
2550 if (unlikely(ifp
== NULL
)) {
2551 ena_trace(ENA_ALERT
, "can not allocate ifnet structure\n");
2554 if_initname(ifp
, device_get_name(pdev
), device_get_unit(pdev
));
2555 ifp
->if_softc
= adapter
;
2557 ifp
->if_flags
= IFF_BROADCAST
| IFF_SIMPLEX
| IFF_MULTICAST
;
2559 ifp
->if_init
= ena_init
;
2560 ifp
->if_start
= ena_start_xmit
;
2561 ifp
->if_ioctl
= ena_ioctl
;
2562 #if 0 /* XXX swildner counter */
2563 if_setgetcounterfn(ifp
, ena_get_counter
);
2566 ifq_set_maxlen(&ifp
->if_snd
, adapter
->tx_ring_size
);
2567 ifq_set_ready(&ifp
->if_snd
);
2568 ifp
->if_mtu
= ETHERMTU
;
2569 ifp
->if_baudrate
= 0;
2570 /* Zeroize capabilities... */
2571 ifp
->if_capabilities
= 0;
2572 ifp
->if_capenable
= 0;
2573 /* check hardware support */
2574 caps
= ena_get_dev_offloads(feat
);
2575 /* ... and set them */
2576 //if_setcapabilitiesbit(ifp, caps, 0);
2577 ((struct ifnet
*)ifp
)->if_capabilities
|= caps
;
2578 ((struct ifnet
*)ifp
)->if_capabilities
&= ~0;
2580 /* TSO parameters */
2581 //ifp->if_hw_tsomax = ENA_TSO_MAXSIZE -
2582 // (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
2583 ifp
->if_tsolen
= adapter
->max_tx_sgl_size
- 1;
2584 //ifp->if_hw_tsomaxsegsize = ENA_TSO_MAXSIZE;
2586 ifp
->if_hdrlen
= sizeof(struct ether_vlan_header
);
2587 ifp
->if_capenable
= ifp
->if_capabilities
;
2590 * Specify the media types supported by this adapter and register
2591 * callbacks to update media and link information
2593 ifmedia_init(&adapter
->media
, IFM_IMASK
,
2594 ena_media_change
, ena_media_status
);
2595 ifmedia_add(&adapter
->media
, IFM_ETHER
| IFM_AUTO
, 0, NULL
);
2596 ifmedia_set(&adapter
->media
, IFM_ETHER
| IFM_AUTO
);
2598 ether_ifattach(ifp
, adapter
->mac_addr
, NULL
);
2604 ena_down(struct ena_adapter
*adapter
)
2609 device_printf(adapter
->pdev
, "device is going DOWN\n");
2611 callout_drain(&adapter
->timer_service
);
2613 adapter
->up
= false;
2614 ifq_set_oactive(&adapter
->ifp
->if_snd
);
2615 adapter
->ifp
->if_flags
&= ~IFF_RUNNING
;
2617 ena_free_io_irq(adapter
);
2619 if (adapter
->trigger_reset
) {
2620 rc
= ena_com_dev_reset(adapter
->ena_dev
,
2621 adapter
->reset_reason
);
2622 if (unlikely(rc
!= 0))
2623 device_printf(adapter
->pdev
,
2624 "Device reset failed\n");
2627 ena_destroy_all_io_queues(adapter
);
2629 ena_free_all_tx_bufs(adapter
);
2630 ena_free_all_rx_bufs(adapter
);
2631 ena_free_all_tx_resources(adapter
);
2632 ena_free_all_rx_resources(adapter
);
2634 #if 0 /* XXX swildner counters */
2635 counter_u64_add(adapter
->dev_stats
.interface_down
, 1);
2641 ena_tx_csum(struct ena_com_tx_ctx
*ena_tx_ctx
, struct mbuf
*mbuf
)
2643 struct ena_com_tx_meta
*ena_meta
;
2644 struct ether_vlan_header
*eh
;
2654 ena_meta
= &ena_tx_ctx
->ena_meta
;
2655 mss
= mbuf
->m_pkthdr
.tso_segsz
;
2660 if ((mbuf
->m_pkthdr
.csum_flags
& CSUM_TSO
) != 0)
2663 if ((mbuf
->m_pkthdr
.csum_flags
& CSUM_OFFLOAD
) != 0)
2667 ena_tx_ctx
->meta_valid
= 0;
2671 /* Determine where frame payload starts. */
2672 eh
= mtod(mbuf
, struct ether_vlan_header
*);
2673 if (eh
->evl_encap_proto
== htons(ETHERTYPE_VLAN
)) {
2674 etype
= ntohs(eh
->evl_proto
);
2675 ehdrlen
= ETHER_HDR_LEN
+ ETHER_VLAN_ENCAP_LEN
;
2677 etype
= ntohs(eh
->evl_encap_proto
);
2678 ehdrlen
= ETHER_HDR_LEN
;
2681 ip
= (struct ip
*)(mbuf
->m_data
+ ehdrlen
);
2682 iphlen
= ip
->ip_hl
<< 2;
2683 th
= (struct tcphdr
*)((caddr_t
)ip
+ iphlen
);
2685 if ((mbuf
->m_pkthdr
.csum_flags
& CSUM_IP
) != 0) {
2686 ena_tx_ctx
->l3_csum_enable
= 1;
2688 if ((mbuf
->m_pkthdr
.csum_flags
& CSUM_TSO
) != 0) {
2689 ena_tx_ctx
->tso_enable
= 1;
2690 ena_meta
->l4_hdr_len
= (th
->th_off
);
2695 ena_tx_ctx
->l3_proto
= ENA_ETH_IO_L3_PROTO_IPV4
;
2696 if ((ip
->ip_off
& htons(IP_DF
)) != 0)
2699 case ETHERTYPE_IPV6
:
2700 ena_tx_ctx
->l3_proto
= ENA_ETH_IO_L3_PROTO_IPV6
;
2706 if (ip
->ip_p
== IPPROTO_TCP
) {
2707 ena_tx_ctx
->l4_proto
= ENA_ETH_IO_L4_PROTO_TCP
;
2708 if ((mbuf
->m_pkthdr
.csum_flags
&
2710 ena_tx_ctx
->l4_csum_enable
= 1;
2712 ena_tx_ctx
->l4_csum_enable
= 0;
2713 } else if (ip
->ip_p
== IPPROTO_UDP
) {
2714 ena_tx_ctx
->l4_proto
= ENA_ETH_IO_L4_PROTO_UDP
;
2715 if ((mbuf
->m_pkthdr
.csum_flags
&
2717 ena_tx_ctx
->l4_csum_enable
= 1;
2719 ena_tx_ctx
->l4_csum_enable
= 0;
2721 ena_tx_ctx
->l4_proto
= ENA_ETH_IO_L4_PROTO_UNKNOWN
;
2722 ena_tx_ctx
->l4_csum_enable
= 0;
2725 ena_meta
->mss
= mss
;
2726 ena_meta
->l3_hdr_len
= iphlen
;
2727 ena_meta
->l3_hdr_offset
= ehdrlen
;
2728 ena_tx_ctx
->meta_valid
= 1;
2732 ena_check_and_collapse_mbuf(struct ena_ring
*tx_ring
, struct mbuf
**mbuf
)
2734 struct ena_adapter
*adapter
;
2735 struct mbuf
*collapsed_mbuf
;
2738 adapter
= tx_ring
->adapter
;
2739 num_frags
= ena_mbuf_count(*mbuf
);
2741 /* One segment must be reserved for configuration descriptor. */
2742 if (num_frags
< adapter
->max_tx_sgl_size
)
2744 #if 0 /* XXX swildner counters */
2745 counter_u64_add(tx_ring
->tx_stats
.collapse
, 1);
2748 collapsed_mbuf
= m_defrag(*mbuf
, M_NOWAIT
);
2749 if (unlikely(collapsed_mbuf
== NULL
)) {
2750 IFNET_STAT_INC(tx_ring
->adapter
->ifp
, oerrors
, 1);
2751 #if 0 /* XXX swildner counters */
2752 counter_u64_add(tx_ring
->tx_stats
.collapse_err
, 1);
2757 /* If mbuf was collapsed succesfully, original mbuf is released. */
2758 *mbuf
= collapsed_mbuf
;
2764 ena_xmit_mbuf(struct ena_ring
*tx_ring
, struct mbuf
**mbuf
)
2766 struct ena_adapter
*adapter
;
2767 struct ena_tx_buffer
*tx_info
;
2768 struct ena_com_tx_ctx ena_tx_ctx
;
2769 struct ena_com_dev
*ena_dev
;
2770 struct ena_com_buf
*ena_buf
;
2771 struct ena_com_io_sq
* io_sq
;
2772 bus_dma_segment_t segs
[ENA_BUS_DMA_SEGS
];
2774 uint16_t next_to_use
;
2778 uint32_t len
, nsegs
, header_len
;
2782 ena_qid
= ENA_IO_TXQ_IDX(tx_ring
->que
->id
);
2783 adapter
= tx_ring
->que
->adapter
;
2784 ena_dev
= adapter
->ena_dev
;
2785 io_sq
= &ena_dev
->io_sq_queues
[ena_qid
];
2787 //tx_ring is just used to grab the adapter
2788 rc
= ena_check_and_collapse_mbuf(tx_ring
, mbuf
);
2789 if (unlikely(rc
!= 0)) {
2790 ena_trace(ENA_WARNING
,
2791 "Failed to collapse mbuf! err: %d", rc
);
2795 next_to_use
= tx_ring
->next_to_use
;
2796 req_id
= tx_ring
->free_tx_ids
[next_to_use
];
2797 tx_info
= &tx_ring
->tx_buffer_info
[req_id
];
2799 tx_info
->mbuf
= *mbuf
;
2800 tx_info
->num_of_bufs
= 0;
2802 ena_buf
= tx_info
->bufs
;
2803 len
= (*mbuf
)->m_len
;
2805 ena_trace(ENA_DBG
| ENA_TXPTH
, "Tx: %d bytes", (*mbuf
)->m_pkthdr
.len
);
2808 header_len
= min_t(uint32_t, len
, tx_ring
->tx_max_header_size
);
2811 rc
= bus_dmamap_load_mbuf_segment(adapter
->tx_buf_tag
, tx_info
->map
,
2812 *mbuf
, segs
, adapter
->max_tx_sgl_size
, &nsegs
, BUS_DMA_NOWAIT
);
2814 if (unlikely((rc
!= 0) || (nsegs
== 0))) {
2815 ena_trace(ENA_WARNING
,
2816 "dmamap load failed! err: %d nsegs: %d", rc
, nsegs
);
2817 IFNET_STAT_INC(tx_ring
->adapter
->ifp
, oerrors
, 1);
2818 #if 0 /* XXX swildner counters */
2819 counter_u64_add(tx_ring
->tx_stats
.dma_mapping_err
, 1);
2821 tx_info
->mbuf
= NULL
;
2823 return (ENA_COM_NO_MEM
);
2825 return (ENA_COM_INVAL
);
2828 for (i
= 0; i
< nsegs
; i
++) {
2829 ena_buf
->len
= segs
[i
].ds_len
;
2830 ena_buf
->paddr
= segs
[i
].ds_addr
;
2833 tx_info
->num_of_bufs
= nsegs
;
2835 memset(&ena_tx_ctx
, 0x0, sizeof(struct ena_com_tx_ctx
));
2836 ena_tx_ctx
.ena_bufs
= tx_info
->bufs
;
2837 ena_tx_ctx
.push_header
= push_hdr
;
2838 ena_tx_ctx
.num_bufs
= tx_info
->num_of_bufs
;
2839 ena_tx_ctx
.req_id
= req_id
;
2840 ena_tx_ctx
.header_len
= header_len
;
2842 /* Set flags and meta data */
2843 ena_tx_csum(&ena_tx_ctx
, *mbuf
);
2844 /* Prepare the packet's descriptors and send them to device */
2845 rc
= ena_com_prepare_tx(io_sq
, &ena_tx_ctx
, &nb_hw_desc
);
2846 if (unlikely(rc
!= 0)) {
2847 device_printf(adapter
->pdev
, "failed to prepare tx bufs\n");
2848 IFNET_STAT_INC(tx_ring
->adapter
->ifp
, oerrors
, 1);
2849 #if 0 /* XXX swildner counters */
2850 counter_u64_add(tx_ring
->tx_stats
.prepare_ctx_err
, 1);
2855 IFNET_STAT_INC(tx_ring
->adapter
->ifp
, opackets
, 1);
2856 #if 0 /* XXX swildner counters */
2858 counter_u64_add_protected(tx_ring
->tx_stats
.cnt
, 1);
2859 counter_u64_add_protected(tx_ring
->tx_stats
.bytes
,
2860 (*mbuf
)->m_pkthdr
.len
);
2862 counter_u64_add_protected(adapter
->hw_stats
.tx_packets
, 1);
2863 counter_u64_add_protected(adapter
->hw_stats
.tx_bytes
,
2864 (*mbuf
)->m_pkthdr
.len
);
2868 tx_info
->tx_descs
= nb_hw_desc
;
2869 getmicrouptime(&tx_info
->timestamp
);
2870 tx_info
->print_once
= true;
2872 tx_ring
->next_to_use
= ENA_TX_RING_IDX_NEXT(next_to_use
,
2873 tx_ring
->ring_size
);
2875 bus_dmamap_sync(adapter
->tx_buf_tag
, tx_info
->map
,
2876 BUS_DMASYNC_PREWRITE
);
2881 tx_info
->mbuf
= NULL
;
2882 bus_dmamap_unload(adapter
->tx_buf_tag
, tx_info
->map
);
2888 ena_start_xmit(struct ifnet
*ifp
, struct ifaltq_subque
*ifsq
)
2891 * TODO: Might need to initialize an ena_ring with the
2892 * ifaltq_subque in it
2894 struct ena_adapter
*adapter
= ifp
->if_softc
;
2895 struct ena_com_io_sq
*io_sq
;
2896 struct ena_ring
*tx_ring
;
2901 if (unlikely((adapter
->ifp
->if_flags
& IFF_RUNNING
) == 0) ||
2902 ifsq_is_oactive(ifsq
)) {
2906 /* Check is link_active and some other shit. If it is, purge. */
2909 if (unlikely(!adapter
->link_status
))
2916 while (!ifsq_is_empty(ifsq
)) {
2917 struct mbuf
*m_head
;
2920 //Grab head from mbuf list
2921 m_head
= ifsq_dequeue(ifsq
);
2925 //pick the associated tx_ring based on hash
2926 i
= m_head
->m_pkthdr
.hash
% adapter
->num_queues
;
2928 tx_ring
= &adapter
->tx_ring
[i
];
2929 ENA_RING_MTX_LOCK(tx_ring
);
2930 ena_qid
= ENA_IO_TXQ_IDX(tx_ring
->que
->id
);
2931 io_sq
= &adapter
->ena_dev
->io_sq_queues
[ena_qid
];
2933 if (unlikely(!ena_com_sq_have_enough_space(io_sq
, ENA_TX_CLEANUP_THRESHOLD
)))
2934 ena_tx_cleanup(tx_ring
);
2936 if (unlikely((ret
= ena_xmit_mbuf(tx_ring
, &m_head
)) != 0)) {
2937 if (ret
== ENA_COM_NO_MEM
) {
2938 /* XXX put mbuf back on queue */
2939 } else if (ret
== ENA_COM_NO_SPACE
) {
2940 /* XXX put mbuf back on queue */
2943 /* XXX advance mbuf queue aka move it forward? */
2945 ENA_RING_MTX_UNLOCK(tx_ring
);
2949 //advance mbuf queue, might already be handled by dequeue
2952 // dillon - wtf is this doing here?
2953 // NOT SURE WHAT TO DO WITH THIS CODE
2954 if (unlikely((adapter
->ifp
->if_flags
& IFF_RUNNING
) == 0))
2955 return; // break here, not return. tx_ring locked
2960 ENA_RING_MTX_UNLOCK(tx_ring
);
2961 BPF_MTAP(adapter
->ifp
, m_head
);
2963 if (unlikely(acum_pkts
== DB_THRESHOLD
)) {
2966 /* Trigger the dma engine */
2967 ena_com_write_sq_doorbell(io_sq
);
2968 #if 0 /* XXX swildner counters */
2969 counter_u64_add(tx_ring
->tx_stats
.doorbells
, 1);
2975 if (likely(acum_pkts
!= 0)) {
2977 /* Trigger the dma engine */
2978 ena_com_write_sq_doorbell(io_sq
);
2979 #if 0 /* XXX swildner counters */
2980 counter_u64_add(tx_ring
->tx_stats
.doorbells
, 1);
2985 !ena_com_sq_have_enough_space(io_sq
, ENA_TX_CLEANUP_THRESHOLD
)) {
2986 ENA_RING_MTX_LOCK(tx_ring
);
2987 ena_tx_cleanup(tx_ring
);
2988 ENA_RING_MTX_UNLOCK(tx_ring
);
2993 ena_calc_io_queue_num(struct ena_adapter
*adapter
,
2994 struct ena_com_dev_get_features_ctx
*get_feat_ctx
)
2996 int io_sq_num
, io_cq_num
, io_queue_num
;
2998 io_sq_num
= get_feat_ctx
->max_queues
.max_sq_num
;
2999 io_cq_num
= get_feat_ctx
->max_queues
.max_cq_num
;
3001 io_queue_num
= min_t(int, ncpus
, ENA_MAX_NUM_IO_QUEUES
);
3002 io_queue_num
= min_t(int, io_queue_num
, io_sq_num
);
3003 io_queue_num
= min_t(int, io_queue_num
, io_cq_num
);
3004 /* 1 IRQ for for mgmnt and 1 IRQ for each TX/RX pair */
3005 io_queue_num
= min_t(int, io_queue_num
,
3006 pci_msix_count(adapter
->pdev
) - 1);
3008 io_queue_num
= min_t(int, io_queue_num
, rss_getnumbuckets());
3011 return (io_queue_num
);
3015 ena_calc_queue_size(struct ena_adapter
*adapter
, uint16_t *max_tx_sgl_size
,
3016 uint16_t *max_rx_sgl_size
, struct ena_com_dev_get_features_ctx
*feat
)
3018 uint32_t queue_size
= ENA_DEFAULT_RING_SIZE
;
3022 queue_size
= min_t(uint32_t, queue_size
,
3023 feat
->max_queues
.max_cq_depth
);
3024 queue_size
= min_t(uint32_t, queue_size
,
3025 feat
->max_queues
.max_sq_depth
);
3027 /* round down to the nearest power of 2 */
3030 if (powerof2(queue_size
) != 0)
3033 q
= rounddown2(queue_size
, v
);
3040 if (unlikely(queue_size
== 0)) {
3041 device_printf(adapter
->pdev
, "Invalid queue size\n");
3042 return (ENA_COM_FAULT
);
3045 *max_tx_sgl_size
= min_t(uint16_t, ENA_PKT_MAX_BUFS
,
3046 feat
->max_queues
.max_packet_tx_descs
);
3047 *max_rx_sgl_size
= min_t(uint16_t, ENA_PKT_MAX_BUFS
,
3048 feat
->max_queues
.max_packet_rx_descs
);
3050 return (queue_size
);
3054 ena_rss_init_default(struct ena_adapter
*adapter
)
3056 struct ena_com_dev
*ena_dev
= adapter
->ena_dev
;
3057 device_t dev
= adapter
->pdev
;
3060 rc
= ena_com_rss_init(ena_dev
, ENA_RX_RSS_TABLE_LOG_SIZE
);
3061 if (unlikely(rc
!= 0)) {
3062 device_printf(dev
, "Cannot init indirect table\n");
3066 for (i
= 0; i
< ENA_RX_RSS_TABLE_SIZE
; i
++) {
3068 qid
= rss_get_indirection_to_bucket(i
);
3069 qid
= qid
% adapter
->num_queues
;
3071 qid
= i
% adapter
->num_queues
;
3073 rc
= ena_com_indirect_table_fill_entry(ena_dev
, i
,
3074 ENA_IO_RXQ_IDX(qid
));
3075 if (unlikely((rc
!= 0) && (rc
!= EOPNOTSUPP
))) {
3076 device_printf(dev
, "Cannot fill indirect table\n");
3077 goto err_rss_destroy
;
3081 rc
= ena_com_fill_hash_function(ena_dev
, ENA_ADMIN_CRC32
, NULL
,
3082 ENA_HASH_KEY_SIZE
, 0xFFFFFFFF);
3083 if (unlikely((rc
!= 0) && (rc
!= EOPNOTSUPP
))) {
3084 device_printf(dev
, "Cannot fill hash function\n");
3085 goto err_rss_destroy
;
3088 rc
= ena_com_set_default_hash_ctrl(ena_dev
);
3089 if (unlikely((rc
!= 0) && (rc
!= EOPNOTSUPP
))) {
3090 device_printf(dev
, "Cannot fill hash control\n");
3091 goto err_rss_destroy
;
3097 ena_com_rss_destroy(ena_dev
);
3102 ena_rss_init_default_deferred(void *arg
)
3104 struct ena_adapter
*adapter
;
3109 dc
= devclass_find("ena");
3110 if (unlikely(dc
== NULL
)) {
3111 ena_trace(ENA_ALERT
, "No devclass ena\n");
3115 max
= devclass_get_maxunit(dc
);
3116 while (max
-- >= 0) {
3117 adapter
= devclass_get_softc(dc
, max
);
3118 if (adapter
!= NULL
) {
3119 rc
= ena_rss_init_default(adapter
);
3120 adapter
->rss_support
= true;
3121 if (unlikely(rc
!= 0)) {
3122 device_printf(adapter
->pdev
,
3123 "WARNING: RSS was not properly initialized,"
3124 " it will affect bandwidth\n");
3125 adapter
->rss_support
= false;
3130 SYSINIT(ena_rss_init
, SI_SUB_KICK_SCHEDULER
, SI_ORDER_SECOND
, ena_rss_init_default_deferred
, NULL
);
3133 ena_config_host_info(struct ena_com_dev
*ena_dev
)
3135 struct ena_admin_host_info
*host_info
;
3138 /* Allocate only the host info */
3139 rc
= ena_com_allocate_host_info(ena_dev
);
3140 if (unlikely(rc
!= 0)) {
3141 ena_trace(ENA_ALERT
, "Cannot allocate host info\n");
3145 host_info
= ena_dev
->host_attr
.host_info
;
3147 host_info
->os_type
= ENA_ADMIN_OS_FREEBSD
;
3148 host_info
->kernel_ver
= osreldate
;
3150 ksprintf(host_info
->kernel_ver_str
, "%d", osreldate
);
3151 host_info
->os_dist
= 0;
3152 strncpy(host_info
->os_dist_str
, osrelease
,
3153 sizeof(host_info
->os_dist_str
) - 1);
3155 host_info
->driver_version
=
3156 (DRV_MODULE_VER_MAJOR
) |
3157 (DRV_MODULE_VER_MINOR
<< ENA_ADMIN_HOST_INFO_MINOR_SHIFT
) |
3158 (DRV_MODULE_VER_SUBMINOR
<< ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT
);
3160 rc
= ena_com_set_host_attributes(ena_dev
);
3161 if (unlikely(rc
!= 0)) {
3162 if (rc
== EOPNOTSUPP
)
3163 ena_trace(ENA_WARNING
, "Cannot set host attributes\n");
3165 ena_trace(ENA_ALERT
, "Cannot set host attributes\n");
3173 ena_com_delete_host_info(ena_dev
);
3177 ena_device_init(struct ena_adapter
*adapter
, device_t pdev
,
3178 struct ena_com_dev_get_features_ctx
*get_feat_ctx
, int *wd_active
)
3180 struct ena_com_dev
* ena_dev
= adapter
->ena_dev
;
3181 bool readless_supported
;
3182 uint32_t aenq_groups
;
3186 rc
= ena_com_mmio_reg_read_request_init(ena_dev
);
3187 if (unlikely(rc
!= 0)) {
3188 device_printf(pdev
, "failed to init mmio read less\n");
3193 * The PCIe configuration space revision id indicate if mmio reg
3196 readless_supported
= !(pci_get_revid(pdev
) & ENA_MMIO_DISABLE_REG_READ
);
3197 ena_com_set_mmio_read_mode(ena_dev
, readless_supported
);
3199 rc
= ena_com_dev_reset(ena_dev
, ENA_REGS_RESET_NORMAL
);
3200 if (unlikely(rc
!= 0)) {
3201 device_printf(pdev
, "Can not reset device\n");
3202 goto err_mmio_read_less
;
3205 rc
= ena_com_validate_version(ena_dev
);
3206 if (unlikely(rc
!= 0)) {
3207 device_printf(pdev
, "device version is too low\n");
3208 goto err_mmio_read_less
;
3211 dma_width
= ena_com_get_dma_width(ena_dev
);
3212 if (unlikely(dma_width
< 0)) {
3213 device_printf(pdev
, "Invalid dma width value %d", dma_width
);
3215 goto err_mmio_read_less
;
3217 adapter
->dma_width
= dma_width
;
3219 /* ENA admin level init */
3220 rc
= ena_com_admin_init(ena_dev
, &aenq_handlers
, true);
3221 if (unlikely(rc
!= 0)) {
3223 "Can not initialize ena admin queue with device\n");
3224 goto err_mmio_read_less
;
3228 * To enable the msix interrupts the driver needs to know the number
3229 * of queues. So the driver uses polling mode to retrieve this
3232 ena_com_set_admin_polling_mode(ena_dev
, true);
3234 ena_config_host_info(ena_dev
);
3236 /* Get Device Attributes */
3237 rc
= ena_com_get_dev_attr_feat(ena_dev
, get_feat_ctx
);
3238 if (unlikely(rc
!= 0)) {
3240 "Cannot get attribute for ena device rc: %d\n", rc
);
3241 goto err_admin_init
;
3244 aenq_groups
= BIT(ENA_ADMIN_LINK_CHANGE
) | BIT(ENA_ADMIN_KEEP_ALIVE
);
3246 aenq_groups
&= get_feat_ctx
->aenq
.supported_groups
;
3247 rc
= ena_com_set_aenq_config(ena_dev
, aenq_groups
);
3248 if (unlikely(rc
!= 0)) {
3249 device_printf(pdev
, "Cannot configure aenq groups rc: %d\n", rc
);
3250 goto err_admin_init
;
3253 *wd_active
= !!(aenq_groups
& BIT(ENA_ADMIN_KEEP_ALIVE
));
3258 ena_com_delete_host_info(ena_dev
);
3259 ena_com_admin_destroy(ena_dev
);
3261 ena_com_mmio_reg_read_request_destroy(ena_dev
);
3266 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter
*adapter
,
3269 struct ena_com_dev
*ena_dev
= adapter
->ena_dev
;
3272 rc
= ena_enable_msix(adapter
);
3273 if (unlikely(rc
!= 0)) {
3274 device_printf(adapter
->pdev
, "Error with MSI-X enablement\n");
3278 ena_setup_mgmnt_intr(adapter
);
3280 rc
= ena_request_mgmnt_irq(adapter
);
3281 if (unlikely(rc
!= 0)) {
3282 device_printf(adapter
->pdev
, "Cannot setup mgmnt queue intr\n");
3283 goto err_disable_msix
;
3286 pci_enable_msix(adapter
->pdev
);
3288 ena_com_set_admin_polling_mode(ena_dev
, false);
3290 ena_com_admin_aenq_enable(ena_dev
);
3295 ena_disable_msix(adapter
);
3300 /* Function called on ENA_ADMIN_KEEP_ALIVE event */
3301 static void ena_keep_alive_wd(void *adapter_data
,
3302 struct ena_admin_aenq_entry
*aenq_e
)
3304 struct ena_adapter
*adapter
= (struct ena_adapter
*)adapter_data
;
3305 struct ena_admin_aenq_keep_alive_desc
*desc
;
3306 struct timeval time
;
3309 desc
= (struct ena_admin_aenq_keep_alive_desc
*)aenq_e
;
3311 rx_drops
= ((uint64_t)desc
->rx_drops_high
<< 32) | desc
->rx_drops_low
;
3312 IFNET_STAT_INC(adapter
->ifp
, iqdrops
, 1);
3313 #if 0 /* XXX swildner counters */
3314 counter_u64_zero(adapter
->hw_stats
.rx_drops
);
3315 counter_u64_add(adapter
->hw_stats
.rx_drops
, rx_drops
);
3318 getmicrouptime(&time
);
3319 atomic_store_rel_64(&adapter
->keep_alive_timestamp
.tv_sec
, time
.tv_sec
);
3322 /* Check for keep alive expiration */
3323 static void check_for_missing_keep_alive(struct ena_adapter
*adapter
)
3325 struct timeval timestamp
, time
;
3327 if (adapter
->wd_active
== 0)
3330 if (likely(adapter
->keep_alive_timeout
== 0))
3333 timestamp
.tv_sec
= atomic_load_acq_64(&adapter
->keep_alive_timestamp
.tv_sec
);
3334 getmicrouptime(&time
);
3335 timevalsub(&time
, ×tamp
);
3336 if (unlikely(time
.tv_sec
> adapter
->keep_alive_timeout
)) {
3337 device_printf(adapter
->pdev
,
3338 "Keep alive watchdog timeout.\n");
3339 #if 0 /* XXX swildner counters */
3340 counter_u64_add(adapter
->dev_stats
.wd_expired
, 1);
3342 adapter
->reset_reason
= ENA_REGS_RESET_KEEP_ALIVE_TO
;
3343 adapter
->trigger_reset
= true;
3347 /* Check if admin queue is enabled */
3348 static void check_for_admin_com_state(struct ena_adapter
*adapter
)
3350 if (unlikely(ena_com_get_admin_running_state(adapter
->ena_dev
) ==
3352 device_printf(adapter
->pdev
,
3353 "ENA admin queue is not in running state!\n");
3354 #if 0 /* XXX swildner counters */
3355 counter_u64_add(adapter
->dev_stats
.admin_q_pause
, 1);
3357 adapter
->reset_reason
= ENA_REGS_RESET_ADMIN_TO
;
3358 adapter
->trigger_reset
= true;
3363 check_missing_comp_in_queue(struct ena_adapter
*adapter
,
3364 struct ena_ring
*tx_ring
)
3366 struct timeval curtime
, time
;
3367 struct ena_tx_buffer
*tx_buf
;
3368 uint32_t missed_tx
= 0;
3371 getmicrouptime(&curtime
);
3373 for (i
= 0; i
< tx_ring
->ring_size
; i
++) {
3374 tx_buf
= &tx_ring
->tx_buffer_info
[i
];
3376 if (timevalisset(&tx_buf
->timestamp
) == 0)
3380 timevalsub(&time
, &tx_buf
->timestamp
);
3382 /* Check again if packet is still waiting */
3383 //WATCH: Might not be exactly comparable
3384 if (unlikely(time
.tv_sec
> adapter
->missing_tx_timeout
)) {
3386 if (!tx_buf
->print_once
)
3387 ena_trace(ENA_WARNING
, "Found a Tx that wasn't "
3388 "completed on time, qid %d, index %d.\n",
3391 tx_buf
->print_once
= true;
3393 #if 0 /* XXX swildner counters */
3394 counter_u64_add(tx_ring
->tx_stats
.missing_tx_comp
, 1);
3397 if (unlikely(missed_tx
>
3398 adapter
->missing_tx_threshold
)) {
3399 device_printf(adapter
->pdev
,
3400 "The number of lost tx completion "
3401 "is above the threshold (%d > %d). "
3402 "Reset the device\n",
3403 missed_tx
, adapter
->missing_tx_threshold
);
3404 adapter
->reset_reason
=
3405 ENA_REGS_RESET_MISS_TX_CMPL
;
3406 adapter
->trigger_reset
= true;
3416 * Check for TX which were not completed on time.
3417 * Timeout is defined by "missing_tx_timeout".
3418 * Reset will be performed if number of incompleted
3419 * transactions exceeds "missing_tx_threshold".
3422 check_for_missing_tx_completions(struct ena_adapter
*adapter
)
3424 struct ena_ring
*tx_ring
;
3427 /* Make sure the driver doesn't turn the device in other process */
3433 if (adapter
->trigger_reset
)
3436 if (adapter
->missing_tx_timeout
== 0)
3439 budget
= adapter
->missing_tx_max_queues
;
3441 for (i
= adapter
->next_monitored_tx_qid
; i
< adapter
->num_queues
; i
++) {
3442 tx_ring
= &adapter
->tx_ring
[i
];
3444 rc
= check_missing_comp_in_queue(adapter
, tx_ring
);
3445 if (unlikely(rc
!= 0))
3455 adapter
->next_monitored_tx_qid
= i
% adapter
->num_queues
;
3458 /* trigger deferred rx cleanup after 2 consecutive detections */
3459 #define EMPTY_RX_REFILL 2
3460 /* For the rare case where the device runs out of Rx descriptors and the
3461 * msix handler failed to refill new Rx descriptors (due to a lack of memory
3463 * This case will lead to a deadlock:
3464 * The device won't send interrupts since all the new Rx packets will be dropped
3465 * The msix handler won't allocate new Rx descriptors so the device won't be
3466 * able to send new packets.
3468 * When such a situation is detected - execute rx cleanup task in another thread
3471 check_for_empty_rx_ring(struct ena_adapter
*adapter
)
3473 struct ena_ring
*rx_ring
;
3474 int i
, refill_required
;
3479 if (adapter
->trigger_reset
)
3482 for (i
= 0; i
< adapter
->num_queues
; i
++) {
3483 rx_ring
= &adapter
->rx_ring
[i
];
3485 refill_required
= ena_com_free_desc(rx_ring
->ena_com_io_sq
);
3486 if (unlikely(refill_required
== (rx_ring
->ring_size
- 1))) {
3487 rx_ring
->empty_rx_queue
++;
3489 if (rx_ring
->empty_rx_queue
>= EMPTY_RX_REFILL
) {
3490 #if 0 /* XXX swildner counters */
3491 counter_u64_add(rx_ring
->rx_stats
.empty_rx_ring
,
3495 device_printf(adapter
->pdev
,
3496 "trigger refill for ring %d\n", i
);
3498 taskqueue_enqueue(rx_ring
->cmpl_tq
,
3499 &rx_ring
->cmpl_task
);
3500 rx_ring
->empty_rx_queue
= 0;
3503 rx_ring
->empty_rx_queue
= 0;
3509 ena_timer_service(void *data
)
3511 struct ena_adapter
*adapter
= (struct ena_adapter
*)data
;
3512 struct ena_admin_host_info
*host_info
=
3513 adapter
->ena_dev
->host_attr
.host_info
;
3515 check_for_missing_keep_alive(adapter
);
3517 check_for_admin_com_state(adapter
);
3519 check_for_missing_tx_completions(adapter
);
3521 check_for_empty_rx_ring(adapter
);
3523 if (host_info
!= NULL
)
3524 ena_update_host_info(host_info
, adapter
->ifp
);
3526 if (unlikely(adapter
->trigger_reset
)) {
3527 device_printf(adapter
->pdev
, "Trigger reset is on\n");
3528 taskqueue_enqueue(adapter
->reset_tq
, &adapter
->reset_task
);
3533 * Schedule another timeout one second from now.
3535 /* XXX swildner callout_schedule_sbt(&adapter->timer_service, SBT_1S, SBT_1S, 0); */
3536 callout_reset(&adapter
->timer_service
, hz
, ena_timer_service
,
3541 ena_reset_task(void *arg
, int pending
)
3543 struct ena_com_dev_get_features_ctx get_feat_ctx
;
3544 struct ena_adapter
*adapter
= (struct ena_adapter
*)arg
;
3545 struct ena_com_dev
*ena_dev
= adapter
->ena_dev
;
3549 if (unlikely(!adapter
->trigger_reset
)) {
3550 device_printf(adapter
->pdev
,
3551 "device reset scheduled but trigger_reset is off\n");
3555 lockmgr(&adapter
->ioctl_lock
, LK_EXCLUSIVE
);
3557 callout_drain(&adapter
->timer_service
);
3559 dev_up
= adapter
->up
;
3561 ena_com_set_admin_running_state(ena_dev
, false);
3563 ena_free_mgmnt_irq(adapter
);
3564 ena_disable_msix(adapter
);
3565 ena_com_abort_admin_commands(ena_dev
);
3566 ena_com_wait_for_abort_completion(ena_dev
);
3567 ena_com_admin_destroy(ena_dev
);
3568 ena_com_mmio_reg_read_request_destroy(ena_dev
);
3570 adapter
->reset_reason
= ENA_REGS_RESET_NORMAL
;
3571 adapter
->trigger_reset
= false;
3573 /* Finished destroy part. Restart the device */
3574 rc
= ena_device_init(adapter
, adapter
->pdev
, &get_feat_ctx
,
3575 &adapter
->wd_active
);
3576 if (unlikely(rc
!= 0)) {
3577 device_printf(adapter
->pdev
,
3578 "ENA device init failed! (err: %d)\n", rc
);
3582 rc
= ena_enable_msix_and_set_admin_interrupts(adapter
,
3583 adapter
->num_queues
);
3584 if (unlikely(rc
!= 0)) {
3585 device_printf(adapter
->pdev
, "Enable MSI-X failed\n");
3589 /* If the interface was up before the reset bring it up */
3591 rc
= ena_up(adapter
);
3592 if (unlikely(rc
!= 0)) {
3593 device_printf(adapter
->pdev
,
3594 "Failed to create I/O queues\n");
3599 callout_reset(&adapter
->timer_service
, hz
,
3600 ena_timer_service
, (void *)adapter
);
3602 lockmgr(&adapter
->ioctl_lock
, LK_RELEASE
);
3607 ena_free_mgmnt_irq(adapter
);
3608 ena_disable_msix(adapter
);
3610 ena_com_admin_destroy(ena_dev
);
3612 device_printf(adapter
->pdev
, "ENA reset failed!\n");
3613 adapter
->running
= false;
3614 lockmgr(&adapter
->ioctl_lock
, LK_RELEASE
);
3618 * ena_attach - Device Initialization Routine
3619 * @pdev: device information struct
3621 * Returns 0 on success, otherwise on failure.
3623 * ena_attach initializes an adapter identified by a device structure.
3624 * The OS initialization, configuring of the adapter private structure,
3625 * and a hardware reset occur.
3628 ena_attach(device_t pdev
)
3630 struct ena_com_dev_get_features_ctx get_feat_ctx
;
3631 static int version_printed
;
3632 struct ena_adapter
*adapter
;
3633 struct ena_com_dev
*ena_dev
= NULL
;
3634 uint16_t tx_sgl_size
= 0;
3635 uint16_t rx_sgl_size
= 0;
3639 adapter
= device_get_softc(pdev
);
3640 adapter
->pdev
= pdev
;
3642 lockinit(&adapter
->global_lock
, "ENA global mtx", 0, LK_CANRECURSE
);
3643 lockinit(&adapter
->ioctl_lock
, "ENA ioctl sx", 0, LK_CANRECURSE
);
3645 /* Set up the timer service */
3646 callout_init_lk(&adapter
->timer_service
, &adapter
->global_lock
);
3647 adapter
->keep_alive_timeout
= DEFAULT_KEEP_ALIVE_TO
;
3648 adapter
->missing_tx_timeout
= DEFAULT_TX_CMP_TO
;
3649 adapter
->missing_tx_max_queues
= DEFAULT_TX_MONITORED_QUEUES
;
3650 adapter
->missing_tx_threshold
= DEFAULT_TX_CMP_THRESHOLD
;
3652 if (version_printed
++ == 0)
3653 device_printf(pdev
, "%s\n", ena_version
);
3655 rc
= ena_allocate_pci_resources(adapter
);
3656 if (unlikely(rc
!= 0)) {
3657 device_printf(pdev
, "PCI resource allocation failed!\n");
3658 ena_free_pci_resources(adapter
);
3662 /* Allocate memory for ena_dev structure */
3663 ena_dev
= kmalloc(sizeof(struct ena_com_dev
), M_DEVBUF
,
3666 adapter
->ena_dev
= ena_dev
;
3667 ena_dev
->dmadev
= pdev
;
3668 ena_dev
->bus
= kmalloc(sizeof(struct ena_bus
), M_DEVBUF
,
3671 /* Store register resources */
3672 ((struct ena_bus
*)(ena_dev
->bus
))->reg_bar_t
=
3673 rman_get_bustag(adapter
->registers
);
3674 ((struct ena_bus
*)(ena_dev
->bus
))->reg_bar_h
=
3675 rman_get_bushandle(adapter
->registers
);
3677 if (unlikely(((struct ena_bus
*)(ena_dev
->bus
))->reg_bar_h
== 0)) {
3678 device_printf(pdev
, "failed to pmap registers bar\n");
3683 ena_dev
->tx_mem_queue_type
= ENA_ADMIN_PLACEMENT_POLICY_HOST
;
3685 /* Device initialization */
3686 rc
= ena_device_init(adapter
, pdev
, &get_feat_ctx
, &adapter
->wd_active
);
3687 if (unlikely(rc
!= 0)) {
3688 device_printf(pdev
, "ENA device init failed! (err: %d)\n", rc
);
3693 getmicrouptime(&adapter
->keep_alive_timestamp
);
3695 adapter
->tx_offload_cap
= get_feat_ctx
.offload
.tx
;
3697 /* Set for sure that interface is not up */
3698 adapter
->up
= false;
3700 memcpy(adapter
->mac_addr
, get_feat_ctx
.dev_attr
.mac_addr
,
3703 /* calculate IO queue number to create */
3704 io_queue_num
= ena_calc_io_queue_num(adapter
, &get_feat_ctx
);
3706 ENA_ASSERT(io_queue_num
> 0, "Invalid queue number: %d\n",
3708 adapter
->num_queues
= io_queue_num
;
3710 adapter
->max_mtu
= get_feat_ctx
.dev_attr
.max_mtu
;
3712 /* calculatre ring sizes */
3713 queue_size
= ena_calc_queue_size(adapter
,&tx_sgl_size
,
3714 &rx_sgl_size
, &get_feat_ctx
);
3715 if (unlikely((queue_size
<= 0) || (io_queue_num
<= 0))) {
3720 adapter
->reset_reason
= ENA_REGS_RESET_NORMAL
;
3722 adapter
->tx_ring_size
= queue_size
;
3723 adapter
->rx_ring_size
= queue_size
;
3725 adapter
->max_tx_sgl_size
= tx_sgl_size
;
3726 adapter
->max_rx_sgl_size
= rx_sgl_size
;
3728 /* set up dma tags for rx and tx buffers */
3729 rc
= ena_setup_tx_dma_tag(adapter
);
3730 if (unlikely(rc
!= 0)) {
3731 device_printf(pdev
, "Failed to create TX DMA tag\n");
3735 rc
= ena_setup_rx_dma_tag(adapter
);
3736 if (unlikely(rc
!= 0)) {
3737 device_printf(pdev
, "Failed to create RX DMA tag\n");
3738 goto err_tx_tag_free
;
3741 /* initialize rings basic information */
3742 device_printf(pdev
, "initialize %d io queues\n", io_queue_num
);
3743 ena_init_io_rings(adapter
);
3745 /* setup network interface */
3746 rc
= ena_setup_ifnet(pdev
, adapter
, &get_feat_ctx
);
3747 if (unlikely(rc
!= 0)) {
3748 device_printf(pdev
, "Error with network interface setup\n");
3752 rc
= ena_enable_msix_and_set_admin_interrupts(adapter
, io_queue_num
);
3753 if (unlikely(rc
!= 0)) {
3755 "Failed to enable and set the admin interrupts\n");
3759 /* Initialize reset task queue */
3760 TASK_INIT(&adapter
->reset_task
, 0, ena_reset_task
, adapter
);
3761 adapter
->reset_tq
= taskqueue_create("ena_reset_enqueue",
3762 M_WAITOK
| M_ZERO
, taskqueue_thread_enqueue
, &adapter
->reset_tq
);
3763 taskqueue_start_threads(&adapter
->reset_tq
, 1, TDPRI_KERN_DAEMON
, -1,
3764 "%s rstq", device_get_nameunit(adapter
->pdev
));
3766 /* Initialize statistics */
3767 #if 0 /* XXX swildner counters */
3768 ena_alloc_counters((counter_u64_t
*)&adapter
->dev_stats
,
3769 sizeof(struct ena_stats_dev
));
3770 ena_alloc_counters((counter_u64_t
*)&adapter
->hw_stats
,
3771 sizeof(struct ena_hw_stats
));
3773 ena_sysctl_add_nodes(adapter
);
3775 /* Tell the stack that the interface is not active */
3776 ifq_set_oactive(&adapter
->ifp
->if_snd
);
3777 adapter
->ifp
->if_flags
&= ~IFF_RUNNING
;
3779 adapter
->running
= true;
3783 if_detach(adapter
->ifp
);
3784 if_free(adapter
->ifp
);
3786 ena_free_all_io_rings_resources(adapter
);
3787 ena_free_rx_dma_tag(adapter
);
3789 ena_free_tx_dma_tag(adapter
);
3791 ena_com_admin_destroy(ena_dev
);
3792 ena_com_delete_host_info(ena_dev
);
3793 ena_com_mmio_reg_read_request_destroy(ena_dev
);
3795 kfree(ena_dev
->bus
, M_DEVBUF
);
3796 kfree(ena_dev
, M_DEVBUF
);
3797 ena_free_pci_resources(adapter
);
3803 * ena_detach - Device Removal Routine
3804 * @pdev: device information struct
3806 * ena_detach is called by the device subsystem to alert the driver
3807 * that it should release a PCI device.
3810 ena_detach(device_t pdev
)
3812 struct ena_adapter
*adapter
= device_get_softc(pdev
);
3813 struct ena_com_dev
*ena_dev
= adapter
->ena_dev
;
3816 /* Make sure VLANS are not using driver */
3817 if (adapter
->ifp
->if_vlantrunks
!= NULL
) {
3818 device_printf(adapter
->pdev
,"VLAN is in use, detach first\n");
3822 /* Free reset task and callout */
3823 callout_drain(&adapter
->timer_service
);
3824 while (taskqueue_cancel(adapter
->reset_tq
, &adapter
->reset_task
, NULL
))
3825 taskqueue_drain(adapter
->reset_tq
, &adapter
->reset_task
);
3826 taskqueue_free(adapter
->reset_tq
);
3828 lockmgr(&adapter
->ioctl_lock
, LK_EXCLUSIVE
);
3830 lockmgr(&adapter
->ioctl_lock
, LK_RELEASE
);
3832 if (adapter
->ifp
!= NULL
) {
3833 ether_ifdetach(adapter
->ifp
);
3834 if_free(adapter
->ifp
);
3837 ena_free_all_io_rings_resources(adapter
);
3839 #if 0 /* XXX swildner counters */
3840 ena_free_counters((counter_u64_t
*)&adapter
->hw_stats
,
3841 sizeof(struct ena_hw_stats
));
3842 ena_free_counters((counter_u64_t
*)&adapter
->dev_stats
,
3843 sizeof(struct ena_stats_dev
));
3846 if (likely(adapter
->rss_support
))
3847 ena_com_rss_destroy(ena_dev
);
3849 rc
= ena_free_rx_dma_tag(adapter
);
3850 if (unlikely(rc
!= 0))
3851 device_printf(adapter
->pdev
,
3852 "Unmapped RX DMA tag associations\n");
3854 rc
= ena_free_tx_dma_tag(adapter
);
3855 if (unlikely(rc
!= 0))
3856 device_printf(adapter
->pdev
,
3857 "Unmapped TX DMA tag associations\n");
3859 /* Reset the device only if the device is running. */
3860 if (adapter
->running
)
3861 ena_com_dev_reset(ena_dev
, adapter
->reset_reason
);
3863 ena_com_delete_host_info(ena_dev
);
3865 ena_free_irqs(adapter
);
3867 ena_com_abort_admin_commands(ena_dev
);
3869 ena_com_wait_for_abort_completion(ena_dev
);
3871 ena_com_admin_destroy(ena_dev
);
3873 ena_com_mmio_reg_read_request_destroy(ena_dev
);
3875 ena_free_pci_resources(adapter
);
3877 lockuninit(&adapter
->global_lock
);
3878 lockuninit(&adapter
->ioctl_lock
);
3880 if (ena_dev
->bus
!= NULL
)
3881 kfree(ena_dev
->bus
, M_DEVBUF
);
3883 if (ena_dev
!= NULL
)
3884 kfree(ena_dev
, M_DEVBUF
);
3886 return (bus_generic_detach(pdev
));
3889 /******************************************************************************
3890 ******************************** AENQ Handlers *******************************
3891 *****************************************************************************/
3893 * ena_update_on_link_change:
3894 * Notify the network interface about the change in link status
3897 ena_update_on_link_change(void *adapter_data
,
3898 struct ena_admin_aenq_entry
*aenq_e
)
3900 struct ena_adapter
*adapter
= (struct ena_adapter
*)adapter_data
;
3901 struct ena_admin_aenq_link_change_desc
*aenq_desc
;
3905 aenq_desc
= (struct ena_admin_aenq_link_change_desc
*)aenq_e
;
3907 status
= aenq_desc
->flags
&
3908 ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK
;
3911 device_printf(adapter
->pdev
, "link is UP\n");
3912 ifp
->if_link_state
= LINK_STATE_UP
;
3913 if_link_state_change(ifp
);
3914 } else if (status
== 0) {
3915 device_printf(adapter
->pdev
, "link is DOWN\n");
3916 ifp
->if_link_state
= LINK_STATE_DOWN
;
3917 if_link_state_change(ifp
);
3919 device_printf(adapter
->pdev
, "invalid value recvd\n");
3923 adapter
->link_status
= status
;
3927 * This handler will called for unknown event group or unimplemented handlers
3930 unimplemented_aenq_handler(void *data
,
3931 struct ena_admin_aenq_entry
*aenq_e
)
3936 static struct ena_aenq_handlers aenq_handlers
= {
3938 [ENA_ADMIN_LINK_CHANGE
] = ena_update_on_link_change
,
3939 [ENA_ADMIN_KEEP_ALIVE
] = ena_keep_alive_wd
,
3941 .unimplemented_handler
= unimplemented_aenq_handler
3944 /*********************************************************************
3945 * FreeBSD Device Interface Entry Points
3946 *********************************************************************/
3948 static device_method_t ena_methods
[] = {
3949 /* Device interface */
3950 DEVMETHOD(device_probe
, ena_probe
),
3951 DEVMETHOD(device_attach
, ena_attach
),
3952 DEVMETHOD(device_detach
, ena_detach
),
3956 static driver_t ena_driver
= {
3957 "ena", ena_methods
, sizeof(struct ena_adapter
),
3960 devclass_t ena_devclass
;
3961 DRIVER_MODULE(ena
, pci
, ena_driver
, ena_devclass
, NULL
, NULL
);
3962 MODULE_DEPEND(ena
, pci
, 1, 1, 1);
3963 MODULE_DEPEND(ena
, ether
, 1, 1, 1);
3965 /*********************************************************************/