4 * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 * $FreeBSD: head/sys/dev/ena/ena.c 325593 2017-11-09 13:38:17Z mw $
33 #include <sys/param.h>
34 #include <sys/systm.h>
36 #include <sys/endian.h>
37 #include <sys/kernel.h>
38 #include <sys/kthread.h>
39 #include <sys/malloc.h>
41 #include <sys/module.h>
43 #include <sys/socket.h>
44 #include <sys/sockio.h>
45 #include <sys/sysctl.h>
46 #include <sys/taskqueue.h>
48 #include <sys/eventhandler.h>
52 #include <net/if_var.h>
53 #include <net/if_arp.h>
54 #include <net/if_dl.h>
55 #include <net/if_media.h>
56 #include <net/if_types.h>
57 #include <net/ifq_var.h>
58 #include <net/vlan/if_vlan_var.h>
60 #include <netinet/in_systm.h>
61 #include <netinet/in.h>
62 #include <netinet/if_ether.h>
63 #include <netinet/ip.h>
64 #include <netinet/ip6.h>
65 #include <netinet/tcp.h>
66 #include <netinet/udp.h>
68 #include <bus/pci/pcivar.h>
69 #include <bus/pci/pcireg.h>
72 #include "ena_sysctl.h"
74 /*********************************************************
76 *********************************************************/
77 static int ena_probe(device_t
);
78 static void ena_intr_msix_mgmnt(void *);
79 static int ena_allocate_pci_resources(struct ena_adapter
*);
80 static void ena_free_pci_resources(struct ena_adapter
*);
81 static int ena_change_mtu(if_t
, int);
82 #if 0 /* XXX swildner counters */
83 static inline void ena_alloc_counters(counter_u64_t
*, int);
84 static inline void ena_free_counters(counter_u64_t
*, int);
85 static inline void ena_reset_counters(counter_u64_t
*, int);
87 static void ena_init_io_rings_common(struct ena_adapter
*,
88 struct ena_ring
*, uint16_t);
89 static void ena_init_io_rings(struct ena_adapter
*);
90 static void ena_free_io_ring_resources(struct ena_adapter
*, unsigned int);
91 static void ena_free_all_io_rings_resources(struct ena_adapter
*);
92 static int ena_setup_tx_dma_tag(struct ena_adapter
*);
93 static int ena_free_tx_dma_tag(struct ena_adapter
*);
94 static int ena_setup_rx_dma_tag(struct ena_adapter
*);
95 static int ena_free_rx_dma_tag(struct ena_adapter
*);
96 static int ena_setup_tx_resources(struct ena_adapter
*, int);
97 static void ena_free_tx_resources(struct ena_adapter
*, int);
98 static int ena_setup_all_tx_resources(struct ena_adapter
*);
99 static void ena_free_all_tx_resources(struct ena_adapter
*);
100 static inline int validate_rx_req_id(struct ena_ring
*, uint16_t);
101 static int ena_setup_rx_resources(struct ena_adapter
*, unsigned int);
102 static void ena_free_rx_resources(struct ena_adapter
*, unsigned int);
103 static int ena_setup_all_rx_resources(struct ena_adapter
*);
104 static void ena_free_all_rx_resources(struct ena_adapter
*);
105 static inline int ena_alloc_rx_mbuf(struct ena_adapter
*, struct ena_ring
*,
106 struct ena_rx_buffer
*);
107 static void ena_free_rx_mbuf(struct ena_adapter
*, struct ena_ring
*,
108 struct ena_rx_buffer
*);
109 static int ena_refill_rx_bufs(struct ena_ring
*, uint32_t);
110 static void ena_free_rx_bufs(struct ena_adapter
*, unsigned int);
111 static void ena_refill_all_rx_bufs(struct ena_adapter
*);
112 static void ena_free_all_rx_bufs(struct ena_adapter
*);
113 static void ena_free_tx_bufs(struct ena_adapter
*, unsigned int);
114 static void ena_free_all_tx_bufs(struct ena_adapter
*);
115 static void ena_destroy_all_tx_queues(struct ena_adapter
*);
116 static void ena_destroy_all_rx_queues(struct ena_adapter
*);
117 static void ena_destroy_all_io_queues(struct ena_adapter
*);
118 static int ena_create_io_queues(struct ena_adapter
*);
119 static int ena_tx_cleanup(struct ena_ring
*);
120 static void ena_deferred_rx_cleanup(void *, int);
121 static int ena_rx_cleanup(struct ena_ring
*);
122 static inline int validate_tx_req_id(struct ena_ring
*, uint16_t);
123 static void ena_rx_hash_mbuf(struct ena_ring
*, struct ena_com_rx_ctx
*,
125 static struct mbuf
* ena_rx_mbuf(struct ena_ring
*, struct ena_com_rx_buf_info
*,
126 struct ena_com_rx_ctx
*, uint16_t *);
127 static inline void ena_rx_checksum(struct ena_ring
*, struct ena_com_rx_ctx
*,
129 static void ena_handle_msix(void *);
130 static int ena_enable_msix(struct ena_adapter
*);
131 static void ena_setup_mgmnt_intr(struct ena_adapter
*);
132 static void ena_setup_io_intr(struct ena_adapter
*);
133 static int ena_request_mgmnt_irq(struct ena_adapter
*);
134 static int ena_request_io_irq(struct ena_adapter
*);
135 static void ena_free_mgmnt_irq(struct ena_adapter
*);
136 static void ena_free_io_irq(struct ena_adapter
*);
137 static void ena_free_irqs(struct ena_adapter
*);
138 static void ena_disable_msix(struct ena_adapter
*);
139 static void ena_unmask_all_io_irqs(struct ena_adapter
*);
140 static int ena_rss_configure(struct ena_adapter
*);
141 static int ena_up_complete(struct ena_adapter
*);
142 static int ena_up(struct ena_adapter
*);
143 static void ena_down(struct ena_adapter
*);
144 #if 0 /* XXX swildner counters */
145 static uint64_t ena_get_counter(if_t
, ift_counter
);
147 static int ena_media_change(if_t
);
148 static void ena_media_status(if_t
, struct ifmediareq
*);
149 static void ena_init(void *);
150 static int ena_ioctl(if_t
, u_long
, caddr_t
, struct ucred
*);
151 static int ena_get_dev_offloads(struct ena_com_dev_get_features_ctx
*);
152 static void ena_update_host_info(struct ena_admin_host_info
*, if_t
);
153 static void ena_update_hwassist(struct ena_adapter
*);
154 static int ena_setup_ifnet(device_t
, struct ena_adapter
*,
155 struct ena_com_dev_get_features_ctx
*);
156 static void ena_tx_csum(struct ena_com_tx_ctx
*, struct mbuf
*);
157 static int ena_check_and_collapse_mbuf(struct ena_ring
*tx_ring
,
159 static int ena_xmit_mbuf(struct ena_ring
*, struct mbuf
**);
160 static void ena_start_xmit(struct ifnet
*, struct ifaltq_subque
*);
161 static int ena_calc_io_queue_num(struct ena_adapter
*,
162 struct ena_com_dev_get_features_ctx
*);
163 static int ena_calc_queue_size(struct ena_adapter
*, uint16_t *,
164 uint16_t *, struct ena_com_dev_get_features_ctx
*);
165 static int ena_rss_init_default(struct ena_adapter
*);
166 static void ena_rss_init_default_deferred(void *);
167 static void ena_config_host_info(struct ena_com_dev
*);
168 static int ena_attach(device_t
);
169 static int ena_detach(device_t
);
170 static int ena_device_init(struct ena_adapter
*, device_t
,
171 struct ena_com_dev_get_features_ctx
*, int *);
172 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter
*,
174 static void ena_update_on_link_change(void *, struct ena_admin_aenq_entry
*);
175 static void unimplemented_aenq_handler(void *,
176 struct ena_admin_aenq_entry
*);
177 static void ena_timer_service(void *);
179 static char ena_version
[] = DEVICE_NAME DRV_MODULE_NAME
" v" DRV_MODULE_VERSION
;
181 static SYSCTL_NODE(_hw
, OID_AUTO
, ena
, CTLFLAG_RD
, 0, "ENA driver parameters");
184 * Logging level for changing verbosity of the output
186 int ena_log_level
= ENA_ALERT
| ENA_WARNING
;
187 TUNABLE_INT("hw.ena.ena_log_level", &ena_log_level
);
188 SYSCTL_INT(_hw_ena
, OID_AUTO
, log_level
, CTLFLAG_RW
,
189 &ena_log_level
, 0, "Logging level indicating verbosity of the logs");
191 static ena_vendor_info_t ena_vendor_info_array
[] = {
192 { PCI_VENDOR_ID_AMAZON
, PCI_DEV_ID_ENA_PF
, 0},
193 { PCI_VENDOR_ID_AMAZON
, PCI_DEV_ID_ENA_LLQ_PF
, 0},
194 { PCI_VENDOR_ID_AMAZON
, PCI_DEV_ID_ENA_VF
, 0},
195 { PCI_VENDOR_ID_AMAZON
, PCI_DEV_ID_ENA_LLQ_VF
, 0},
201 * Contains pointers to event handlers, e.g. link state chage.
203 static struct ena_aenq_handlers aenq_handlers
;
206 ena_dmamap_callback(void *arg
, bus_dma_segment_t
*segs
, int nseg
, int error
)
210 *(bus_addr_t
*) arg
= segs
[0].ds_addr
;
214 ena_dma_alloc(device_t dmadev
, bus_size_t size
,
215 ena_mem_handle_t
*dma
, int mapflags
)
217 struct ena_adapter
* adapter
= device_get_softc(dmadev
);
219 uint64_t dma_space_addr
;
222 maxsize
= ((size
- 1) / PAGE_SIZE
+ 1) * PAGE_SIZE
;
224 dma_space_addr
= ENA_DMA_BIT_MASK(adapter
->dma_width
);
225 if (unlikely(dma_space_addr
== 0))
226 dma_space_addr
= BUS_SPACE_MAXADDR
;
228 error
= bus_dma_tag_create(bus_get_dma_tag(dmadev
), /* parent */
229 8, 0, /* alignment, bounds */
230 dma_space_addr
, /* lowaddr of exclusion window */
231 BUS_SPACE_MAXADDR
,/* highaddr of exclusion window */
232 NULL
, NULL
, /* filter, filterarg */
233 maxsize
, /* maxsize */
235 maxsize
, /* maxsegsize */
236 BUS_DMA_ALLOCNOW
, /* flags */
238 if (unlikely(error
!= 0)) {
239 ena_trace(ENA_ALERT
, "bus_dma_tag_create failed: %d\n", error
);
243 error
= bus_dmamem_alloc(dma
->tag
, (void**) &dma
->vaddr
,
244 BUS_DMA_COHERENT
| BUS_DMA_ZERO
, &dma
->map
);
245 if (unlikely(error
!= 0)) {
246 ena_trace(ENA_ALERT
, "bus_dmamem_alloc(%ju) failed: %d\n",
247 (uintmax_t)size
, error
);
248 goto fail_map_create
;
252 error
= bus_dmamap_load(dma
->tag
, dma
->map
, dma
->vaddr
,
253 size
, ena_dmamap_callback
, &dma
->paddr
, mapflags
);
254 if (unlikely((error
!= 0) || (dma
->paddr
== 0))) {
255 ena_trace(ENA_ALERT
, ": bus_dmamap_load failed: %d\n", error
);
262 bus_dmamem_free(dma
->tag
, dma
->vaddr
, dma
->map
);
264 bus_dma_tag_destroy(dma
->tag
);
272 ena_allocate_pci_resources(struct ena_adapter
* adapter
)
274 device_t pdev
= adapter
->pdev
;
277 rid
= PCIR_BAR(ENA_REG_BAR
);
278 adapter
->memory
= NULL
;
279 adapter
->registers
= bus_alloc_resource_any(pdev
, SYS_RES_MEMORY
,
281 if (unlikely(adapter
->registers
== NULL
)) {
282 device_printf(pdev
, "Unable to allocate bus resource: "
291 ena_free_pci_resources(struct ena_adapter
*adapter
)
293 device_t pdev
= adapter
->pdev
;
295 if (adapter
->memory
!= NULL
) {
296 bus_release_resource(pdev
, SYS_RES_MEMORY
,
297 PCIR_BAR(ENA_MEM_BAR
), adapter
->memory
);
300 if (adapter
->registers
!= NULL
) {
301 bus_release_resource(pdev
, SYS_RES_MEMORY
,
302 PCIR_BAR(ENA_REG_BAR
), adapter
->registers
);
307 ena_probe(device_t dev
)
309 ena_vendor_info_t
*ent
;
310 char adapter_name
[60];
311 uint16_t pci_vendor_id
= 0;
312 uint16_t pci_device_id
= 0;
314 pci_vendor_id
= pci_get_vendor(dev
);
315 pci_device_id
= pci_get_device(dev
);
317 ent
= ena_vendor_info_array
;
318 while (ent
->vendor_id
!= 0) {
319 if ((pci_vendor_id
== ent
->vendor_id
) &&
320 (pci_device_id
== ent
->device_id
)) {
321 ena_trace(ENA_DBG
, "vendor=%x device=%x ",
322 pci_vendor_id
, pci_device_id
);
324 ksprintf(adapter_name
, DEVICE_DESC
);
325 device_set_desc_copy(dev
, adapter_name
);
326 return (BUS_PROBE_DEFAULT
);
337 ena_change_mtu(if_t ifp
, int new_mtu
)
339 struct ena_adapter
*adapter
= ifp
->if_softc
;
342 if ((new_mtu
> adapter
->max_mtu
) || (new_mtu
< ENA_MIN_MTU
)) {
343 device_printf(adapter
->pdev
, "Invalid MTU setting. "
344 "new_mtu: %d max mtu: %d min mtu: %d\n",
345 new_mtu
, adapter
->max_mtu
, ENA_MIN_MTU
);
349 rc
= ena_com_set_dev_mtu(adapter
->ena_dev
, new_mtu
);
350 if (likely(rc
== 0)) {
351 ena_trace(ENA_DBG
, "set MTU to %d\n", new_mtu
);
352 ifp
->if_mtu
= new_mtu
;
354 device_printf(adapter
->pdev
, "Failed to set MTU to %d\n",
361 #if 0 /* XXX swildner counters */
363 ena_alloc_counters(counter_u64_t
*begin
, int size
)
365 counter_u64_t
*end
= (counter_u64_t
*)((char *)begin
+ size
);
367 for (; begin
< end
; ++begin
)
368 *begin
= counter_u64_alloc(M_WAITOK
);
372 ena_free_counters(counter_u64_t
*begin
, int size
)
374 counter_u64_t
*end
= (counter_u64_t
*)((char *)begin
+ size
);
376 for (; begin
< end
; ++begin
)
377 counter_u64_free(*begin
);
381 ena_reset_counters(counter_u64_t
*begin
, int size
)
383 counter_u64_t
*end
= (counter_u64_t
*)((char *)begin
+ size
);
385 for (; begin
< end
; ++begin
)
386 counter_u64_zero(*begin
);
391 ena_init_io_rings_common(struct ena_adapter
*adapter
, struct ena_ring
*ring
,
396 ring
->adapter
= adapter
;
397 ring
->ena_dev
= adapter
->ena_dev
;
401 ena_init_io_rings(struct ena_adapter
*adapter
)
403 struct ena_com_dev
*ena_dev
;
404 struct ena_ring
*txr
, *rxr
;
408 ena_dev
= adapter
->ena_dev
;
410 for (i
= 0; i
< adapter
->num_queues
; i
++) {
411 txr
= &adapter
->tx_ring
[i
];
412 rxr
= &adapter
->rx_ring
[i
];
414 /* TX/RX common ring state */
415 ena_init_io_rings_common(adapter
, txr
, i
);
416 ena_init_io_rings_common(adapter
, rxr
, i
);
418 /* TX specific ring state */
419 txr
->ring_size
= adapter
->tx_ring_size
;
420 txr
->tx_max_header_size
= ena_dev
->tx_max_header_size
;
421 txr
->tx_mem_queue_type
= ena_dev
->tx_mem_queue_type
;
422 txr
->smoothed_interval
=
423 ena_com_get_nonadaptive_moderation_interval_tx(ena_dev
);
425 #if 0 /* XXX swildner counters */
426 /* Alloc TX statistics. */
427 ena_alloc_counters((counter_u64_t
*)&txr
->tx_stats
,
428 sizeof(txr
->tx_stats
));
431 /* RX specific ring state */
432 rxr
->ring_size
= adapter
->rx_ring_size
;
433 rxr
->smoothed_interval
=
434 ena_com_get_nonadaptive_moderation_interval_rx(ena_dev
);
436 #if 0 /* XXX swildner counters */
437 /* Alloc RX statistics. */
438 ena_alloc_counters((counter_u64_t
*)&rxr
->rx_stats
,
439 sizeof(rxr
->rx_stats
));
442 /* Initialize locks */
443 ksnprintf(txr
->lock_name
, nitems(txr
->lock_name
), "%s:tx(%d)",
444 device_get_nameunit(adapter
->pdev
), i
);
445 ksnprintf(rxr
->lock_name
, nitems(rxr
->lock_name
), "%s:rx(%d)",
446 device_get_nameunit(adapter
->pdev
), i
);
448 lockinit(&txr
->ring_lock
, txr
->lock_name
, 0, LK_CANRECURSE
);
449 lockinit(&rxr
->ring_lock
, rxr
->lock_name
, 0, LK_CANRECURSE
);
451 que
= &adapter
->que
[i
];
452 que
->adapter
= adapter
;
460 rxr
->empty_rx_queue
= 0;
465 ena_free_io_ring_resources(struct ena_adapter
*adapter
, unsigned int qid
)
467 struct ena_ring
*txr
= &adapter
->tx_ring
[qid
];
468 struct ena_ring
*rxr
= &adapter
->rx_ring
[qid
];
470 #if 0 /* XXX swildner counters */
471 ena_free_counters((counter_u64_t
*)&txr
->tx_stats
,
472 sizeof(txr
->tx_stats
));
473 ena_free_counters((counter_u64_t
*)&rxr
->rx_stats
,
474 sizeof(rxr
->rx_stats
));
477 lockuninit(&txr
->ring_lock
);
478 lockuninit(&rxr
->ring_lock
);
482 ena_free_all_io_rings_resources(struct ena_adapter
*adapter
)
486 for (i
= 0; i
< adapter
->num_queues
; i
++)
487 ena_free_io_ring_resources(adapter
, i
);
492 ena_setup_tx_dma_tag(struct ena_adapter
*adapter
)
496 /* Create DMA tag for Tx buffers */
497 ret
= bus_dma_tag_create(bus_get_dma_tag(adapter
->pdev
),
498 1, 0, /* alignment, bounds */
499 ENA_DMA_BIT_MASK(adapter
->dma_width
), /* lowaddr of excl window */
500 BUS_SPACE_MAXADDR
, /* highaddr of excl window */
501 NULL
, NULL
, /* filter, filterarg */
502 ENA_TSO_MAXSIZE
, /* maxsize */
503 ENA_BUS_DMA_SEGS
, /* nsegments */
504 ENA_TSO_MAXSIZE
, /* maxsegsize */
506 &adapter
->tx_buf_tag
);
512 ena_free_tx_dma_tag(struct ena_adapter
*adapter
)
516 ret
= bus_dma_tag_destroy(adapter
->tx_buf_tag
);
518 if (likely(ret
== 0))
519 adapter
->tx_buf_tag
= NULL
;
525 ena_setup_rx_dma_tag(struct ena_adapter
*adapter
)
529 /* Create DMA tag for Rx buffers*/
530 ret
= bus_dma_tag_create(bus_get_dma_tag(adapter
->pdev
), /* parent */
531 1, 0, /* alignment, bounds */
532 ENA_DMA_BIT_MASK(adapter
->dma_width
), /* lowaddr of excl window */
533 BUS_SPACE_MAXADDR
, /* highaddr of excl window */
534 NULL
, NULL
, /* filter, filterarg */
535 MJUM16BYTES
, /* maxsize */
536 adapter
->max_rx_sgl_size
, /* nsegments */
537 MJUM16BYTES
, /* maxsegsize */
539 &adapter
->rx_buf_tag
);
545 ena_free_rx_dma_tag(struct ena_adapter
*adapter
)
549 ret
= bus_dma_tag_destroy(adapter
->rx_buf_tag
);
551 if (likely(ret
== 0))
552 adapter
->rx_buf_tag
= NULL
;
558 * ena_setup_tx_resources - allocate Tx resources (Descriptors)
559 * @adapter: network interface device structure
562 * Returns 0 on success, otherwise on failure.
565 ena_setup_tx_resources(struct ena_adapter
*adapter
, int qid
)
567 struct ena_que
*que
= &adapter
->que
[qid
];
568 struct ena_ring
*tx_ring
= que
->tx_ring
;
574 size
= sizeof(struct ena_tx_buffer
) * tx_ring
->ring_size
;
576 tx_ring
->tx_buffer_info
= kmalloc(size
, M_DEVBUF
, M_NOWAIT
| M_ZERO
);
577 if (unlikely(tx_ring
->tx_buffer_info
== NULL
))
580 size
= sizeof(uint16_t) * tx_ring
->ring_size
;
581 tx_ring
->free_tx_ids
= kmalloc(size
, M_DEVBUF
, M_NOWAIT
| M_ZERO
);
582 if (unlikely(tx_ring
->free_tx_ids
== NULL
))
583 goto err_buf_info_free
;
585 /* Req id stack for TX OOO completions */
586 for (i
= 0; i
< tx_ring
->ring_size
; i
++)
587 tx_ring
->free_tx_ids
[i
] = i
;
589 #if 0 /* XXX swildner counters */
590 /* Reset TX statistics. */
591 ena_reset_counters((counter_u64_t
*)&tx_ring
->tx_stats
,
592 sizeof(tx_ring
->tx_stats
));
595 tx_ring
->next_to_use
= 0;
596 tx_ring
->next_to_clean
= 0;
598 /* ... and create the buffer DMA maps */
599 for (i
= 0; i
< tx_ring
->ring_size
; i
++) {
600 err
= bus_dmamap_create(adapter
->tx_buf_tag
, 0,
601 &tx_ring
->tx_buffer_info
[i
].map
);
602 if (unlikely(err
!= 0)) {
604 "Unable to create Tx DMA map for buffer %d\n", i
);
605 goto err_buf_info_unmap
;
613 bus_dmamap_destroy(adapter
->tx_buf_tag
,
614 tx_ring
->tx_buffer_info
[i
].map
);
616 kfree(tx_ring
->free_tx_ids
, M_DEVBUF
);
617 tx_ring
->free_tx_ids
= NULL
;
619 kfree(tx_ring
->tx_buffer_info
, M_DEVBUF
);
620 tx_ring
->tx_buffer_info
= NULL
;
626 * ena_free_tx_resources - Free Tx Resources per Queue
627 * @adapter: network interface device structure
630 * Free all transmit software resources
633 ena_free_tx_resources(struct ena_adapter
*adapter
, int qid
)
635 struct ena_ring
*tx_ring
= &adapter
->tx_ring
[qid
];
637 ENA_RING_MTX_LOCK(tx_ring
);
639 /* Free buffer DMA maps, */
640 for (int i
= 0; i
< tx_ring
->ring_size
; i
++) {
641 m_freem(tx_ring
->tx_buffer_info
[i
].mbuf
);
642 tx_ring
->tx_buffer_info
[i
].mbuf
= NULL
;
643 bus_dmamap_unload(adapter
->tx_buf_tag
,
644 tx_ring
->tx_buffer_info
[i
].map
);
645 bus_dmamap_destroy(adapter
->tx_buf_tag
,
646 tx_ring
->tx_buffer_info
[i
].map
);
648 ENA_RING_MTX_UNLOCK(tx_ring
);
650 /* And free allocated memory. */
651 kfree(tx_ring
->tx_buffer_info
, M_DEVBUF
);
652 tx_ring
->tx_buffer_info
= NULL
;
654 kfree(tx_ring
->free_tx_ids
, M_DEVBUF
);
655 tx_ring
->free_tx_ids
= NULL
;
659 * ena_setup_all_tx_resources - allocate all queues Tx resources
660 * @adapter: network interface device structure
662 * Returns 0 on success, otherwise on failure.
665 ena_setup_all_tx_resources(struct ena_adapter
*adapter
)
669 for (i
= 0; i
< adapter
->num_queues
; i
++) {
670 rc
= ena_setup_tx_resources(adapter
, i
);
672 device_printf(adapter
->pdev
,
673 "Allocation for Tx Queue %u failed\n", i
);
681 /* Rewind the index freeing the rings as we go */
683 ena_free_tx_resources(adapter
, i
);
688 * ena_free_all_tx_resources - Free Tx Resources for All Queues
689 * @adapter: network interface device structure
691 * Free all transmit software resources
694 ena_free_all_tx_resources(struct ena_adapter
*adapter
)
698 for (i
= 0; i
< adapter
->num_queues
; i
++)
699 ena_free_tx_resources(adapter
, i
);
703 validate_rx_req_id(struct ena_ring
*rx_ring
, uint16_t req_id
)
705 if (likely(req_id
< rx_ring
->ring_size
))
708 device_printf(rx_ring
->adapter
->pdev
, "Invalid rx req_id: %hu\n",
710 IFNET_STAT_INC(rx_ring
->adapter
->ifp
, ierrors
, 1);
711 #if 0 /* XXX swildner counters */
712 counter_u64_add(rx_ring
->rx_stats
.bad_req_id
, 1);
715 /* Trigger device reset */
716 rx_ring
->adapter
->reset_reason
= ENA_REGS_RESET_INV_RX_REQ_ID
;
717 rx_ring
->adapter
->trigger_reset
= true;
723 * ena_setup_rx_resources - allocate Rx resources (Descriptors)
724 * @adapter: network interface device structure
727 * Returns 0 on success, otherwise on failure.
730 ena_setup_rx_resources(struct ena_adapter
*adapter
, unsigned int qid
)
732 struct ena_que
*que
= &adapter
->que
[qid
];
733 struct ena_ring
*rx_ring
= que
->rx_ring
;
739 size
= sizeof(struct ena_rx_buffer
) * rx_ring
->ring_size
;
742 * Alloc extra element so in rx path
743 * we can always prefetch rx_info + 1
745 size
+= sizeof(struct ena_rx_buffer
);
747 rx_ring
->rx_buffer_info
= kmalloc(size
, M_DEVBUF
, M_WAITOK
| M_ZERO
);
749 size
= sizeof(uint16_t) * rx_ring
->ring_size
;
750 rx_ring
->free_rx_ids
= kmalloc(size
, M_DEVBUF
, M_WAITOK
);
752 for (i
= 0; i
< rx_ring
->ring_size
; i
++)
753 rx_ring
->free_rx_ids
[i
] = i
;
755 #if 0 /* XXX swildner counters */
756 /* Reset RX statistics. */
757 ena_reset_counters((counter_u64_t
*)&rx_ring
->rx_stats
,
758 sizeof(rx_ring
->rx_stats
));
761 rx_ring
->next_to_clean
= 0;
762 rx_ring
->next_to_use
= 0;
764 /* ... and create the buffer DMA maps */
765 for (i
= 0; i
< rx_ring
->ring_size
; i
++) {
766 err
= bus_dmamap_create(adapter
->rx_buf_tag
, 0,
767 &(rx_ring
->rx_buffer_info
[i
].map
));
770 "Unable to create Rx DMA map for buffer %d\n", i
);
771 goto err_buf_info_unmap
;
776 /* Create LRO for the ring */
777 if ((adapter
->ifp
->if_capenable
& IFCAP_LRO
) != 0) {
778 int err
= tcp_lro_init(&rx_ring
->lro
);
780 device_printf(adapter
->pdev
,
781 "LRO[%d] Initialization failed!\n", qid
);
784 "RX Soft LRO[%d] Initialized\n", qid
);
785 rx_ring
->lro
.ifp
= adapter
->ifp
;
790 /* Allocate taskqueues */
791 TASK_INIT(&rx_ring
->cmpl_task
, 0, ena_deferred_rx_cleanup
, rx_ring
);
792 rx_ring
->cmpl_tq
= taskqueue_create("ena RX completion", M_WAITOK
,
793 taskqueue_thread_enqueue
, &rx_ring
->cmpl_tq
);
795 /* RSS set cpu for thread */
797 CPU_SETOF(que
->cpu
, &cpu_mask
);
798 taskqueue_start_threads_cpuset(&rx_ring
->cmpl_tq
, 1, PI_NET
, &cpu_mask
,
799 "%s rx_ring cmpl (bucket %d)",
800 device_get_nameunit(adapter
->pdev
), que
->cpu
);
802 taskqueue_start_threads(&rx_ring
->cmpl_tq
, 1, TDPRI_KERN_DAEMON
, -1,
803 "%s rx_ring cmpl", device_get_nameunit(adapter
->pdev
));
810 bus_dmamap_destroy(adapter
->rx_buf_tag
,
811 rx_ring
->rx_buffer_info
[i
].map
);
814 kfree(rx_ring
->free_rx_ids
, M_DEVBUF
);
815 rx_ring
->free_rx_ids
= NULL
;
816 kfree(rx_ring
->rx_buffer_info
, M_DEVBUF
);
817 rx_ring
->rx_buffer_info
= NULL
;
822 * ena_free_rx_resources - Free Rx Resources
823 * @adapter: network interface device structure
826 * Free all receive software resources
829 ena_free_rx_resources(struct ena_adapter
*adapter
, unsigned int qid
)
831 struct ena_ring
*rx_ring
= &adapter
->rx_ring
[qid
];
833 while (taskqueue_cancel(rx_ring
->cmpl_tq
, &rx_ring
->cmpl_task
, NULL
) != 0)
834 taskqueue_drain(rx_ring
->cmpl_tq
, &rx_ring
->cmpl_task
);
836 taskqueue_free(rx_ring
->cmpl_tq
);
838 /* Free buffer DMA maps, */
839 for (int i
= 0; i
< rx_ring
->ring_size
; i
++) {
840 m_freem(rx_ring
->rx_buffer_info
[i
].mbuf
);
841 rx_ring
->rx_buffer_info
[i
].mbuf
= NULL
;
842 bus_dmamap_unload(adapter
->rx_buf_tag
,
843 rx_ring
->rx_buffer_info
[i
].map
);
844 bus_dmamap_destroy(adapter
->rx_buf_tag
,
845 rx_ring
->rx_buffer_info
[i
].map
);
849 /* free LRO resources, */
850 tcp_lro_free(&rx_ring
->lro
);
853 /* free allocated memory */
854 kfree(rx_ring
->rx_buffer_info
, M_DEVBUF
);
855 rx_ring
->rx_buffer_info
= NULL
;
857 kfree(rx_ring
->free_rx_ids
, M_DEVBUF
);
858 rx_ring
->free_rx_ids
= NULL
;
862 * ena_setup_all_rx_resources - allocate all queues Rx resources
863 * @adapter: network interface device structure
865 * Returns 0 on success, otherwise on failure.
868 ena_setup_all_rx_resources(struct ena_adapter
*adapter
)
872 for (i
= 0; i
< adapter
->num_queues
; i
++) {
873 rc
= ena_setup_rx_resources(adapter
, i
);
875 device_printf(adapter
->pdev
,
876 "Allocation for Rx Queue %u failed\n", i
);
883 /* rewind the index freeing the rings as we go */
885 ena_free_rx_resources(adapter
, i
);
890 * ena_free_all_rx_resources - Free Rx resources for all queues
891 * @adapter: network interface device structure
893 * Free all receive software resources
896 ena_free_all_rx_resources(struct ena_adapter
*adapter
)
900 for (i
= 0; i
< adapter
->num_queues
; i
++)
901 ena_free_rx_resources(adapter
, i
);
905 ena_alloc_rx_mbuf(struct ena_adapter
*adapter
,
906 struct ena_ring
*rx_ring
, struct ena_rx_buffer
*rx_info
)
908 struct ena_com_buf
*ena_buf
;
909 bus_dma_segment_t segs
[1];
913 /* if previous allocated frag is not used */
914 if (unlikely(rx_info
->mbuf
!= NULL
))
917 /* Get mbuf using UMA allocator */
918 rx_info
->mbuf
= m_getjcl(M_NOWAIT
, MT_DATA
, M_PKTHDR
, MJUMPAGESIZE
);
920 if (unlikely(rx_info
->mbuf
== NULL
)) {
921 #if 0 /* XXX swildner counters */
922 counter_u64_add(rx_ring
->rx_stats
.mjum_alloc_fail
, 1);
924 rx_info
->mbuf
= m_getcl(M_NOWAIT
, MT_DATA
, M_PKTHDR
);
925 if (unlikely(rx_info
->mbuf
== NULL
)) {
926 IFNET_STAT_INC(rx_ring
->adapter
->ifp
, ierrors
, 1);
927 #if 0 /* XXX swildner counters */
928 counter_u64_add(rx_ring
->rx_stats
.mbuf_alloc_fail
, 1);
937 rx_info
->mbuf
->m_pkthdr
.len
= rx_info
->mbuf
->m_len
= mlen
;
939 /* Map packets for DMA */
940 ena_trace(ENA_DBG
| ENA_RSC
| ENA_RXPTH
,
941 "Using tag %p for buffers' DMA mapping, mbuf %p len: %d",
942 adapter
->rx_buf_tag
,rx_info
->mbuf
, rx_info
->mbuf
->m_len
);
943 error
= bus_dmamap_load_mbuf_segment(adapter
->rx_buf_tag
, rx_info
->map
,
944 rx_info
->mbuf
, segs
, 1, &nsegs
, BUS_DMA_NOWAIT
);
945 if (unlikely((error
!= 0) || (nsegs
!= 1))) {
946 ena_trace(ENA_WARNING
, "failed to map mbuf, error: %d, "
947 "nsegs: %d\n", error
, nsegs
);
948 IFNET_STAT_INC(rx_ring
->adapter
->ifp
, ierrors
, 1);
949 #if 0 /* XXX swildner counters */
950 counter_u64_add(rx_ring
->rx_stats
.dma_mapping_err
, 1);
956 bus_dmamap_sync(adapter
->rx_buf_tag
, rx_info
->map
, BUS_DMASYNC_PREREAD
);
958 ena_buf
= &rx_info
->ena_buf
;
959 ena_buf
->paddr
= segs
[0].ds_addr
;
962 ena_trace(ENA_DBG
| ENA_RSC
| ENA_RXPTH
,
963 "ALLOC RX BUF: mbuf %p, rx_info %p, len %d, paddr %#jx\n",
964 rx_info
->mbuf
, rx_info
,ena_buf
->len
, (uintmax_t)ena_buf
->paddr
);
969 m_freem(rx_info
->mbuf
);
970 rx_info
->mbuf
= NULL
;
975 ena_free_rx_mbuf(struct ena_adapter
*adapter
, struct ena_ring
*rx_ring
,
976 struct ena_rx_buffer
*rx_info
)
979 if (rx_info
->mbuf
== NULL
) {
980 ena_trace(ENA_WARNING
, "Trying to free unallocated buffer\n");
984 bus_dmamap_unload(adapter
->rx_buf_tag
, rx_info
->map
);
985 m_freem(rx_info
->mbuf
);
986 rx_info
->mbuf
= NULL
;
990 * ena_refill_rx_bufs - Refills ring with descriptors
991 * @rx_ring: the ring which we want to feed with free descriptors
992 * @num: number of descriptors to refill
993 * Refills the ring with newly allocated DMA-mapped mbufs for receiving
996 ena_refill_rx_bufs(struct ena_ring
*rx_ring
, uint32_t num
)
998 struct ena_adapter
*adapter
= rx_ring
->adapter
;
999 uint16_t next_to_use
, req_id
;
1003 ena_trace(ENA_DBG
| ENA_RXPTH
| ENA_RSC
, "refill qid: %d",
1006 next_to_use
= rx_ring
->next_to_use
;
1008 for (i
= 0; i
< num
; i
++) {
1009 struct ena_rx_buffer
*rx_info
;
1011 ena_trace(ENA_DBG
| ENA_RXPTH
| ENA_RSC
,
1012 "RX buffer - next to use: %d", next_to_use
);
1014 req_id
= rx_ring
->free_rx_ids
[next_to_use
];
1015 rc
= validate_rx_req_id(rx_ring
, req_id
);
1016 if (unlikely(rc
!= 0))
1019 rx_info
= &rx_ring
->rx_buffer_info
[req_id
];
1021 rc
= ena_alloc_rx_mbuf(adapter
, rx_ring
, rx_info
);
1022 if (unlikely(rc
!= 0)) {
1023 ena_trace(ENA_WARNING
,
1024 "failed to alloc buffer for rx queue %d\n",
1028 rc
= ena_com_add_single_rx_desc(rx_ring
->ena_com_io_sq
,
1029 &rx_info
->ena_buf
, req_id
);
1030 if (unlikely(rc
!= 0)) {
1031 ena_trace(ENA_WARNING
,
1032 "failed to add buffer for rx queue %d\n",
1036 next_to_use
= ENA_RX_RING_IDX_NEXT(next_to_use
,
1037 rx_ring
->ring_size
);
1040 if (unlikely(i
< num
)) {
1041 IFNET_STAT_INC(rx_ring
->adapter
->ifp
, ierrors
, 1);
1042 #if 0 /* XXX swildner counters */
1043 counter_u64_add(rx_ring
->rx_stats
.refil_partial
, 1);
1045 ena_trace(ENA_WARNING
,
1046 "refilled rx qid %d with only %d mbufs (from %d)\n",
1047 rx_ring
->qid
, i
, num
);
1050 if (likely(i
!= 0)) {
1052 ena_com_write_sq_doorbell(rx_ring
->ena_com_io_sq
);
1054 rx_ring
->next_to_use
= next_to_use
;
1059 ena_free_rx_bufs(struct ena_adapter
*adapter
, unsigned int qid
)
1061 struct ena_ring
*rx_ring
= &adapter
->rx_ring
[qid
];
1064 for (i
= 0; i
< rx_ring
->ring_size
; i
++) {
1065 struct ena_rx_buffer
*rx_info
= &rx_ring
->rx_buffer_info
[i
];
1067 if (rx_info
->mbuf
!= NULL
)
1068 ena_free_rx_mbuf(adapter
, rx_ring
, rx_info
);
1073 * ena_refill_all_rx_bufs - allocate all queues Rx buffers
1074 * @adapter: network interface device structure
1078 ena_refill_all_rx_bufs(struct ena_adapter
*adapter
)
1080 struct ena_ring
*rx_ring
;
1081 int i
, rc
, bufs_num
;
1083 for (i
= 0; i
< adapter
->num_queues
; i
++) {
1084 rx_ring
= &adapter
->rx_ring
[i
];
1085 bufs_num
= rx_ring
->ring_size
- 1;
1086 rc
= ena_refill_rx_bufs(rx_ring
, bufs_num
);
1088 if (unlikely(rc
!= bufs_num
))
1089 ena_trace(ENA_WARNING
, "refilling Queue %d failed. "
1090 "Allocated %d buffers from: %d\n", i
, rc
, bufs_num
);
1095 ena_free_all_rx_bufs(struct ena_adapter
*adapter
)
1099 for (i
= 0; i
< adapter
->num_queues
; i
++)
1100 ena_free_rx_bufs(adapter
, i
);
1104 * ena_free_tx_bufs - Free Tx Buffers per Queue
1105 * @adapter: network interface device structure
1109 ena_free_tx_bufs(struct ena_adapter
*adapter
, unsigned int qid
)
1111 bool print_once
= true;
1112 struct ena_ring
*tx_ring
= &adapter
->tx_ring
[qid
];
1114 ENA_RING_MTX_LOCK(tx_ring
);
1115 for (int i
= 0; i
< tx_ring
->ring_size
; i
++) {
1116 struct ena_tx_buffer
*tx_info
= &tx_ring
->tx_buffer_info
[i
];
1118 if (tx_info
->mbuf
== NULL
)
1122 device_printf(adapter
->pdev
,
1123 "free uncompleted tx mbuf qid %d idx 0x%x",
1128 "free uncompleted tx mbuf qid %d idx 0x%x",
1132 bus_dmamap_unload(adapter
->tx_buf_tag
, tx_info
->map
);
1133 m_free(tx_info
->mbuf
);
1134 tx_info
->mbuf
= NULL
;
1136 ENA_RING_MTX_UNLOCK(tx_ring
);
1140 ena_free_all_tx_bufs(struct ena_adapter
*adapter
)
1143 for (int i
= 0; i
< adapter
->num_queues
; i
++)
1144 ena_free_tx_bufs(adapter
, i
);
1148 ena_destroy_all_tx_queues(struct ena_adapter
*adapter
)
1153 for (i
= 0; i
< adapter
->num_queues
; i
++) {
1154 ena_qid
= ENA_IO_TXQ_IDX(i
);
1155 ena_com_destroy_io_queue(adapter
->ena_dev
, ena_qid
);
1160 ena_destroy_all_rx_queues(struct ena_adapter
*adapter
)
1165 for (i
= 0; i
< adapter
->num_queues
; i
++) {
1166 ena_qid
= ENA_IO_RXQ_IDX(i
);
1167 ena_com_destroy_io_queue(adapter
->ena_dev
, ena_qid
);
1172 ena_destroy_all_io_queues(struct ena_adapter
*adapter
)
1174 ena_destroy_all_tx_queues(adapter
);
1175 ena_destroy_all_rx_queues(adapter
);
1179 validate_tx_req_id(struct ena_ring
*tx_ring
, uint16_t req_id
)
1181 struct ena_adapter
*adapter
= tx_ring
->adapter
;
1182 struct ena_tx_buffer
*tx_info
= NULL
;
1184 if (likely(req_id
< tx_ring
->ring_size
)) {
1185 tx_info
= &tx_ring
->tx_buffer_info
[req_id
];
1186 if (tx_info
->mbuf
!= NULL
)
1190 if (tx_info
->mbuf
== NULL
)
1191 device_printf(adapter
->pdev
,
1192 "tx_info doesn't have valid mbuf\n");
1194 device_printf(adapter
->pdev
, "Invalid req_id: %hu\n", req_id
);
1196 IFNET_STAT_INC(tx_ring
->adapter
->ifp
, oerrors
, 1);
1197 #if 0 /* XXX swildner counters */
1198 counter_u64_add(tx_ring
->tx_stats
.bad_req_id
, 1);
1205 ena_create_io_queues(struct ena_adapter
*adapter
)
1207 struct ena_com_dev
*ena_dev
= adapter
->ena_dev
;
1208 struct ena_com_create_io_ctx ctx
;
1209 struct ena_ring
*ring
;
1211 uint32_t msix_vector
;
1214 /* Create TX queues */
1215 for (i
= 0; i
< adapter
->num_queues
; i
++) {
1216 msix_vector
= ENA_IO_IRQ_IDX(i
);
1217 ena_qid
= ENA_IO_TXQ_IDX(i
);
1218 ctx
.mem_queue_type
= ena_dev
->tx_mem_queue_type
;
1219 ctx
.direction
= ENA_COM_IO_QUEUE_DIRECTION_TX
;
1220 ctx
.queue_size
= adapter
->tx_ring_size
;
1221 ctx
.msix_vector
= msix_vector
;
1223 rc
= ena_com_create_io_queue(ena_dev
, &ctx
);
1225 device_printf(adapter
->pdev
,
1226 "Failed to create io TX queue #%d rc: %d\n", i
, rc
);
1229 ring
= &adapter
->tx_ring
[i
];
1230 rc
= ena_com_get_io_handlers(ena_dev
, ena_qid
,
1231 &ring
->ena_com_io_sq
,
1232 &ring
->ena_com_io_cq
);
1234 device_printf(adapter
->pdev
,
1235 "Failed to get TX queue handlers. TX queue num"
1236 " %d rc: %d\n", i
, rc
);
1237 ena_com_destroy_io_queue(ena_dev
, ena_qid
);
1242 /* Create RX queues */
1243 for (i
= 0; i
< adapter
->num_queues
; i
++) {
1244 msix_vector
= ENA_IO_IRQ_IDX(i
);
1245 ena_qid
= ENA_IO_RXQ_IDX(i
);
1246 ctx
.mem_queue_type
= ENA_ADMIN_PLACEMENT_POLICY_HOST
;
1247 ctx
.direction
= ENA_COM_IO_QUEUE_DIRECTION_RX
;
1248 ctx
.queue_size
= adapter
->rx_ring_size
;
1249 ctx
.msix_vector
= msix_vector
;
1251 rc
= ena_com_create_io_queue(ena_dev
, &ctx
);
1252 if (unlikely(rc
!= 0)) {
1253 device_printf(adapter
->pdev
,
1254 "Failed to create io RX queue[%d] rc: %d\n", i
, rc
);
1258 ring
= &adapter
->rx_ring
[i
];
1259 rc
= ena_com_get_io_handlers(ena_dev
, ena_qid
,
1260 &ring
->ena_com_io_sq
,
1261 &ring
->ena_com_io_cq
);
1262 if (unlikely(rc
!= 0)) {
1263 device_printf(adapter
->pdev
,
1264 "Failed to get RX queue handlers. RX queue num"
1265 " %d rc: %d\n", i
, rc
);
1266 ena_com_destroy_io_queue(ena_dev
, ena_qid
);
1275 ena_com_destroy_io_queue(ena_dev
, ENA_IO_RXQ_IDX(i
));
1276 i
= adapter
->num_queues
;
1279 ena_com_destroy_io_queue(ena_dev
, ENA_IO_TXQ_IDX(i
));
1285 * ena_tx_cleanup - clear sent packets and corresponding descriptors
1286 * @tx_ring: ring for which we want to clean packets
1288 * Once packets are sent, we ask the device in a loop for no longer used
1289 * descriptors. We find the related mbuf chain in a map (index in an array)
1290 * and free it, then update ring state.
1291 * This is performed in "endless" loop, updating ring pointers every
1292 * TX_COMMIT. The first check of free descriptor is performed before the actual
1293 * loop, then repeated at the loop end.
1296 ena_tx_cleanup(struct ena_ring
*tx_ring
)
1298 struct ena_adapter
*adapter
;
1299 struct ena_com_io_cq
* io_cq
;
1300 uint16_t next_to_clean
;
1303 unsigned int total_done
= 0;
1305 int commit
= TX_COMMIT
;
1306 int budget
= TX_BUDGET
;
1309 adapter
= tx_ring
->que
->adapter
;
1310 ena_qid
= ENA_IO_TXQ_IDX(tx_ring
->que
->id
);
1311 io_cq
= &adapter
->ena_dev
->io_cq_queues
[ena_qid
];
1312 next_to_clean
= tx_ring
->next_to_clean
;
1315 struct ena_tx_buffer
*tx_info
;
1318 rc
= ena_com_tx_comp_req_id_get(io_cq
, &req_id
);
1319 if (unlikely(rc
!= 0))
1322 rc
= validate_tx_req_id(tx_ring
, req_id
);
1323 if (unlikely(rc
!= 0))
1326 tx_info
= &tx_ring
->tx_buffer_info
[req_id
];
1328 mbuf
= tx_info
->mbuf
;
1330 tx_info
->mbuf
= NULL
;
1331 timevalclear(&tx_info
->timestamp
);
1333 if (likely(tx_info
->num_of_bufs
!= 0)) {
1334 /* Map is no longer required */
1335 bus_dmamap_unload(adapter
->tx_buf_tag
, tx_info
->map
);
1338 ena_trace(ENA_DBG
| ENA_TXPTH
, "tx: q %d mbuf %p completed",
1339 tx_ring
->qid
, mbuf
);
1343 total_done
+= tx_info
->tx_descs
;
1345 tx_ring
->free_tx_ids
[next_to_clean
] = req_id
;
1346 next_to_clean
= ENA_TX_RING_IDX_NEXT(next_to_clean
,
1347 tx_ring
->ring_size
);
1349 if (unlikely(--commit
== 0)) {
1351 /* update ring state every TX_COMMIT descriptor */
1352 tx_ring
->next_to_clean
= next_to_clean
;
1354 &adapter
->ena_dev
->io_sq_queues
[ena_qid
],
1356 ena_com_update_dev_comp_head(io_cq
);
1359 } while (likely(--budget
));
1361 work_done
= TX_BUDGET
- budget
;
1363 ena_trace(ENA_DBG
| ENA_TXPTH
, "tx: q %d done. total pkts: %d",
1364 tx_ring
->qid
, work_done
);
1366 /* If there is still something to commit update ring state */
1367 if (likely(commit
!= TX_COMMIT
)) {
1368 tx_ring
->next_to_clean
= next_to_clean
;
1369 ena_com_comp_ack(&adapter
->ena_dev
->io_sq_queues
[ena_qid
],
1371 ena_com_update_dev_comp_head(io_cq
);
1378 ena_rx_hash_mbuf(struct ena_ring
*rx_ring
, struct ena_com_rx_ctx
*ena_rx_ctx
,
1381 struct ena_adapter
*adapter
= rx_ring
->adapter
;
1383 if (likely(adapter
->rss_support
)) {
1384 //mbuf->m_pkthdr.flowid = ena_rx_ctx->hash;
1385 m_sethash(mbuf
, ena_rx_ctx
->hash
);
1387 #if 0 /* XXX rsstype doesn't seem to be needed by the network stack, we will only supply the hash. */
1388 if (ena_rx_ctx
->frag
&&
1389 (ena_rx_ctx
->l3_proto
!= ENA_ETH_IO_L3_PROTO_UNKNOWN
)) {
1390 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_OPAQUE_HASH
);
1394 switch (ena_rx_ctx
->l3_proto
) {
1395 case ENA_ETH_IO_L3_PROTO_IPV4
:
1396 switch (ena_rx_ctx
->l4_proto
) {
1397 case ENA_ETH_IO_L4_PROTO_TCP
:
1398 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_RSS_TCP_IPV4
);
1400 case ENA_ETH_IO_L4_PROTO_UDP
:
1401 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_RSS_UDP_IPV4
);
1404 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_RSS_IPV4
);
1407 case ENA_ETH_IO_L3_PROTO_IPV6
:
1408 switch (ena_rx_ctx
->l4_proto
) {
1409 case ENA_ETH_IO_L4_PROTO_TCP
:
1410 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_RSS_TCP_IPV6
);
1412 case ENA_ETH_IO_L4_PROTO_UDP
:
1413 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_RSS_UDP_IPV6
);
1416 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_RSS_IPV6
);
1419 case ENA_ETH_IO_L3_PROTO_UNKNOWN
:
1420 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_NONE
);
1423 M_HASHTYPE_SET(mbuf
, M_HASHTYPE_OPAQUE_HASH
);
1427 //mbuf->m_pkthdr.flowid = rx_ring->qid;
1428 //M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
1429 m_sethash(mbuf
, rx_ring
->qid
);
1434 * ena_rx_mbuf - assemble mbuf from descriptors
1435 * @rx_ring: ring for which we want to clean packets
1436 * @ena_bufs: buffer info
1437 * @ena_rx_ctx: metadata for this packet(s)
1438 * @next_to_clean: ring pointer, will be updated only upon success
1442 ena_rx_mbuf(struct ena_ring
*rx_ring
, struct ena_com_rx_buf_info
*ena_bufs
,
1443 struct ena_com_rx_ctx
*ena_rx_ctx
, uint16_t *next_to_clean
)
1446 struct ena_rx_buffer
*rx_info
;
1447 struct ena_adapter
*adapter
;
1448 unsigned int descs
= ena_rx_ctx
->descs
;
1449 uint16_t ntc
, len
, req_id
, buf
= 0;
1451 ntc
= *next_to_clean
;
1452 adapter
= rx_ring
->adapter
;
1453 rx_info
= &rx_ring
->rx_buffer_info
[ntc
];
1455 if (unlikely(rx_info
->mbuf
== NULL
)) {
1456 device_printf(adapter
->pdev
, "NULL mbuf in rx_info");
1460 len
= ena_bufs
[buf
].len
;
1461 req_id
= ena_bufs
[buf
].req_id
;
1462 rx_info
= &rx_ring
->rx_buffer_info
[req_id
];
1464 ena_trace(ENA_DBG
| ENA_RXPTH
, "rx_info %p, mbuf %p, paddr %jx",
1465 rx_info
, rx_info
->mbuf
, (uintmax_t)rx_info
->ena_buf
.paddr
);
1467 mbuf
= rx_info
->mbuf
;
1468 mbuf
->m_flags
|= M_PKTHDR
;
1469 mbuf
->m_pkthdr
.len
= len
;
1471 mbuf
->m_pkthdr
.rcvif
= rx_ring
->que
->adapter
->ifp
;
1473 /* Fill mbuf with hash key and it's interpretation for optimization */
1474 ena_rx_hash_mbuf(rx_ring
, ena_rx_ctx
, mbuf
);
1476 ena_trace(ENA_DBG
| ENA_RXPTH
, "rx mbuf 0x%p, flags=0x%x, len: %d",
1477 mbuf
, mbuf
->m_flags
, mbuf
->m_pkthdr
.len
);
1479 /* DMA address is not needed anymore, unmap it */
1480 bus_dmamap_unload(rx_ring
->adapter
->rx_buf_tag
, rx_info
->map
);
1482 rx_info
->mbuf
= NULL
;
1483 rx_ring
->free_rx_ids
[ntc
] = req_id
;
1484 ntc
= ENA_RX_RING_IDX_NEXT(ntc
, rx_ring
->ring_size
);
1487 * While we have more than 1 descriptors for one rcvd packet, append
1488 * other mbufs to the main one
1492 len
= ena_bufs
[buf
].len
;
1493 req_id
= ena_bufs
[buf
].req_id
;
1494 rx_info
= &rx_ring
->rx_buffer_info
[req_id
];
1496 if (unlikely(rx_info
->mbuf
== NULL
)) {
1497 device_printf(adapter
->pdev
, "NULL mbuf in rx_info");
1499 * If one of the required mbufs was not allocated yet,
1500 * we can break there.
1501 * All earlier used descriptors will be reallocated
1502 * later and not used mbufs can be reused.
1503 * The next_to_clean pointer will not be updated in case
1504 * of an error, so caller should advance it manually
1505 * in error handling routine to keep it up to date
1512 if (unlikely(m_append(mbuf
, len
, rx_info
->mbuf
->m_data
) == 0)) {
1513 IFNET_STAT_INC(rx_ring
->adapter
->ifp
, ierrors
, 1);
1514 #if 0 /* XXX swildner counters */
1515 counter_u64_add(rx_ring
->rx_stats
.mbuf_alloc_fail
, 1);
1517 ena_trace(ENA_WARNING
, "Failed to append Rx mbuf %p",
1521 ena_trace(ENA_DBG
| ENA_RXPTH
,
1522 "rx mbuf updated. len %d", mbuf
->m_pkthdr
.len
);
1524 /* Free already appended mbuf, it won't be useful anymore */
1525 bus_dmamap_unload(rx_ring
->adapter
->rx_buf_tag
, rx_info
->map
);
1526 m_freem(rx_info
->mbuf
);
1527 rx_info
->mbuf
= NULL
;
1529 rx_ring
->free_rx_ids
[ntc
] = req_id
;
1530 ntc
= ENA_RX_RING_IDX_NEXT(ntc
, rx_ring
->ring_size
);
1533 *next_to_clean
= ntc
;
1539 * ena_rx_checksum - indicate in mbuf if hw indicated a good cksum
1542 ena_rx_checksum(struct ena_ring
*rx_ring
, struct ena_com_rx_ctx
*ena_rx_ctx
,
1546 /* if IP and error */
1547 if (unlikely((ena_rx_ctx
->l3_proto
== ENA_ETH_IO_L3_PROTO_IPV4
) &&
1548 ena_rx_ctx
->l3_csum_err
)) {
1549 /* ipv4 checksum error */
1550 mbuf
->m_pkthdr
.csum_flags
= 0;
1551 IFNET_STAT_INC(rx_ring
->adapter
->ifp
, ierrors
, 1);
1552 #if 0 /* XXX swildner counters */
1553 counter_u64_add(rx_ring
->rx_stats
.bad_csum
, 1);
1555 ena_trace(ENA_DBG
, "RX IPv4 header checksum error");
1560 if ((ena_rx_ctx
->l4_proto
== ENA_ETH_IO_L4_PROTO_TCP
) ||
1561 (ena_rx_ctx
->l4_proto
== ENA_ETH_IO_L4_PROTO_UDP
)) {
1562 if (ena_rx_ctx
->l4_csum_err
) {
1563 /* TCP/UDP checksum error */
1564 mbuf
->m_pkthdr
.csum_flags
= 0;
1565 IFNET_STAT_INC(rx_ring
->adapter
->ifp
, ierrors
, 1);
1566 #if 0 /* XXX swildner counters */
1567 counter_u64_add(rx_ring
->rx_stats
.bad_csum
, 1);
1569 ena_trace(ENA_DBG
, "RX L4 checksum error");
1571 mbuf
->m_pkthdr
.csum_flags
= CSUM_IP_CHECKED
;
1572 mbuf
->m_pkthdr
.csum_flags
|= CSUM_IP_VALID
;
1578 ena_deferred_rx_cleanup(void *arg
, int pending
)
1580 struct ena_ring
*rx_ring
= arg
;
1581 int budget
= CLEAN_BUDGET
;
1583 ENA_RING_MTX_LOCK(rx_ring
);
1585 * If deferred task was executed, perform cleanup of all awaiting
1586 * descs (or until given budget is depleted to avoid infinite loop).
1588 while (likely(budget
--)) {
1589 if (ena_rx_cleanup(rx_ring
) == 0)
1592 ENA_RING_MTX_UNLOCK(rx_ring
);
1596 * ena_rx_cleanup - handle rx irq
1597 * @arg: ring for which irq is being handled
1600 ena_rx_cleanup(struct ena_ring
*rx_ring
)
1602 struct ena_adapter
*adapter
;
1604 struct ena_com_rx_ctx ena_rx_ctx
;
1605 struct ena_com_io_cq
* io_cq
;
1606 struct ena_com_io_sq
* io_sq
;
1609 uint16_t next_to_clean
;
1610 uint32_t refill_required
;
1611 uint32_t refill_threshold
;
1612 uint32_t do_if_input
= 0;
1615 int budget
= RX_BUDGET
;
1617 adapter
= rx_ring
->que
->adapter
;
1619 qid
= rx_ring
->que
->id
;
1620 ena_qid
= ENA_IO_RXQ_IDX(qid
);
1621 io_cq
= &adapter
->ena_dev
->io_cq_queues
[ena_qid
];
1622 io_sq
= &adapter
->ena_dev
->io_sq_queues
[ena_qid
];
1623 next_to_clean
= rx_ring
->next_to_clean
;
1625 ena_trace(ENA_DBG
, "rx: qid %d", qid
);
1628 ena_rx_ctx
.ena_bufs
= rx_ring
->ena_bufs
;
1629 ena_rx_ctx
.max_bufs
= adapter
->max_rx_sgl_size
;
1630 ena_rx_ctx
.descs
= 0;
1631 rc
= ena_com_rx_pkt(io_cq
, io_sq
, &ena_rx_ctx
);
1633 if (unlikely(rc
!= 0))
1636 if (unlikely(ena_rx_ctx
.descs
== 0))
1639 ena_trace(ENA_DBG
| ENA_RXPTH
, "rx: q %d got packet from ena. "
1640 "descs #: %d l3 proto %d l4 proto %d hash: %x",
1641 rx_ring
->qid
, ena_rx_ctx
.descs
, ena_rx_ctx
.l3_proto
,
1642 ena_rx_ctx
.l4_proto
, ena_rx_ctx
.hash
);
1644 /* Receive mbuf from the ring */
1645 mbuf
= ena_rx_mbuf(rx_ring
, rx_ring
->ena_bufs
,
1646 &ena_rx_ctx
, &next_to_clean
);
1648 /* Exit if we failed to retrieve a buffer */
1649 if (unlikely(mbuf
== NULL
)) {
1650 for (i
= 0; i
< ena_rx_ctx
.descs
; ++i
) {
1651 rx_ring
->free_rx_ids
[next_to_clean
] =
1652 rx_ring
->ena_bufs
[i
].req_id
;
1654 ENA_RX_RING_IDX_NEXT(next_to_clean
,
1655 rx_ring
->ring_size
);
1662 * XXX Removed IFCAP_RXCSUM_IPV6 check because DragonFly
1663 * does not seem to support it
1665 if ((ifp
->if_capenable
& IFCAP_RXCSUM
) != 0) {
1666 ena_rx_checksum(rx_ring
, &ena_rx_ctx
, mbuf
);
1669 #if 0 /* XXX swildner counters */
1671 counter_u64_add_protected(rx_ring
->rx_stats
.bytes
,
1672 mbuf
->m_pkthdr
.len
);
1673 counter_u64_add_protected(adapter
->hw_stats
.rx_bytes
,
1674 mbuf
->m_pkthdr
.len
);
1678 * LRO is only for IP/TCP packets and TCP checksum of the packet
1679 * should be computed by hardware.
1683 if (((ifp
->if_capenable
& IFCAP_LRO
) != 0) &&
1684 ((mbuf
->m_pkthdr
.csum_flags
& CSUM_IP_VALID
) != 0) &&
1685 (ena_rx_ctx
.l4_proto
== ENA_ETH_IO_L4_PROTO_TCP
)) {
1687 * Send to the stack if:
1688 * - LRO not enabled, or
1689 * - no LRO resources, or
1690 * - lro enqueue fails
1692 if ((rx_ring
->lro
.lro_cnt
!= 0) &&
1693 (tcp_lro_rx(&rx_ring
->lro
, mbuf
, 0) == 0))
1697 if (do_if_input
!= 0) {
1698 ena_trace(ENA_DBG
| ENA_RXPTH
,
1699 "calling if_input() with mbuf %p", mbuf
);
1700 ENA_RING_MTX_UNLOCK(rx_ring
);
1701 (*ifp
->if_input
)(ifp
, mbuf
, NULL
, -1);
1702 ENA_RING_MTX_LOCK(rx_ring
);
1705 IFNET_STAT_INC(ifp
, ipackets
, 1);
1706 #if 0 /* XXX swildner counters */
1708 counter_u64_add_protected(rx_ring
->rx_stats
.cnt
, 1);
1709 counter_u64_add_protected(adapter
->hw_stats
.rx_packets
, 1);
1714 rx_ring
->next_to_clean
= next_to_clean
;
1716 refill_required
= ena_com_free_desc(io_sq
);
1717 refill_threshold
= rx_ring
->ring_size
/ ENA_RX_REFILL_THRESH_DIVIDER
;
1719 if (refill_required
> refill_threshold
) {
1720 ena_com_update_dev_comp_head(rx_ring
->ena_com_io_cq
);
1721 ena_refill_rx_bufs(rx_ring
, refill_required
);
1725 tcp_lro_flush_all(&rx_ring
->lro
);
1728 return (RX_BUDGET
- budget
);
1731 IFNET_STAT_INC(rx_ring
->adapter
->ifp
, ierrors
, 1);
1732 #if 0 /* XXX swildner counters */
1733 counter_u64_add(rx_ring
->rx_stats
.bad_desc_num
, 1);
1735 return (RX_BUDGET
- budget
);
1738 /*********************************************************************
1740 * MSIX & Interrupt Service routine
1742 **********************************************************************/
1745 * ena_handle_msix - MSIX Interrupt Handler for admin/async queue
1746 * @arg: interrupt number
1749 ena_intr_msix_mgmnt(void *arg
)
1751 struct ena_adapter
*adapter
= (struct ena_adapter
*)arg
;
1753 ena_com_admin_q_comp_intr_handler(adapter
->ena_dev
);
1754 if (likely(adapter
->running
))
1755 ena_com_aenq_intr_handler(adapter
->ena_dev
, arg
);
1759 * ena_handle_msix - MSIX Interrupt Handler for Tx/Rx
1760 * @arg: interrupt number
1763 ena_handle_msix(void *arg
)
1765 struct ena_que
*que
= arg
;
1766 struct ena_adapter
*adapter
= que
->adapter
;
1767 if_t ifp
= adapter
->ifp
;
1768 struct ena_ring
*tx_ring
;
1769 struct ena_ring
*rx_ring
;
1770 struct ena_com_io_cq
* io_cq
;
1771 struct ena_eth_io_intr_reg intr_reg
;
1775 if (unlikely((ifp
->if_flags
& IFF_RUNNING
) == 0))
1778 ena_trace(ENA_DBG
, "MSI-X TX/RX routine");
1780 tx_ring
= que
->tx_ring
;
1781 rx_ring
= que
->rx_ring
;
1783 ena_qid
= ENA_IO_TXQ_IDX(qid
);
1784 io_cq
= &adapter
->ena_dev
->io_cq_queues
[ena_qid
];
1786 for (i
= 0; i
< CLEAN_BUDGET
; ++i
) {
1788 * If lock cannot be acquired, then deferred cleanup task was
1789 * being executed and rx ring is being cleaned up in
1792 if (likely(ENA_RING_MTX_TRYLOCK(rx_ring
) != 0)) {
1793 rxc
= ena_rx_cleanup(rx_ring
);
1794 ENA_RING_MTX_UNLOCK(rx_ring
);
1799 /* Protection from calling ena_tx_cleanup from ena_start_xmit */
1800 ENA_RING_MTX_LOCK(tx_ring
);
1801 txc
= ena_tx_cleanup(tx_ring
);
1802 ENA_RING_MTX_UNLOCK(tx_ring
);
1804 if (unlikely((ifp
->if_flags
& IFF_RUNNING
) == 0))
1807 if ((txc
!= TX_BUDGET
) && (rxc
!= RX_BUDGET
))
1811 /* Signal that work is done and unmask interrupt */
1812 ena_com_update_intr_reg(&intr_reg
,
1816 ena_com_unmask_intr(io_cq
, &intr_reg
);
1820 ena_enable_msix(struct ena_adapter
*adapter
)
1822 device_t dev
= adapter
->pdev
;
1824 int error
, i
, rc
= 0;
1826 /* Reserved the max msix vectors we might need */
1827 msix_vecs
= ENA_MAX_MSIX_VEC(adapter
->num_queues
);
1829 adapter
->msix_entries
= kmalloc(msix_vecs
* sizeof(struct msix_entry
),
1830 M_DEVBUF
, M_WAITOK
| M_ZERO
);
1832 ena_trace(ENA_DBG
, "trying to enable MSI-X, vectors: %d", msix_vecs
);
1834 for (i
= 0; i
< msix_vecs
; i
++) {
1835 adapter
->msix_entries
[i
].entry
= i
;
1836 /* Vectors must start from 1 */
1837 adapter
->msix_entries
[i
].vector
= i
+ 1;
1840 error
= pci_setup_msix(dev
);
1842 device_printf(dev
, "pci_setup_msix() failed\n");
1846 adapter
->msix_vecs
= msix_vecs
;
1847 adapter
->msix_enabled
= true;
1852 kfree(adapter
->msix_entries
, M_DEVBUF
);
1853 adapter
->msix_entries
= NULL
;
1859 ena_setup_mgmnt_intr(struct ena_adapter
*adapter
)
1862 ksnprintf(adapter
->irq_tbl
[ENA_MGMNT_IRQ_IDX
].name
,
1863 ENA_IRQNAME_SIZE
, "ena-mgmnt@pci:%s",
1864 device_get_nameunit(adapter
->pdev
));
1866 * Handler is NULL on purpose, it will be set
1867 * when mgmnt interrupt is acquired
1869 adapter
->irq_tbl
[ENA_MGMNT_IRQ_IDX
].handler
= NULL
;
1870 adapter
->irq_tbl
[ENA_MGMNT_IRQ_IDX
].data
= adapter
;
1871 adapter
->irq_tbl
[ENA_MGMNT_IRQ_IDX
].vector
=
1872 adapter
->msix_entries
[ENA_MGMNT_IRQ_IDX
].vector
;
1876 ena_setup_io_intr(struct ena_adapter
*adapter
)
1878 static int last_bind_cpu
= -1;
1881 for (int i
= 0; i
< adapter
->num_queues
; i
++) {
1882 irq_idx
= ENA_IO_IRQ_IDX(i
);
1884 ksnprintf(adapter
->irq_tbl
[irq_idx
].name
, ENA_IRQNAME_SIZE
,
1885 "%s-TxRx-%d", device_get_nameunit(adapter
->pdev
), i
);
1886 adapter
->irq_tbl
[irq_idx
].handler
= ena_handle_msix
;
1887 adapter
->irq_tbl
[irq_idx
].data
= &adapter
->que
[i
];
1888 adapter
->irq_tbl
[irq_idx
].vector
=
1889 adapter
->msix_entries
[irq_idx
].vector
;
1890 ena_trace(ENA_INFO
| ENA_IOQ
, "ena_setup_io_intr vector: %d\n",
1891 adapter
->msix_entries
[irq_idx
].vector
);
1893 adapter
->que
[i
].cpu
= adapter
->irq_tbl
[irq_idx
].cpu
=
1894 rss_getcpu(i
% rss_getnumbuckets());
1897 * We still want to bind rings to the corresponding cpu
1898 * using something similar to the RSS round-robin technique.
1900 * XXX It seems that this can be removed since DragonFly has
1901 * native support for RSS. DragonFly also does not have
1902 * support for CPU_FIRST or CPU_NEXT.
1905 if (last_bind_cpu
< 0)
1906 last_bind_cpu
= (last_bind_cpu
+ 1) % ncpus
;
1907 adapter
->que
[i
].cpu
= adapter
->irq_tbl
[irq_idx
].cpu
=
1909 last_bind_cpu
= (last_bind_cpu
+ 1) % ncpus
;
1915 ena_request_mgmnt_irq(struct ena_adapter
*adapter
)
1917 struct ena_irq
*irq
;
1918 unsigned long flags
;
1921 flags
= RF_ACTIVE
| RF_SHAREABLE
;
1923 irq
= &adapter
->irq_tbl
[ENA_MGMNT_IRQ_IDX
];
1925 error
= pci_alloc_msix_vector(adapter
->pdev
, 0, &irq
->vector
, 0);
1927 device_printf(adapter
->pdev
, "Could not initialize MGMNT MSI-X Vector on cpu0\n");
1931 irq
->res
= bus_alloc_resource_any(adapter
->pdev
, SYS_RES_IRQ
,
1932 &irq
->vector
, flags
);
1934 if (unlikely(irq
->res
== NULL
)) {
1935 device_printf(adapter
->pdev
, "could not allocate "
1936 "irq vector: %d\n", irq
->vector
);
1940 rc
= bus_activate_resource(adapter
->pdev
, SYS_RES_IRQ
,
1941 irq
->vector
, irq
->res
);
1942 if (unlikely(rc
!= 0)) {
1943 device_printf(adapter
->pdev
, "could not activate "
1944 "irq vector: %d\n", irq
->vector
);
1948 rc
= bus_setup_intr(adapter
->pdev
, irq
->res
,
1949 INTR_MPSAFE
, ena_intr_msix_mgmnt
,
1950 irq
->data
, &irq
->cookie
, NULL
);
1951 if (unlikely(rc
!= 0)) {
1952 device_printf(adapter
->pdev
, "failed to register "
1953 "interrupt handler for irq %ju: %d\n",
1954 rman_get_start(irq
->res
), rc
);
1957 irq
->requested
= true;
1962 ena_trace(ENA_INFO
| ENA_ADMQ
, "releasing resource for irq %d\n",
1964 rcc
= bus_release_resource(adapter
->pdev
, SYS_RES_IRQ
,
1965 irq
->vector
, irq
->res
);
1966 pci_release_msix_vector(adapter
->pdev
, irq
->vector
);
1967 if (unlikely(rcc
!= 0))
1968 device_printf(adapter
->pdev
, "dev has no parent while "
1969 "releasing res for irq: %d\n", irq
->vector
);
1976 ena_request_io_irq(struct ena_adapter
*adapter
)
1978 struct ena_irq
*irq
;
1979 unsigned long flags
= 0;
1980 int rc
= 0, i
, rcc
, error
;
1982 if (unlikely(adapter
->msix_enabled
== 0)) {
1983 device_printf(adapter
->pdev
,
1984 "failed to request I/O IRQ: MSI-X is not enabled\n");
1987 flags
= RF_ACTIVE
| RF_SHAREABLE
;
1990 for (i
= ENA_IO_IRQ_FIRST_IDX
; i
< adapter
->msix_vecs
; i
++) {
1991 irq
= &adapter
->irq_tbl
[i
];
1993 if (unlikely(irq
->requested
))
1996 error
= pci_alloc_msix_vector(adapter
->pdev
, i
, &irq
->vector
, irq
->cpu
);
1998 device_printf(adapter
->pdev
, "Unable to allocated MSI-X %d on cpu%d\n", i
, irq
->cpu
);
2002 irq
->res
= bus_alloc_resource_any(adapter
->pdev
, SYS_RES_IRQ
,
2003 &irq
->vector
, flags
);
2004 if (unlikely(irq
->res
== NULL
)) {
2005 device_printf(adapter
->pdev
, "could not allocate "
2006 "irq vector: %d\n", irq
->vector
);
2012 * TODO: Might need to setup desc and use irq->name as the
2015 rc
= bus_setup_intr(adapter
->pdev
, irq
->res
,
2017 irq
->handler
, irq
->data
, &irq
->cookie
, NULL
);
2018 if (unlikely(rc
!= 0)) {
2019 device_printf(adapter
->pdev
, "failed to register "
2020 "interrupt handler for irq %ju: %d\n",
2021 rman_get_start(irq
->res
), rc
);
2024 irq
->requested
= true;
2027 ena_trace(ENA_INFO
, "queue %d - RSS bucket %d\n",
2028 i
- ENA_IO_IRQ_FIRST_IDX
, irq
->cpu
);
2030 ena_trace(ENA_INFO
, "queue %d - cpu %d\n",
2031 i
- ENA_IO_IRQ_FIRST_IDX
, irq
->cpu
);
2039 for (; i
>= ENA_IO_IRQ_FIRST_IDX
; i
--) {
2040 irq
= &adapter
->irq_tbl
[i
];
2043 /* Once we entered err: section and irq->requested is true we
2044 free both intr and resources */
2046 rcc
= bus_teardown_intr(adapter
->pdev
, irq
->res
, irq
->cookie
);
2047 if (unlikely(rcc
!= 0))
2048 device_printf(adapter
->pdev
, "could not release"
2049 " irq: %d, error: %d\n", irq
->vector
, rcc
);
2051 /* If we entred err: section without irq->requested set we know
2052 it was bus_alloc_resource_any() that needs cleanup, provided
2053 res is not NULL. In case res is NULL no work in needed in
2056 if (irq
->res
!= NULL
) {
2057 rcc
= bus_release_resource(adapter
->pdev
, SYS_RES_IRQ
,
2058 irq
->vector
, irq
->res
);
2059 pci_release_msix_vector(adapter
->pdev
, irq
->vector
);
2061 if (unlikely(rcc
!= 0))
2062 device_printf(adapter
->pdev
, "dev has no parent while "
2063 "releasing res for irq: %d\n", irq
->vector
);
2064 irq
->requested
= false;
2072 ena_free_mgmnt_irq(struct ena_adapter
*adapter
)
2074 struct ena_irq
*irq
;
2077 irq
= &adapter
->irq_tbl
[ENA_MGMNT_IRQ_IDX
];
2078 if (irq
->requested
) {
2079 ena_trace(ENA_INFO
| ENA_ADMQ
, "tear down irq: %d\n",
2081 rc
= bus_teardown_intr(adapter
->pdev
, irq
->res
, irq
->cookie
);
2082 if (unlikely(rc
!= 0))
2083 device_printf(adapter
->pdev
, "failed to tear "
2084 "down irq: %d\n", irq
->vector
);
2088 if (irq
->res
!= NULL
) {
2089 ena_trace(ENA_INFO
| ENA_ADMQ
, "release resource irq: %d\n",
2091 rc
= bus_release_resource(adapter
->pdev
, SYS_RES_IRQ
,
2092 irq
->vector
, irq
->res
);
2093 pci_release_msix_vector(adapter
->pdev
, irq
->vector
);
2095 if (unlikely(rc
!= 0))
2096 device_printf(adapter
->pdev
, "dev has no parent while "
2097 "releasing res for irq: %d\n", irq
->vector
);
2102 ena_free_io_irq(struct ena_adapter
*adapter
)
2104 struct ena_irq
*irq
;
2107 for (int i
= ENA_IO_IRQ_FIRST_IDX
; i
< adapter
->msix_vecs
; i
++) {
2108 irq
= &adapter
->irq_tbl
[i
];
2109 if (irq
->requested
) {
2110 ena_trace(ENA_INFO
| ENA_IOQ
, "tear down irq: %d\n",
2112 rc
= bus_teardown_intr(adapter
->pdev
, irq
->res
,
2114 if (unlikely(rc
!= 0)) {
2115 device_printf(adapter
->pdev
, "failed to tear "
2116 "down irq: %d\n", irq
->vector
);
2121 if (irq
->res
!= NULL
) {
2122 ena_trace(ENA_INFO
| ENA_IOQ
, "release resource irq: %d\n",
2124 rc
= bus_release_resource(adapter
->pdev
, SYS_RES_IRQ
,
2125 irq
->vector
, irq
->res
);
2126 pci_release_msix_vector(adapter
->pdev
, irq
->vector
);
2128 if (unlikely(rc
!= 0)) {
2129 device_printf(adapter
->pdev
, "dev has no parent"
2130 " while releasing res for irq: %d\n",
2138 ena_free_irqs(struct ena_adapter
* adapter
)
2141 ena_free_io_irq(adapter
);
2142 ena_free_mgmnt_irq(adapter
);
2143 ena_disable_msix(adapter
);
2147 ena_disable_msix(struct ena_adapter
*adapter
)
2150 pci_release_msi(adapter
->pdev
);
2152 adapter
->msix_vecs
= 0;
2153 kfree(adapter
->msix_entries
, M_DEVBUF
);
2154 adapter
->msix_entries
= NULL
;
2158 ena_unmask_all_io_irqs(struct ena_adapter
*adapter
)
2160 struct ena_com_io_cq
* io_cq
;
2161 struct ena_eth_io_intr_reg intr_reg
;
2165 /* Unmask interrupts for all queues */
2166 for (i
= 0; i
< adapter
->num_queues
; i
++) {
2167 ena_qid
= ENA_IO_TXQ_IDX(i
);
2168 io_cq
= &adapter
->ena_dev
->io_cq_queues
[ena_qid
];
2169 ena_com_update_intr_reg(&intr_reg
, 0, 0, true);
2170 ena_com_unmask_intr(io_cq
, &intr_reg
);
2174 /* Configure the Rx forwarding */
2176 ena_rss_configure(struct ena_adapter
*adapter
)
2178 struct ena_com_dev
*ena_dev
= adapter
->ena_dev
;
2181 /* Set indirect table */
2182 rc
= ena_com_indirect_table_set(ena_dev
);
2183 if (unlikely((rc
!= 0) && (rc
!= EOPNOTSUPP
)))
2186 /* Configure hash function (if supported) */
2187 rc
= ena_com_set_hash_function(ena_dev
);
2188 if (unlikely((rc
!= 0) && (rc
!= EOPNOTSUPP
)))
2191 /* Configure hash inputs (if supported) */
2192 rc
= ena_com_set_hash_ctrl(ena_dev
);
2193 if (unlikely((rc
!= 0) && (rc
!= EOPNOTSUPP
)))
2200 ena_up_complete(struct ena_adapter
*adapter
)
2204 if (likely(adapter
->rss_support
)) {
2205 rc
= ena_rss_configure(adapter
);
2210 rc
= ena_change_mtu(adapter
->ifp
, adapter
->ifp
->if_mtu
);
2211 if (unlikely(rc
!= 0))
2214 ena_refill_all_rx_bufs(adapter
);
2215 #if 0 /* XXX swildner counters */
2216 ena_reset_counters((counter_u64_t
*)&adapter
->hw_stats
,
2217 sizeof(adapter
->hw_stats
));
2224 ena_up(struct ena_adapter
*adapter
)
2228 if (unlikely(device_is_attached(adapter
->pdev
) == 0)) {
2229 device_printf(adapter
->pdev
, "device is not attached!\n");
2233 if (unlikely(!adapter
->running
)) {
2234 device_printf(adapter
->pdev
, "device is not running!\n");
2239 device_printf(adapter
->pdev
, "device is going UP\n");
2241 /* setup interrupts for IO queues */
2242 ena_setup_io_intr(adapter
);
2243 rc
= ena_request_io_irq(adapter
);
2244 if (unlikely(rc
!= 0)) {
2245 ena_trace(ENA_ALERT
, "err_req_irq");
2249 /* allocate transmit descriptors */
2250 rc
= ena_setup_all_tx_resources(adapter
);
2251 if (unlikely(rc
!= 0)) {
2252 ena_trace(ENA_ALERT
, "err_setup_tx");
2256 /* allocate receive descriptors */
2257 rc
= ena_setup_all_rx_resources(adapter
);
2258 if (unlikely(rc
!= 0)) {
2259 ena_trace(ENA_ALERT
, "err_setup_rx");
2263 /* create IO queues for Rx & Tx */
2264 rc
= ena_create_io_queues(adapter
);
2265 if (unlikely(rc
!= 0)) {
2266 ena_trace(ENA_ALERT
,
2267 "create IO queues failed");
2271 if (unlikely(adapter
->link_status
)) {
2272 adapter
->ifp
->if_link_state
= LINK_STATE_UP
;
2273 if_link_state_change(adapter
->ifp
);
2276 rc
= ena_up_complete(adapter
);
2277 if (unlikely(rc
!= 0))
2278 goto err_up_complete
;
2280 #if 0 /* XXX swildner counters */
2281 counter_u64_add(adapter
->dev_stats
.interface_up
, 1);
2284 ena_update_hwassist(adapter
);
2286 adapter
->ifp
->if_flags
|= IFF_RUNNING
;
2287 ifq_clr_oactive(&adapter
->ifp
->if_snd
);
2289 callout_reset(&adapter
->timer_service
, hz
,
2290 ena_timer_service
, (void *)adapter
);
2294 ena_unmask_all_io_irqs(adapter
);
2300 ena_destroy_all_io_queues(adapter
);
2302 ena_free_all_rx_resources(adapter
);
2304 ena_free_all_tx_resources(adapter
);
2306 ena_free_io_irq(adapter
);
2311 #if 0 /* XXX swildner counters */
2313 ena_get_counter(if_t ifp
, ift_counter cnt
)
2315 struct ena_adapter
*adapter
;
2316 struct ena_hw_stats
*stats
;
2318 adapter
= ifp
->if_softc
;
2319 stats
= &adapter
->hw_stats
;
2322 case IFCOUNTER_IPACKETS
:
2323 return (counter_u64_fetch(stats
->rx_packets
));
2324 case IFCOUNTER_OPACKETS
:
2325 return (counter_u64_fetch(stats
->tx_packets
));
2326 case IFCOUNTER_IBYTES
:
2327 return (counter_u64_fetch(stats
->rx_bytes
));
2328 case IFCOUNTER_OBYTES
:
2329 return (counter_u64_fetch(stats
->tx_bytes
));
2330 case IFCOUNTER_IQDROPS
:
2331 return (counter_u64_fetch(stats
->rx_drops
));
2333 return (if_get_counter_default(ifp
, cnt
));
2339 ena_media_change(if_t ifp
)
2341 /* Media Change is not supported by firmware */
2346 ena_media_status(if_t ifp
, struct ifmediareq
*ifmr
)
2348 struct ena_adapter
*adapter
= ifp
->if_softc
;
2349 ena_trace(ENA_DBG
, "enter");
2351 lockmgr(&adapter
->global_lock
, LK_EXCLUSIVE
);
2353 ifmr
->ifm_status
= IFM_AVALID
;
2354 ifmr
->ifm_active
= IFM_ETHER
;
2356 if (!adapter
->link_status
) {
2357 lockmgr(&adapter
->global_lock
, LK_RELEASE
);
2358 ena_trace(ENA_INFO
, "link_status = false");
2362 ifmr
->ifm_status
|= IFM_ACTIVE
;
2363 ifmr
->ifm_active
|= IFM_10G_T
| IFM_FDX
;
2365 lockmgr(&adapter
->global_lock
, LK_RELEASE
);
2371 struct ena_adapter
*adapter
= (struct ena_adapter
*)arg
;
2374 lockmgr(&adapter
->ioctl_lock
, LK_EXCLUSIVE
);
2376 lockmgr(&adapter
->ioctl_lock
, LK_RELEASE
);
2381 ena_ioctl(if_t ifp
, u_long command
, caddr_t data
, struct ucred
*cred
)
2383 struct ena_adapter
*adapter
;
2387 adapter
= ifp
->if_softc
;
2388 ifr
= (struct ifreq
*)data
;
2391 * Acquiring lock to prevent from running up and down routines parallel.
2396 lockmgr(&adapter
->ioctl_lock
, LK_EXCLUSIVE
);
2399 ena_change_mtu(ifp
, ifr
->ifr_mtu
);
2401 rc
= ena_up(adapter
);
2402 lockmgr(&adapter
->ioctl_lock
, LK_RELEASE
);
2406 if ((ifp
->if_flags
& IFF_UP
) != 0) {
2407 if ((ifp
->if_flags
& IFF_RUNNING
) != 0) {
2408 if ((ifp
->if_flags
& (IFF_PROMISC
|
2409 IFF_ALLMULTI
)) != 0) {
2410 device_printf(adapter
->pdev
,
2411 "ioctl promisc/allmulti\n");
2414 lockmgr(&adapter
->ioctl_lock
, LK_EXCLUSIVE
);
2415 rc
= ena_up(adapter
);
2416 lockmgr(&adapter
->ioctl_lock
, LK_RELEASE
);
2419 if ((ifp
->if_flags
& IFF_RUNNING
) != 0) {
2420 lockmgr(&adapter
->ioctl_lock
, LK_EXCLUSIVE
);
2422 lockmgr(&adapter
->ioctl_lock
, LK_RELEASE
);
2433 rc
= ifmedia_ioctl(ifp
, ifr
, &adapter
->media
, command
);
2440 if (ifr
->ifr_reqcap
!= ifp
->if_capenable
) {
2441 ifp
->if_capenable
= ifr
->ifr_reqcap
;
2445 if ((reinit
!= 0) &&
2446 ((ifp
->if_flags
& IFF_RUNNING
) != 0)) {
2447 lockmgr(&adapter
->ioctl_lock
, LK_EXCLUSIVE
);
2449 rc
= ena_up(adapter
);
2450 lockmgr(&adapter
->ioctl_lock
, LK_RELEASE
);
2456 rc
= ether_ioctl(ifp
, command
, data
);
2464 ena_get_dev_offloads(struct ena_com_dev_get_features_ctx
*feat
)
2468 if ((feat
->offload
.tx
&
2469 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK
|
2470 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK
|
2471 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK
)) != 0)
2472 caps
|= IFCAP_TXCSUM
;
2474 if ((feat
->offload
.tx
&
2475 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK
|
2476 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK
)) != 0)
2477 caps
|= IFCAP_TXCSUM
;
2479 if ((feat
->offload
.tx
&
2480 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK
) != 0)
2483 if ((feat
->offload
.tx
&
2484 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK
) != 0)
2487 if ((feat
->offload
.rx_supported
&
2488 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK
|
2489 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK
)) != 0)
2490 caps
|= IFCAP_RXCSUM
;
2493 if ((feat
->offload
.rx_supported
&
2494 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK
) != 0)
2495 caps
|= IFCAP_RXCSUM_IPV6
;
2500 caps
|= IFCAP_JUMBO_MTU
;
2506 ena_update_host_info(struct ena_admin_host_info
*host_info
, if_t ifp
)
2509 host_info
->supported_network_features
[0] =
2510 (uint32_t)ifp
->if_capabilities
;
2514 ena_update_hwassist(struct ena_adapter
*adapter
)
2516 if_t ifp
= adapter
->ifp
;
2517 uint32_t feat
= adapter
->tx_offload_cap
;
2518 int cap
= ifp
->if_capenable
;
2521 ifp
->if_hwassist
= 0;
2523 if ((cap
& IFCAP_TXCSUM
) != 0) {
2525 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK
) != 0)
2528 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK
|
2529 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK
)) != 0)
2530 flags
|= CSUM_UDP
| CSUM_TCP
;
2534 if ((cap
& IFCAP_TXCSUM_IPV6
) != 0)
2535 flags
|= CSUM_IP6_UDP
| CSUM_IP6_TCP
;
2538 if ((cap
& IFCAP_TSO4
) != 0 || (cap
& IFCAP_TSO6
) != 0)
2541 ifp
->if_hwassist
|= flags
;
2545 ena_setup_ifnet(device_t pdev
, struct ena_adapter
*adapter
,
2546 struct ena_com_dev_get_features_ctx
*feat
)
2551 ifp
= adapter
->ifp
= if_alloc(IFT_ETHER
);
2552 if (unlikely(ifp
== NULL
)) {
2553 ena_trace(ENA_ALERT
, "can not allocate ifnet structure\n");
2556 if_initname(ifp
, device_get_name(pdev
), device_get_unit(pdev
));
2557 ifp
->if_softc
= adapter
;
2559 ifp
->if_flags
= IFF_BROADCAST
| IFF_SIMPLEX
| IFF_MULTICAST
;
2561 ifp
->if_init
= ena_init
;
2562 ifp
->if_start
= ena_start_xmit
;
2563 ifp
->if_ioctl
= ena_ioctl
;
2564 #if 0 /* XXX swildner counter */
2565 if_setgetcounterfn(ifp
, ena_get_counter
);
2568 ifq_set_maxlen(&ifp
->if_snd
, adapter
->tx_ring_size
);
2569 ifq_set_ready(&ifp
->if_snd
);
2570 ifp
->if_mtu
= ETHERMTU
;
2571 ifp
->if_baudrate
= 0;
2572 /* Zeroize capabilities... */
2573 ifp
->if_capabilities
= 0;
2574 ifp
->if_capenable
= 0;
2575 /* check hardware support */
2576 caps
= ena_get_dev_offloads(feat
);
2577 /* ... and set them */
2578 //if_setcapabilitiesbit(ifp, caps, 0);
2579 ((struct ifnet
*)ifp
)->if_capabilities
|= caps
;
2580 ((struct ifnet
*)ifp
)->if_capabilities
&= ~0;
2582 /* TSO parameters */
2583 //ifp->if_hw_tsomax = ENA_TSO_MAXSIZE -
2584 // (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
2585 ifp
->if_tsolen
= adapter
->max_tx_sgl_size
- 1;
2586 //ifp->if_hw_tsomaxsegsize = ENA_TSO_MAXSIZE;
2588 ifp
->if_hdrlen
= sizeof(struct ether_vlan_header
);
2589 ifp
->if_capenable
= ifp
->if_capabilities
;
2592 * Specify the media types supported by this adapter and register
2593 * callbacks to update media and link information
2595 ifmedia_init(&adapter
->media
, IFM_IMASK
,
2596 ena_media_change
, ena_media_status
);
2597 ifmedia_add(&adapter
->media
, IFM_ETHER
| IFM_AUTO
, 0, NULL
);
2598 ifmedia_set(&adapter
->media
, IFM_ETHER
| IFM_AUTO
);
2600 ether_ifattach(ifp
, adapter
->mac_addr
, NULL
);
2606 ena_down(struct ena_adapter
*adapter
)
2611 device_printf(adapter
->pdev
, "device is going DOWN\n");
2613 callout_drain(&adapter
->timer_service
);
2615 adapter
->up
= false;
2616 ifq_set_oactive(&adapter
->ifp
->if_snd
);
2617 adapter
->ifp
->if_flags
&= ~IFF_RUNNING
;
2619 ena_free_io_irq(adapter
);
2621 if (adapter
->trigger_reset
) {
2622 rc
= ena_com_dev_reset(adapter
->ena_dev
,
2623 adapter
->reset_reason
);
2624 if (unlikely(rc
!= 0))
2625 device_printf(adapter
->pdev
,
2626 "Device reset failed\n");
2629 ena_destroy_all_io_queues(adapter
);
2631 ena_free_all_tx_bufs(adapter
);
2632 ena_free_all_rx_bufs(adapter
);
2633 ena_free_all_tx_resources(adapter
);
2634 ena_free_all_rx_resources(adapter
);
2636 #if 0 /* XXX swildner counters */
2637 counter_u64_add(adapter
->dev_stats
.interface_down
, 1);
2643 ena_tx_csum(struct ena_com_tx_ctx
*ena_tx_ctx
, struct mbuf
*mbuf
)
2645 struct ena_com_tx_meta
*ena_meta
;
2646 struct ether_vlan_header
*eh
;
2656 ena_meta
= &ena_tx_ctx
->ena_meta
;
2657 mss
= mbuf
->m_pkthdr
.tso_segsz
;
2662 if ((mbuf
->m_pkthdr
.csum_flags
& CSUM_TSO
) != 0)
2665 if ((mbuf
->m_pkthdr
.csum_flags
& CSUM_OFFLOAD
) != 0)
2669 ena_tx_ctx
->meta_valid
= 0;
2673 /* Determine where frame payload starts. */
2674 eh
= mtod(mbuf
, struct ether_vlan_header
*);
2675 if (eh
->evl_encap_proto
== htons(ETHERTYPE_VLAN
)) {
2676 etype
= ntohs(eh
->evl_proto
);
2677 ehdrlen
= ETHER_HDR_LEN
+ ETHER_VLAN_ENCAP_LEN
;
2679 etype
= ntohs(eh
->evl_encap_proto
);
2680 ehdrlen
= ETHER_HDR_LEN
;
2683 ip
= (struct ip
*)(mbuf
->m_data
+ ehdrlen
);
2684 iphlen
= ip
->ip_hl
<< 2;
2685 th
= (struct tcphdr
*)((caddr_t
)ip
+ iphlen
);
2687 if ((mbuf
->m_pkthdr
.csum_flags
& CSUM_IP
) != 0) {
2688 ena_tx_ctx
->l3_csum_enable
= 1;
2690 if ((mbuf
->m_pkthdr
.csum_flags
& CSUM_TSO
) != 0) {
2691 ena_tx_ctx
->tso_enable
= 1;
2692 ena_meta
->l4_hdr_len
= (th
->th_off
);
2697 ena_tx_ctx
->l3_proto
= ENA_ETH_IO_L3_PROTO_IPV4
;
2698 if ((ip
->ip_off
& htons(IP_DF
)) != 0)
2701 case ETHERTYPE_IPV6
:
2702 ena_tx_ctx
->l3_proto
= ENA_ETH_IO_L3_PROTO_IPV6
;
2708 if (ip
->ip_p
== IPPROTO_TCP
) {
2709 ena_tx_ctx
->l4_proto
= ENA_ETH_IO_L4_PROTO_TCP
;
2710 if ((mbuf
->m_pkthdr
.csum_flags
&
2712 ena_tx_ctx
->l4_csum_enable
= 1;
2714 ena_tx_ctx
->l4_csum_enable
= 0;
2715 } else if (ip
->ip_p
== IPPROTO_UDP
) {
2716 ena_tx_ctx
->l4_proto
= ENA_ETH_IO_L4_PROTO_UDP
;
2717 if ((mbuf
->m_pkthdr
.csum_flags
&
2719 ena_tx_ctx
->l4_csum_enable
= 1;
2721 ena_tx_ctx
->l4_csum_enable
= 0;
2723 ena_tx_ctx
->l4_proto
= ENA_ETH_IO_L4_PROTO_UNKNOWN
;
2724 ena_tx_ctx
->l4_csum_enable
= 0;
2727 ena_meta
->mss
= mss
;
2728 ena_meta
->l3_hdr_len
= iphlen
;
2729 ena_meta
->l3_hdr_offset
= ehdrlen
;
2730 ena_tx_ctx
->meta_valid
= 1;
2734 ena_check_and_collapse_mbuf(struct ena_ring
*tx_ring
, struct mbuf
**mbuf
)
2736 struct ena_adapter
*adapter
;
2737 struct mbuf
*collapsed_mbuf
;
2740 adapter
= tx_ring
->adapter
;
2741 num_frags
= ena_mbuf_count(*mbuf
);
2743 /* One segment must be reserved for configuration descriptor. */
2744 if (num_frags
< adapter
->max_tx_sgl_size
)
2746 #if 0 /* XXX swildner counters */
2747 counter_u64_add(tx_ring
->tx_stats
.collapse
, 1);
2750 collapsed_mbuf
= m_defrag(*mbuf
, M_NOWAIT
);
2751 if (unlikely(collapsed_mbuf
== NULL
)) {
2752 IFNET_STAT_INC(tx_ring
->adapter
->ifp
, oerrors
, 1);
2753 #if 0 /* XXX swildner counters */
2754 counter_u64_add(tx_ring
->tx_stats
.collapse_err
, 1);
2759 /* If mbuf was collapsed succesfully, original mbuf is released. */
2760 *mbuf
= collapsed_mbuf
;
2766 ena_xmit_mbuf(struct ena_ring
*tx_ring
, struct mbuf
**mbuf
)
2768 struct ena_adapter
*adapter
;
2769 struct ena_tx_buffer
*tx_info
;
2770 struct ena_com_tx_ctx ena_tx_ctx
;
2771 struct ena_com_dev
*ena_dev
;
2772 struct ena_com_buf
*ena_buf
;
2773 struct ena_com_io_sq
* io_sq
;
2774 bus_dma_segment_t segs
[ENA_BUS_DMA_SEGS
];
2776 uint16_t next_to_use
;
2780 uint32_t len
, nsegs
, header_len
;
2784 ena_qid
= ENA_IO_TXQ_IDX(tx_ring
->que
->id
);
2785 adapter
= tx_ring
->que
->adapter
;
2786 ena_dev
= adapter
->ena_dev
;
2787 io_sq
= &ena_dev
->io_sq_queues
[ena_qid
];
2789 //tx_ring is just used to grab the adapter
2790 rc
= ena_check_and_collapse_mbuf(tx_ring
, mbuf
);
2791 if (unlikely(rc
!= 0)) {
2792 ena_trace(ENA_WARNING
,
2793 "Failed to collapse mbuf! err: %d", rc
);
2797 next_to_use
= tx_ring
->next_to_use
;
2798 req_id
= tx_ring
->free_tx_ids
[next_to_use
];
2799 tx_info
= &tx_ring
->tx_buffer_info
[req_id
];
2801 tx_info
->mbuf
= *mbuf
;
2802 tx_info
->num_of_bufs
= 0;
2804 ena_buf
= tx_info
->bufs
;
2805 len
= (*mbuf
)->m_len
;
2807 ena_trace(ENA_DBG
| ENA_TXPTH
, "Tx: %d bytes", (*mbuf
)->m_pkthdr
.len
);
2810 header_len
= min_t(uint32_t, len
, tx_ring
->tx_max_header_size
);
2813 rc
= bus_dmamap_load_mbuf_segment(adapter
->tx_buf_tag
, tx_info
->map
,
2814 *mbuf
, segs
, adapter
->max_tx_sgl_size
, &nsegs
, BUS_DMA_NOWAIT
);
2816 if (unlikely((rc
!= 0) || (nsegs
== 0))) {
2817 ena_trace(ENA_WARNING
,
2818 "dmamap load failed! err: %d nsegs: %d", rc
, nsegs
);
2819 IFNET_STAT_INC(tx_ring
->adapter
->ifp
, oerrors
, 1);
2820 #if 0 /* XXX swildner counters */
2821 counter_u64_add(tx_ring
->tx_stats
.dma_mapping_err
, 1);
2823 tx_info
->mbuf
= NULL
;
2825 return (ENA_COM_NO_MEM
);
2827 return (ENA_COM_INVAL
);
2830 for (i
= 0; i
< nsegs
; i
++) {
2831 ena_buf
->len
= segs
[i
].ds_len
;
2832 ena_buf
->paddr
= segs
[i
].ds_addr
;
2835 tx_info
->num_of_bufs
= nsegs
;
2837 memset(&ena_tx_ctx
, 0x0, sizeof(struct ena_com_tx_ctx
));
2838 ena_tx_ctx
.ena_bufs
= tx_info
->bufs
;
2839 ena_tx_ctx
.push_header
= push_hdr
;
2840 ena_tx_ctx
.num_bufs
= tx_info
->num_of_bufs
;
2841 ena_tx_ctx
.req_id
= req_id
;
2842 ena_tx_ctx
.header_len
= header_len
;
2844 /* Set flags and meta data */
2845 ena_tx_csum(&ena_tx_ctx
, *mbuf
);
2846 /* Prepare the packet's descriptors and send them to device */
2847 rc
= ena_com_prepare_tx(io_sq
, &ena_tx_ctx
, &nb_hw_desc
);
2848 if (unlikely(rc
!= 0)) {
2849 device_printf(adapter
->pdev
, "failed to prepare tx bufs\n");
2850 IFNET_STAT_INC(tx_ring
->adapter
->ifp
, oerrors
, 1);
2851 #if 0 /* XXX swildner counters */
2852 counter_u64_add(tx_ring
->tx_stats
.prepare_ctx_err
, 1);
2857 IFNET_STAT_INC(tx_ring
->adapter
->ifp
, opackets
, 1);
2858 #if 0 /* XXX swildner counters */
2860 counter_u64_add_protected(tx_ring
->tx_stats
.cnt
, 1);
2861 counter_u64_add_protected(tx_ring
->tx_stats
.bytes
,
2862 (*mbuf
)->m_pkthdr
.len
);
2864 counter_u64_add_protected(adapter
->hw_stats
.tx_packets
, 1);
2865 counter_u64_add_protected(adapter
->hw_stats
.tx_bytes
,
2866 (*mbuf
)->m_pkthdr
.len
);
2870 tx_info
->tx_descs
= nb_hw_desc
;
2871 getmicrouptime(&tx_info
->timestamp
);
2872 tx_info
->print_once
= true;
2874 tx_ring
->next_to_use
= ENA_TX_RING_IDX_NEXT(next_to_use
,
2875 tx_ring
->ring_size
);
2877 bus_dmamap_sync(adapter
->tx_buf_tag
, tx_info
->map
,
2878 BUS_DMASYNC_PREWRITE
);
2883 tx_info
->mbuf
= NULL
;
2884 bus_dmamap_unload(adapter
->tx_buf_tag
, tx_info
->map
);
2890 ena_start_xmit(struct ifnet
*ifp
, struct ifaltq_subque
*ifsq
)
2893 * TODO: Might need to initialize an ena_ring with the
2894 * ifaltq_subque in it
2896 struct ena_adapter
*adapter
= ifp
->if_softc
;
2897 struct ena_com_io_sq
*io_sq
;
2898 struct ena_ring
*tx_ring
;
2903 if (unlikely((adapter
->ifp
->if_flags
& IFF_RUNNING
) == 0) ||
2904 ifsq_is_oactive(ifsq
)) {
2908 /* Check is link_active and some other shit. If it is, purge. */
2911 if (unlikely(!adapter
->link_status
))
2918 while (!ifsq_is_empty(ifsq
)) {
2919 struct mbuf
*m_head
;
2922 //Grab head from mbuf list
2923 m_head
= ifsq_dequeue(ifsq
);
2927 //pick the associated tx_ring based on hash
2928 i
= m_head
->m_pkthdr
.hash
% adapter
->num_queues
;
2930 tx_ring
= &adapter
->tx_ring
[i
];
2931 ENA_RING_MTX_LOCK(tx_ring
);
2932 ena_qid
= ENA_IO_TXQ_IDX(tx_ring
->que
->id
);
2933 io_sq
= &adapter
->ena_dev
->io_sq_queues
[ena_qid
];
2935 if (unlikely(!ena_com_sq_have_enough_space(io_sq
, ENA_TX_CLEANUP_THRESHOLD
)))
2936 ena_tx_cleanup(tx_ring
);
2938 if (unlikely((ret
= ena_xmit_mbuf(tx_ring
, &m_head
)) != 0)) {
2939 if (ret
== ENA_COM_NO_MEM
) {
2940 /* XXX put mbuf back on queue */
2941 } else if (ret
== ENA_COM_NO_SPACE
) {
2942 /* XXX put mbuf back on queue */
2945 /* XXX advance mbuf queue aka move it forward? */
2947 ENA_RING_MTX_UNLOCK(tx_ring
);
2951 //advance mbuf queue, might already be handled by dequeue
2954 // dillon - wtf is this doing here?
2955 // NOT SURE WHAT TO DO WITH THIS CODE
2956 if (unlikely((adapter
->ifp
->if_flags
& IFF_RUNNING
) == 0))
2957 return; // break here, not return. tx_ring locked
2962 ENA_RING_MTX_UNLOCK(tx_ring
);
2963 BPF_MTAP(adapter
->ifp
, m_head
);
2965 if (unlikely(acum_pkts
== DB_THRESHOLD
)) {
2968 /* Trigger the dma engine */
2969 ena_com_write_sq_doorbell(io_sq
);
2970 #if 0 /* XXX swildner counters */
2971 counter_u64_add(tx_ring
->tx_stats
.doorbells
, 1);
2977 if (likely(acum_pkts
!= 0)) {
2979 /* Trigger the dma engine */
2980 ena_com_write_sq_doorbell(io_sq
);
2981 #if 0 /* XXX swildner counters */
2982 counter_u64_add(tx_ring
->tx_stats
.doorbells
, 1);
2987 !ena_com_sq_have_enough_space(io_sq
, ENA_TX_CLEANUP_THRESHOLD
)) {
2988 ENA_RING_MTX_LOCK(tx_ring
);
2989 ena_tx_cleanup(tx_ring
);
2990 ENA_RING_MTX_UNLOCK(tx_ring
);
2995 ena_calc_io_queue_num(struct ena_adapter
*adapter
,
2996 struct ena_com_dev_get_features_ctx
*get_feat_ctx
)
2998 int io_sq_num
, io_cq_num
, io_queue_num
;
3000 io_sq_num
= get_feat_ctx
->max_queues
.max_sq_num
;
3001 io_cq_num
= get_feat_ctx
->max_queues
.max_cq_num
;
3003 io_queue_num
= min_t(int, ncpus
, ENA_MAX_NUM_IO_QUEUES
);
3004 io_queue_num
= min_t(int, io_queue_num
, io_sq_num
);
3005 io_queue_num
= min_t(int, io_queue_num
, io_cq_num
);
3006 /* 1 IRQ for for mgmnt and 1 IRQ for each TX/RX pair */
3007 io_queue_num
= min_t(int, io_queue_num
,
3008 pci_msix_count(adapter
->pdev
) - 1);
3010 io_queue_num
= min_t(int, io_queue_num
, rss_getnumbuckets());
3013 return (io_queue_num
);
3017 ena_calc_queue_size(struct ena_adapter
*adapter
, uint16_t *max_tx_sgl_size
,
3018 uint16_t *max_rx_sgl_size
, struct ena_com_dev_get_features_ctx
*feat
)
3020 uint32_t queue_size
= ENA_DEFAULT_RING_SIZE
;
3024 queue_size
= min_t(uint32_t, queue_size
,
3025 feat
->max_queues
.max_cq_depth
);
3026 queue_size
= min_t(uint32_t, queue_size
,
3027 feat
->max_queues
.max_sq_depth
);
3029 /* round down to the nearest power of 2 */
3032 if (powerof2(queue_size
) != 0)
3035 q
= rounddown2(queue_size
, v
);
3042 if (unlikely(queue_size
== 0)) {
3043 device_printf(adapter
->pdev
, "Invalid queue size\n");
3044 return (ENA_COM_FAULT
);
3047 *max_tx_sgl_size
= min_t(uint16_t, ENA_PKT_MAX_BUFS
,
3048 feat
->max_queues
.max_packet_tx_descs
);
3049 *max_rx_sgl_size
= min_t(uint16_t, ENA_PKT_MAX_BUFS
,
3050 feat
->max_queues
.max_packet_rx_descs
);
3052 return (queue_size
);
3056 ena_rss_init_default(struct ena_adapter
*adapter
)
3058 struct ena_com_dev
*ena_dev
= adapter
->ena_dev
;
3059 device_t dev
= adapter
->pdev
;
3062 rc
= ena_com_rss_init(ena_dev
, ENA_RX_RSS_TABLE_LOG_SIZE
);
3063 if (unlikely(rc
!= 0)) {
3064 device_printf(dev
, "Cannot init indirect table\n");
3068 for (i
= 0; i
< ENA_RX_RSS_TABLE_SIZE
; i
++) {
3070 qid
= rss_get_indirection_to_bucket(i
);
3071 qid
= qid
% adapter
->num_queues
;
3073 qid
= i
% adapter
->num_queues
;
3075 rc
= ena_com_indirect_table_fill_entry(ena_dev
, i
,
3076 ENA_IO_RXQ_IDX(qid
));
3077 if (unlikely((rc
!= 0) && (rc
!= EOPNOTSUPP
))) {
3078 device_printf(dev
, "Cannot fill indirect table\n");
3079 goto err_rss_destroy
;
3083 rc
= ena_com_fill_hash_function(ena_dev
, ENA_ADMIN_CRC32
, NULL
,
3084 ENA_HASH_KEY_SIZE
, 0xFFFFFFFF);
3085 if (unlikely((rc
!= 0) && (rc
!= EOPNOTSUPP
))) {
3086 device_printf(dev
, "Cannot fill hash function\n");
3087 goto err_rss_destroy
;
3090 rc
= ena_com_set_default_hash_ctrl(ena_dev
);
3091 if (unlikely((rc
!= 0) && (rc
!= EOPNOTSUPP
))) {
3092 device_printf(dev
, "Cannot fill hash control\n");
3093 goto err_rss_destroy
;
3099 ena_com_rss_destroy(ena_dev
);
3104 ena_rss_init_default_deferred(void *arg
)
3106 struct ena_adapter
*adapter
;
3111 dc
= devclass_find("ena");
3112 if (unlikely(dc
== NULL
)) {
3113 ena_trace(ENA_ALERT
, "No devclass ena\n");
3117 max
= devclass_get_maxunit(dc
);
3118 while (max
-- >= 0) {
3119 adapter
= devclass_get_softc(dc
, max
);
3120 if (adapter
!= NULL
) {
3121 rc
= ena_rss_init_default(adapter
);
3122 adapter
->rss_support
= true;
3123 if (unlikely(rc
!= 0)) {
3124 device_printf(adapter
->pdev
,
3125 "WARNING: RSS was not properly initialized,"
3126 " it will affect bandwidth\n");
3127 adapter
->rss_support
= false;
3132 SYSINIT(ena_rss_init
, SI_SUB_KICK_SCHEDULER
, SI_ORDER_SECOND
, ena_rss_init_default_deferred
, NULL
);
3135 ena_config_host_info(struct ena_com_dev
*ena_dev
)
3137 struct ena_admin_host_info
*host_info
;
3140 /* Allocate only the host info */
3141 rc
= ena_com_allocate_host_info(ena_dev
);
3142 if (unlikely(rc
!= 0)) {
3143 ena_trace(ENA_ALERT
, "Cannot allocate host info\n");
3147 host_info
= ena_dev
->host_attr
.host_info
;
3149 host_info
->os_type
= ENA_ADMIN_OS_FREEBSD
;
3150 host_info
->kernel_ver
= osreldate
;
3152 ksprintf(host_info
->kernel_ver_str
, "%d", osreldate
);
3153 host_info
->os_dist
= 0;
3154 strncpy(host_info
->os_dist_str
, osrelease
,
3155 sizeof(host_info
->os_dist_str
) - 1);
3157 host_info
->driver_version
=
3158 (DRV_MODULE_VER_MAJOR
) |
3159 (DRV_MODULE_VER_MINOR
<< ENA_ADMIN_HOST_INFO_MINOR_SHIFT
) |
3160 (DRV_MODULE_VER_SUBMINOR
<< ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT
);
3162 rc
= ena_com_set_host_attributes(ena_dev
);
3163 if (unlikely(rc
!= 0)) {
3164 if (rc
== EOPNOTSUPP
)
3165 ena_trace(ENA_WARNING
, "Cannot set host attributes\n");
3167 ena_trace(ENA_ALERT
, "Cannot set host attributes\n");
3175 ena_com_delete_host_info(ena_dev
);
3179 ena_device_init(struct ena_adapter
*adapter
, device_t pdev
,
3180 struct ena_com_dev_get_features_ctx
*get_feat_ctx
, int *wd_active
)
3182 struct ena_com_dev
* ena_dev
= adapter
->ena_dev
;
3183 bool readless_supported
;
3184 uint32_t aenq_groups
;
3188 rc
= ena_com_mmio_reg_read_request_init(ena_dev
);
3189 if (unlikely(rc
!= 0)) {
3190 device_printf(pdev
, "failed to init mmio read less\n");
3195 * The PCIe configuration space revision id indicate if mmio reg
3198 readless_supported
= !(pci_get_revid(pdev
) & ENA_MMIO_DISABLE_REG_READ
);
3199 ena_com_set_mmio_read_mode(ena_dev
, readless_supported
);
3201 rc
= ena_com_dev_reset(ena_dev
, ENA_REGS_RESET_NORMAL
);
3202 if (unlikely(rc
!= 0)) {
3203 device_printf(pdev
, "Can not reset device\n");
3204 goto err_mmio_read_less
;
3207 rc
= ena_com_validate_version(ena_dev
);
3208 if (unlikely(rc
!= 0)) {
3209 device_printf(pdev
, "device version is too low\n");
3210 goto err_mmio_read_less
;
3213 dma_width
= ena_com_get_dma_width(ena_dev
);
3214 if (unlikely(dma_width
< 0)) {
3215 device_printf(pdev
, "Invalid dma width value %d", dma_width
);
3217 goto err_mmio_read_less
;
3219 adapter
->dma_width
= dma_width
;
3221 /* ENA admin level init */
3222 rc
= ena_com_admin_init(ena_dev
, &aenq_handlers
, true);
3223 if (unlikely(rc
!= 0)) {
3225 "Can not initialize ena admin queue with device\n");
3226 goto err_mmio_read_less
;
3230 * To enable the msix interrupts the driver needs to know the number
3231 * of queues. So the driver uses polling mode to retrieve this
3234 ena_com_set_admin_polling_mode(ena_dev
, true);
3236 ena_config_host_info(ena_dev
);
3238 /* Get Device Attributes */
3239 rc
= ena_com_get_dev_attr_feat(ena_dev
, get_feat_ctx
);
3240 if (unlikely(rc
!= 0)) {
3242 "Cannot get attribute for ena device rc: %d\n", rc
);
3243 goto err_admin_init
;
3246 aenq_groups
= BIT(ENA_ADMIN_LINK_CHANGE
) | BIT(ENA_ADMIN_KEEP_ALIVE
);
3248 aenq_groups
&= get_feat_ctx
->aenq
.supported_groups
;
3249 rc
= ena_com_set_aenq_config(ena_dev
, aenq_groups
);
3250 if (unlikely(rc
!= 0)) {
3251 device_printf(pdev
, "Cannot configure aenq groups rc: %d\n", rc
);
3252 goto err_admin_init
;
3255 *wd_active
= !!(aenq_groups
& BIT(ENA_ADMIN_KEEP_ALIVE
));
3260 ena_com_delete_host_info(ena_dev
);
3261 ena_com_admin_destroy(ena_dev
);
3263 ena_com_mmio_reg_read_request_destroy(ena_dev
);
3268 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter
*adapter
,
3271 struct ena_com_dev
*ena_dev
= adapter
->ena_dev
;
3274 rc
= ena_enable_msix(adapter
);
3275 if (unlikely(rc
!= 0)) {
3276 device_printf(adapter
->pdev
, "Error with MSI-X enablement\n");
3280 ena_setup_mgmnt_intr(adapter
);
3282 rc
= ena_request_mgmnt_irq(adapter
);
3283 if (unlikely(rc
!= 0)) {
3284 device_printf(adapter
->pdev
, "Cannot setup mgmnt queue intr\n");
3285 goto err_disable_msix
;
3288 pci_enable_msix(adapter
->pdev
);
3290 ena_com_set_admin_polling_mode(ena_dev
, false);
3292 ena_com_admin_aenq_enable(ena_dev
);
3297 ena_disable_msix(adapter
);
3302 /* Function called on ENA_ADMIN_KEEP_ALIVE event */
3303 static void ena_keep_alive_wd(void *adapter_data
,
3304 struct ena_admin_aenq_entry
*aenq_e
)
3306 struct ena_adapter
*adapter
= (struct ena_adapter
*)adapter_data
;
3307 struct ena_admin_aenq_keep_alive_desc
*desc
;
3308 struct timeval time
;
3311 desc
= (struct ena_admin_aenq_keep_alive_desc
*)aenq_e
;
3313 rx_drops
= ((uint64_t)desc
->rx_drops_high
<< 32) | desc
->rx_drops_low
;
3314 IFNET_STAT_INC(adapter
->ifp
, iqdrops
, 1);
3315 #if 0 /* XXX swildner counters */
3316 counter_u64_zero(adapter
->hw_stats
.rx_drops
);
3317 counter_u64_add(adapter
->hw_stats
.rx_drops
, rx_drops
);
3320 getmicrouptime(&time
);
3321 atomic_store_rel_64(&adapter
->keep_alive_timestamp
.tv_sec
, time
.tv_sec
);
3324 /* Check for keep alive expiration */
3325 static void check_for_missing_keep_alive(struct ena_adapter
*adapter
)
3327 struct timeval timestamp
, time
;
3329 if (adapter
->wd_active
== 0)
3332 if (likely(adapter
->keep_alive_timeout
== 0))
3335 timestamp
.tv_sec
= atomic_load_acq_64(&adapter
->keep_alive_timestamp
.tv_sec
);
3336 getmicrouptime(&time
);
3337 timevalsub(&time
, ×tamp
);
3338 if (unlikely(time
.tv_sec
> adapter
->keep_alive_timeout
)) {
3339 device_printf(adapter
->pdev
,
3340 "Keep alive watchdog timeout.\n");
3341 #if 0 /* XXX swildner counters */
3342 counter_u64_add(adapter
->dev_stats
.wd_expired
, 1);
3344 adapter
->reset_reason
= ENA_REGS_RESET_KEEP_ALIVE_TO
;
3345 adapter
->trigger_reset
= true;
3349 /* Check if admin queue is enabled */
3350 static void check_for_admin_com_state(struct ena_adapter
*adapter
)
3352 if (unlikely(ena_com_get_admin_running_state(adapter
->ena_dev
) ==
3354 device_printf(adapter
->pdev
,
3355 "ENA admin queue is not in running state!\n");
3356 #if 0 /* XXX swildner counters */
3357 counter_u64_add(adapter
->dev_stats
.admin_q_pause
, 1);
3359 adapter
->reset_reason
= ENA_REGS_RESET_ADMIN_TO
;
3360 adapter
->trigger_reset
= true;
3365 check_missing_comp_in_queue(struct ena_adapter
*adapter
,
3366 struct ena_ring
*tx_ring
)
3368 struct timeval curtime
, time
;
3369 struct ena_tx_buffer
*tx_buf
;
3370 uint32_t missed_tx
= 0;
3373 getmicrouptime(&curtime
);
3375 for (i
= 0; i
< tx_ring
->ring_size
; i
++) {
3376 tx_buf
= &tx_ring
->tx_buffer_info
[i
];
3378 if (timevalisset(&tx_buf
->timestamp
) == 0)
3382 timevalsub(&time
, &tx_buf
->timestamp
);
3384 /* Check again if packet is still waiting */
3385 //WATCH: Might not be exactly comparable
3386 if (unlikely(time
.tv_sec
> adapter
->missing_tx_timeout
)) {
3388 if (!tx_buf
->print_once
)
3389 ena_trace(ENA_WARNING
, "Found a Tx that wasn't "
3390 "completed on time, qid %d, index %d.\n",
3393 tx_buf
->print_once
= true;
3395 #if 0 /* XXX swildner counters */
3396 counter_u64_add(tx_ring
->tx_stats
.missing_tx_comp
, 1);
3399 if (unlikely(missed_tx
>
3400 adapter
->missing_tx_threshold
)) {
3401 device_printf(adapter
->pdev
,
3402 "The number of lost tx completion "
3403 "is above the threshold (%d > %d). "
3404 "Reset the device\n",
3405 missed_tx
, adapter
->missing_tx_threshold
);
3406 adapter
->reset_reason
=
3407 ENA_REGS_RESET_MISS_TX_CMPL
;
3408 adapter
->trigger_reset
= true;
3418 * Check for TX which were not completed on time.
3419 * Timeout is defined by "missing_tx_timeout".
3420 * Reset will be performed if number of incompleted
3421 * transactions exceeds "missing_tx_threshold".
3424 check_for_missing_tx_completions(struct ena_adapter
*adapter
)
3426 struct ena_ring
*tx_ring
;
3429 /* Make sure the driver doesn't turn the device in other process */
3435 if (adapter
->trigger_reset
)
3438 if (adapter
->missing_tx_timeout
== 0)
3441 budget
= adapter
->missing_tx_max_queues
;
3443 for (i
= adapter
->next_monitored_tx_qid
; i
< adapter
->num_queues
; i
++) {
3444 tx_ring
= &adapter
->tx_ring
[i
];
3446 rc
= check_missing_comp_in_queue(adapter
, tx_ring
);
3447 if (unlikely(rc
!= 0))
3457 adapter
->next_monitored_tx_qid
= i
% adapter
->num_queues
;
3460 /* trigger deferred rx cleanup after 2 consecutive detections */
3461 #define EMPTY_RX_REFILL 2
3462 /* For the rare case where the device runs out of Rx descriptors and the
3463 * msix handler failed to refill new Rx descriptors (due to a lack of memory
3465 * This case will lead to a deadlock:
3466 * The device won't send interrupts since all the new Rx packets will be dropped
3467 * The msix handler won't allocate new Rx descriptors so the device won't be
3468 * able to send new packets.
3470 * When such a situation is detected - execute rx cleanup task in another thread
3473 check_for_empty_rx_ring(struct ena_adapter
*adapter
)
3475 struct ena_ring
*rx_ring
;
3476 int i
, refill_required
;
3481 if (adapter
->trigger_reset
)
3484 for (i
= 0; i
< adapter
->num_queues
; i
++) {
3485 rx_ring
= &adapter
->rx_ring
[i
];
3487 refill_required
= ena_com_free_desc(rx_ring
->ena_com_io_sq
);
3488 if (unlikely(refill_required
== (rx_ring
->ring_size
- 1))) {
3489 rx_ring
->empty_rx_queue
++;
3491 if (rx_ring
->empty_rx_queue
>= EMPTY_RX_REFILL
) {
3492 #if 0 /* XXX swildner counters */
3493 counter_u64_add(rx_ring
->rx_stats
.empty_rx_ring
,
3497 device_printf(adapter
->pdev
,
3498 "trigger refill for ring %d\n", i
);
3500 taskqueue_enqueue(rx_ring
->cmpl_tq
,
3501 &rx_ring
->cmpl_task
);
3502 rx_ring
->empty_rx_queue
= 0;
3505 rx_ring
->empty_rx_queue
= 0;
3511 ena_timer_service(void *data
)
3513 struct ena_adapter
*adapter
= (struct ena_adapter
*)data
;
3514 struct ena_admin_host_info
*host_info
=
3515 adapter
->ena_dev
->host_attr
.host_info
;
3517 check_for_missing_keep_alive(adapter
);
3519 check_for_admin_com_state(adapter
);
3521 check_for_missing_tx_completions(adapter
);
3523 check_for_empty_rx_ring(adapter
);
3525 if (host_info
!= NULL
)
3526 ena_update_host_info(host_info
, adapter
->ifp
);
3528 if (unlikely(adapter
->trigger_reset
)) {
3529 device_printf(adapter
->pdev
, "Trigger reset is on\n");
3530 taskqueue_enqueue(adapter
->reset_tq
, &adapter
->reset_task
);
3535 * Schedule another timeout one second from now.
3537 /* XXX swildner callout_schedule_sbt(&adapter->timer_service, SBT_1S, SBT_1S, 0); */
3538 callout_reset(&adapter
->timer_service
, hz
, ena_timer_service
,
3543 ena_reset_task(void *arg
, int pending
)
3545 struct ena_com_dev_get_features_ctx get_feat_ctx
;
3546 struct ena_adapter
*adapter
= (struct ena_adapter
*)arg
;
3547 struct ena_com_dev
*ena_dev
= adapter
->ena_dev
;
3551 if (unlikely(!adapter
->trigger_reset
)) {
3552 device_printf(adapter
->pdev
,
3553 "device reset scheduled but trigger_reset is off\n");
3557 lockmgr(&adapter
->ioctl_lock
, LK_EXCLUSIVE
);
3559 callout_drain(&adapter
->timer_service
);
3561 dev_up
= adapter
->up
;
3563 ena_com_set_admin_running_state(ena_dev
, false);
3565 ena_free_mgmnt_irq(adapter
);
3566 ena_disable_msix(adapter
);
3567 ena_com_abort_admin_commands(ena_dev
);
3568 ena_com_wait_for_abort_completion(ena_dev
);
3569 ena_com_admin_destroy(ena_dev
);
3570 ena_com_mmio_reg_read_request_destroy(ena_dev
);
3572 adapter
->reset_reason
= ENA_REGS_RESET_NORMAL
;
3573 adapter
->trigger_reset
= false;
3575 /* Finished destroy part. Restart the device */
3576 rc
= ena_device_init(adapter
, adapter
->pdev
, &get_feat_ctx
,
3577 &adapter
->wd_active
);
3578 if (unlikely(rc
!= 0)) {
3579 device_printf(adapter
->pdev
,
3580 "ENA device init failed! (err: %d)\n", rc
);
3584 rc
= ena_enable_msix_and_set_admin_interrupts(adapter
,
3585 adapter
->num_queues
);
3586 if (unlikely(rc
!= 0)) {
3587 device_printf(adapter
->pdev
, "Enable MSI-X failed\n");
3591 /* If the interface was up before the reset bring it up */
3593 rc
= ena_up(adapter
);
3594 if (unlikely(rc
!= 0)) {
3595 device_printf(adapter
->pdev
,
3596 "Failed to create I/O queues\n");
3601 callout_reset(&adapter
->timer_service
, hz
,
3602 ena_timer_service
, (void *)adapter
);
3604 lockmgr(&adapter
->ioctl_lock
, LK_RELEASE
);
3609 ena_free_mgmnt_irq(adapter
);
3610 ena_disable_msix(adapter
);
3612 ena_com_admin_destroy(ena_dev
);
3614 device_printf(adapter
->pdev
, "ENA reset failed!\n");
3615 adapter
->running
= false;
3616 lockmgr(&adapter
->ioctl_lock
, LK_RELEASE
);
3620 * ena_attach - Device Initialization Routine
3621 * @pdev: device information struct
3623 * Returns 0 on success, otherwise on failure.
3625 * ena_attach initializes an adapter identified by a device structure.
3626 * The OS initialization, configuring of the adapter private structure,
3627 * and a hardware reset occur.
3630 ena_attach(device_t pdev
)
3632 struct ena_com_dev_get_features_ctx get_feat_ctx
;
3633 static int version_printed
;
3634 struct ena_adapter
*adapter
;
3635 struct ena_com_dev
*ena_dev
= NULL
;
3636 uint16_t tx_sgl_size
= 0;
3637 uint16_t rx_sgl_size
= 0;
3641 adapter
= device_get_softc(pdev
);
3642 adapter
->pdev
= pdev
;
3644 lockinit(&adapter
->global_lock
, "ENA global mtx", 0, LK_CANRECURSE
);
3645 lockinit(&adapter
->ioctl_lock
, "ENA ioctl sx", 0, LK_CANRECURSE
);
3647 /* Set up the timer service */
3648 callout_init_lk(&adapter
->timer_service
, &adapter
->global_lock
);
3649 adapter
->keep_alive_timeout
= DEFAULT_KEEP_ALIVE_TO
;
3650 adapter
->missing_tx_timeout
= DEFAULT_TX_CMP_TO
;
3651 adapter
->missing_tx_max_queues
= DEFAULT_TX_MONITORED_QUEUES
;
3652 adapter
->missing_tx_threshold
= DEFAULT_TX_CMP_THRESHOLD
;
3654 if (version_printed
++ == 0)
3655 device_printf(pdev
, "%s\n", ena_version
);
3657 rc
= ena_allocate_pci_resources(adapter
);
3658 if (unlikely(rc
!= 0)) {
3659 device_printf(pdev
, "PCI resource allocation failed!\n");
3660 ena_free_pci_resources(adapter
);
3664 /* Allocate memory for ena_dev structure */
3665 ena_dev
= kmalloc(sizeof(struct ena_com_dev
), M_DEVBUF
,
3668 adapter
->ena_dev
= ena_dev
;
3669 ena_dev
->dmadev
= pdev
;
3670 ena_dev
->bus
= kmalloc(sizeof(struct ena_bus
), M_DEVBUF
,
3673 /* Store register resources */
3674 ((struct ena_bus
*)(ena_dev
->bus
))->reg_bar_t
=
3675 rman_get_bustag(adapter
->registers
);
3676 ((struct ena_bus
*)(ena_dev
->bus
))->reg_bar_h
=
3677 rman_get_bushandle(adapter
->registers
);
3679 if (unlikely(((struct ena_bus
*)(ena_dev
->bus
))->reg_bar_h
== 0)) {
3680 device_printf(pdev
, "failed to pmap registers bar\n");
3685 ena_dev
->tx_mem_queue_type
= ENA_ADMIN_PLACEMENT_POLICY_HOST
;
3687 /* Device initialization */
3688 rc
= ena_device_init(adapter
, pdev
, &get_feat_ctx
, &adapter
->wd_active
);
3689 if (unlikely(rc
!= 0)) {
3690 device_printf(pdev
, "ENA device init failed! (err: %d)\n", rc
);
3695 getmicrouptime(&adapter
->keep_alive_timestamp
);
3697 adapter
->tx_offload_cap
= get_feat_ctx
.offload
.tx
;
3699 /* Set for sure that interface is not up */
3700 adapter
->up
= false;
3702 memcpy(adapter
->mac_addr
, get_feat_ctx
.dev_attr
.mac_addr
,
3705 /* calculate IO queue number to create */
3706 io_queue_num
= ena_calc_io_queue_num(adapter
, &get_feat_ctx
);
3708 ENA_ASSERT(io_queue_num
> 0, "Invalid queue number: %d\n",
3710 adapter
->num_queues
= io_queue_num
;
3712 adapter
->max_mtu
= get_feat_ctx
.dev_attr
.max_mtu
;
3714 /* calculatre ring sizes */
3715 queue_size
= ena_calc_queue_size(adapter
,&tx_sgl_size
,
3716 &rx_sgl_size
, &get_feat_ctx
);
3717 if (unlikely((queue_size
<= 0) || (io_queue_num
<= 0))) {
3722 adapter
->reset_reason
= ENA_REGS_RESET_NORMAL
;
3724 adapter
->tx_ring_size
= queue_size
;
3725 adapter
->rx_ring_size
= queue_size
;
3727 adapter
->max_tx_sgl_size
= tx_sgl_size
;
3728 adapter
->max_rx_sgl_size
= rx_sgl_size
;
3730 /* set up dma tags for rx and tx buffers */
3731 rc
= ena_setup_tx_dma_tag(adapter
);
3732 if (unlikely(rc
!= 0)) {
3733 device_printf(pdev
, "Failed to create TX DMA tag\n");
3737 rc
= ena_setup_rx_dma_tag(adapter
);
3738 if (unlikely(rc
!= 0)) {
3739 device_printf(pdev
, "Failed to create RX DMA tag\n");
3740 goto err_tx_tag_free
;
3743 /* initialize rings basic information */
3744 device_printf(pdev
, "initalize %d io queues\n", io_queue_num
);
3745 ena_init_io_rings(adapter
);
3747 /* setup network interface */
3748 rc
= ena_setup_ifnet(pdev
, adapter
, &get_feat_ctx
);
3749 if (unlikely(rc
!= 0)) {
3750 device_printf(pdev
, "Error with network interface setup\n");
3754 rc
= ena_enable_msix_and_set_admin_interrupts(adapter
, io_queue_num
);
3755 if (unlikely(rc
!= 0)) {
3757 "Failed to enable and set the admin interrupts\n");
3761 /* Initialize reset task queue */
3762 TASK_INIT(&adapter
->reset_task
, 0, ena_reset_task
, adapter
);
3763 adapter
->reset_tq
= taskqueue_create("ena_reset_enqueue",
3764 M_WAITOK
| M_ZERO
, taskqueue_thread_enqueue
, &adapter
->reset_tq
);
3765 taskqueue_start_threads(&adapter
->reset_tq
, 1, TDPRI_KERN_DAEMON
, -1,
3766 "%s rstq", device_get_nameunit(adapter
->pdev
));
3768 /* Initialize statistics */
3769 #if 0 /* XXX swildner counters */
3770 ena_alloc_counters((counter_u64_t
*)&adapter
->dev_stats
,
3771 sizeof(struct ena_stats_dev
));
3772 ena_alloc_counters((counter_u64_t
*)&adapter
->hw_stats
,
3773 sizeof(struct ena_hw_stats
));
3775 ena_sysctl_add_nodes(adapter
);
3777 /* Tell the stack that the interface is not active */
3778 ifq_set_oactive(&adapter
->ifp
->if_snd
);
3779 adapter
->ifp
->if_flags
&= ~IFF_RUNNING
;
3781 adapter
->running
= true;
3785 if_detach(adapter
->ifp
);
3786 if_free(adapter
->ifp
);
3788 ena_free_all_io_rings_resources(adapter
);
3789 ena_free_rx_dma_tag(adapter
);
3791 ena_free_tx_dma_tag(adapter
);
3793 ena_com_admin_destroy(ena_dev
);
3794 ena_com_delete_host_info(ena_dev
);
3795 ena_com_mmio_reg_read_request_destroy(ena_dev
);
3797 kfree(ena_dev
->bus
, M_DEVBUF
);
3798 kfree(ena_dev
, M_DEVBUF
);
3799 ena_free_pci_resources(adapter
);
3805 * ena_detach - Device Removal Routine
3806 * @pdev: device information struct
3808 * ena_detach is called by the device subsystem to alert the driver
3809 * that it should release a PCI device.
3812 ena_detach(device_t pdev
)
3814 struct ena_adapter
*adapter
= device_get_softc(pdev
);
3815 struct ena_com_dev
*ena_dev
= adapter
->ena_dev
;
3818 /* Make sure VLANS are not using driver */
3819 if (adapter
->ifp
->if_vlantrunks
!= NULL
) {
3820 device_printf(adapter
->pdev
,"VLAN is in use, detach first\n");
3824 /* Free reset task and callout */
3825 callout_drain(&adapter
->timer_service
);
3826 while (taskqueue_cancel(adapter
->reset_tq
, &adapter
->reset_task
, NULL
))
3827 taskqueue_drain(adapter
->reset_tq
, &adapter
->reset_task
);
3828 taskqueue_free(adapter
->reset_tq
);
3830 lockmgr(&adapter
->ioctl_lock
, LK_EXCLUSIVE
);
3832 lockmgr(&adapter
->ioctl_lock
, LK_RELEASE
);
3834 if (adapter
->ifp
!= NULL
) {
3835 ether_ifdetach(adapter
->ifp
);
3836 if_free(adapter
->ifp
);
3839 ena_free_all_io_rings_resources(adapter
);
3841 #if 0 /* XXX swildner counters */
3842 ena_free_counters((counter_u64_t
*)&adapter
->hw_stats
,
3843 sizeof(struct ena_hw_stats
));
3844 ena_free_counters((counter_u64_t
*)&adapter
->dev_stats
,
3845 sizeof(struct ena_stats_dev
));
3848 if (likely(adapter
->rss_support
))
3849 ena_com_rss_destroy(ena_dev
);
3851 rc
= ena_free_rx_dma_tag(adapter
);
3852 if (unlikely(rc
!= 0))
3853 device_printf(adapter
->pdev
,
3854 "Unmapped RX DMA tag associations\n");
3856 rc
= ena_free_tx_dma_tag(adapter
);
3857 if (unlikely(rc
!= 0))
3858 device_printf(adapter
->pdev
,
3859 "Unmapped TX DMA tag associations\n");
3861 /* Reset the device only if the device is running. */
3862 if (adapter
->running
)
3863 ena_com_dev_reset(ena_dev
, adapter
->reset_reason
);
3865 ena_com_delete_host_info(ena_dev
);
3867 ena_free_irqs(adapter
);
3869 ena_com_abort_admin_commands(ena_dev
);
3871 ena_com_wait_for_abort_completion(ena_dev
);
3873 ena_com_admin_destroy(ena_dev
);
3875 ena_com_mmio_reg_read_request_destroy(ena_dev
);
3877 ena_free_pci_resources(adapter
);
3879 lockuninit(&adapter
->global_lock
);
3880 lockuninit(&adapter
->ioctl_lock
);
3882 if (ena_dev
->bus
!= NULL
)
3883 kfree(ena_dev
->bus
, M_DEVBUF
);
3885 if (ena_dev
!= NULL
)
3886 kfree(ena_dev
, M_DEVBUF
);
3888 return (bus_generic_detach(pdev
));
3891 /******************************************************************************
3892 ******************************** AENQ Handlers *******************************
3893 *****************************************************************************/
3895 * ena_update_on_link_change:
3896 * Notify the network interface about the change in link status
3899 ena_update_on_link_change(void *adapter_data
,
3900 struct ena_admin_aenq_entry
*aenq_e
)
3902 struct ena_adapter
*adapter
= (struct ena_adapter
*)adapter_data
;
3903 struct ena_admin_aenq_link_change_desc
*aenq_desc
;
3907 aenq_desc
= (struct ena_admin_aenq_link_change_desc
*)aenq_e
;
3909 status
= aenq_desc
->flags
&
3910 ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK
;
3913 device_printf(adapter
->pdev
, "link is UP\n");
3914 ifp
->if_link_state
= LINK_STATE_UP
;
3915 if_link_state_change(ifp
);
3916 } else if (status
== 0) {
3917 device_printf(adapter
->pdev
, "link is DOWN\n");
3918 ifp
->if_link_state
= LINK_STATE_DOWN
;
3919 if_link_state_change(ifp
);
3921 device_printf(adapter
->pdev
, "invalid value recvd\n");
3925 adapter
->link_status
= status
;
3929 * This handler will called for unknown event group or unimplemented handlers
3932 unimplemented_aenq_handler(void *data
,
3933 struct ena_admin_aenq_entry
*aenq_e
)
3938 static struct ena_aenq_handlers aenq_handlers
= {
3940 [ENA_ADMIN_LINK_CHANGE
] = ena_update_on_link_change
,
3941 [ENA_ADMIN_KEEP_ALIVE
] = ena_keep_alive_wd
,
3943 .unimplemented_handler
= unimplemented_aenq_handler
3946 /*********************************************************************
3947 * FreeBSD Device Interface Entry Points
3948 *********************************************************************/
3950 static device_method_t ena_methods
[] = {
3951 /* Device interface */
3952 DEVMETHOD(device_probe
, ena_probe
),
3953 DEVMETHOD(device_attach
, ena_attach
),
3954 DEVMETHOD(device_detach
, ena_detach
),
3958 static driver_t ena_driver
= {
3959 "ena", ena_methods
, sizeof(struct ena_adapter
),
3962 devclass_t ena_devclass
;
3963 DRIVER_MODULE(ena
, pci
, ena_driver
, ena_devclass
, NULL
, NULL
);
3964 MODULE_DEPEND(ena
, pci
, 1, 1, 1);
3965 MODULE_DEPEND(ena
, ether
, 1, 1, 1);
3967 /*********************************************************************/