4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
27 #include <hxge_impl.h>
29 extern uint32_t hxge_reclaim_pending
;
30 extern uint32_t hxge_bcopy_thresh
;
31 extern uint32_t hxge_dvma_thresh
;
32 extern uint32_t hxge_dma_stream_thresh
;
33 extern uint32_t hxge_tx_minfree
;
34 extern uint32_t hxge_tx_intr_thres
;
35 extern uint32_t hxge_tx_max_gathers
;
36 extern uint32_t hxge_tx_tiny_pack
;
37 extern uint32_t hxge_tx_use_bcopy
;
39 static int hxge_start(p_hxge_t hxgep
, p_tx_ring_t tx_ring_p
, p_mblk_t mp
);
42 hxge_tx_ring_task(void *arg
)
44 p_tx_ring_t ring
= (p_tx_ring_t
)arg
;
46 MUTEX_ENTER(&ring
->lock
);
47 (void) hxge_txdma_reclaim(ring
->hxgep
, ring
, 0);
48 MUTEX_EXIT(&ring
->lock
);
50 mac_tx_ring_update(ring
->hxgep
->mach
, ring
->ring_handle
);
54 hxge_tx_ring_dispatch(p_tx_ring_t ring
)
57 * Kick the ring task to reclaim some buffers.
59 (void) ddi_taskq_dispatch(ring
->taskq
,
60 hxge_tx_ring_task
, (void *)ring
, DDI_SLEEP
);
64 hxge_tx_ring_send(void *arg
, mblk_t
*mp
)
66 p_hxge_ring_handle_t rhp
= (p_hxge_ring_handle_t
)arg
;
68 p_tx_ring_t tx_ring_p
;
72 ASSERT((rhp
->index
>= 0) && (rhp
->index
< HXGE_MAX_TDCS
));
75 tx_ring_p
= hxgep
->tx_rings
->rings
[rhp
->index
];
76 ASSERT(hxgep
== tx_ring_p
->hxgep
);
78 status
= hxge_start(hxgep
, tx_ring_p
, mp
);
80 hxge_tx_ring_dispatch(tx_ring_p
);
88 hxge_start(p_hxge_t hxgep
, p_tx_ring_t tx_ring_p
, p_mblk_t mp
)
90 int dma_status
, status
= 0;
91 p_tx_desc_t tx_desc_ring_vp
;
92 hpi_handle_t hpi_desc_handle
;
93 hxge_os_dma_handle_t tx_desc_dma_handle
;
94 p_tx_desc_t tx_desc_p
;
95 p_tx_msg_t tx_msg_ring
;
97 tx_desc_t tx_desc
, *tmp_desc_p
;
98 tx_desc_t sop_tx_desc
, *sop_tx_desc_p
;
99 p_tx_pkt_header_t hdrp
;
100 p_tx_pkt_hdr_all_t pkthdrp
;
111 uint32_t pkt_len
, pack_len
, min_len
;
112 uint32_t bcopy_thresh
;
113 int i
, cur_index
, sop_index
;
115 boolean_t tail_wrap
= B_FALSE
;
116 hxge_dma_common_t desc_area
;
117 hxge_os_dma_handle_t dma_handle
;
118 ddi_dma_cookie_t dma_cookie
;
119 hpi_handle_t hpi_handle
;
123 boolean_t good_packet
;
124 boolean_t mark_mode
= B_FALSE
;
125 p_hxge_stats_t statsp
;
126 p_hxge_tx_ring_stats_t tdc_stats
;
127 t_uscalar_t start_offset
= 0;
128 t_uscalar_t stuff_offset
= 0;
129 t_uscalar_t end_offset
= 0;
130 t_uscalar_t value
= 0;
131 t_uscalar_t cksum_flags
= 0;
132 boolean_t cksum_on
= B_FALSE
;
134 uint64_t tot_xfer_len
= 0, tmp_len
= 0;
135 boolean_t header_set
= B_FALSE
;
139 p_tx_desc_t tx_desc_ring_pp
;
140 p_tx_desc_t tx_desc_pp
;
141 tx_desc_t
*save_desc_p
;
149 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
150 "==> hxge_start: tx dma channel %d", tx_ring_p
->tdc
));
151 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
152 "==> hxge_start: Starting tdc %d desc pending %d",
153 tx_ring_p
->tdc
, tx_ring_p
->descs_pending
));
155 statsp
= hxgep
->statsp
;
157 if (hxgep
->statsp
->port_stats
.lb_mode
== hxge_lb_normal
) {
158 if (!statsp
->mac_stats
.link_up
) {
160 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "==> hxge_start: "
161 "link not up or LB mode"));
162 goto hxge_start_fail1
;
166 mac_hcksum_get(mp
, &start_offset
, &stuff_offset
, &end_offset
, &value
,
168 if (!HXGE_IS_VLAN_PACKET(mp
->b_rptr
)) {
169 start_offset
+= sizeof (ether_header_t
);
170 stuff_offset
+= sizeof (ether_header_t
);
172 start_offset
+= sizeof (struct ether_vlan_header
);
173 stuff_offset
+= sizeof (struct ether_vlan_header
);
176 if (cksum_flags
& HCK_PARTIALCKSUM
) {
177 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
178 "==> hxge_start: mp $%p len %d "
179 "cksum_flags 0x%x (partial checksum) ",
180 mp
, MBLKL(mp
), cksum_flags
));
184 MUTEX_ENTER(&tx_ring_p
->lock
);
187 sop_index
= tx_ring_p
->wr_index
;
189 if (tx_ring_p
->descs_pending
) {
190 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
191 "==> hxge_start: desc pending %d ",
192 tx_ring_p
->descs_pending
));
195 dump_len
= (int)(MBLKL(mp
));
196 dump_len
= (dump_len
> 128) ? 128: dump_len
;
198 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
199 "==> hxge_start: tdc %d: dumping ...: b_rptr $%p "
200 "(Before header reserve: ORIGINAL LEN %d)",
201 tx_ring_p
->tdc
, mp
->b_rptr
, dump_len
));
203 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
204 "==> hxge_start: dump packets (IP ORIGINAL b_rptr $%p): %s",
205 mp
->b_rptr
, hxge_dump_packet((char *)mp
->b_rptr
, dump_len
)));
208 tdc_stats
= tx_ring_p
->tdc_stats
;
209 mark_mode
= (tx_ring_p
->descs_pending
&&
210 ((tx_ring_p
->tx_ring_size
- tx_ring_p
->descs_pending
) <
213 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
214 "TX Descriptor ring is channel %d mark mode %d",
215 tx_ring_p
->tdc
, mark_mode
));
217 if (!hxge_txdma_reclaim(hxgep
, tx_ring_p
, hxge_tx_minfree
)) {
218 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
219 "TX Descriptor ring is full: channel %d", tx_ring_p
->tdc
));
220 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
221 "TX Descriptor ring is full: channel %d", tx_ring_p
->tdc
));
222 (void) atomic_cas_32((uint32_t *)&tx_ring_p
->queueing
, 0, 1);
223 tdc_stats
->tx_no_desc
++;
224 MUTEX_EXIT(&tx_ring_p
->lock
);
226 goto hxge_start_fail1
;
230 i
= sop_index
= tx_ring_p
->wr_index
;
237 good_packet
= B_TRUE
;
239 desc_area
= tx_ring_p
->tdc_desc
;
240 hpi_handle
= desc_area
.hpi_handle
;
241 hpi_desc_handle
.regh
= (hxge_os_acc_handle_t
)
242 DMA_COMMON_ACC_HANDLE(desc_area
);
243 hpi_desc_handle
.hxgep
= hxgep
;
244 tx_desc_ring_vp
= (p_tx_desc_t
)DMA_COMMON_VPTR(desc_area
);
247 tx_desc_ring_pp
= (p_tx_desc_t
)(uint32_t)DMA_COMMON_IOADDR(desc_area
);
249 tx_desc_ring_pp
= (p_tx_desc_t
)DMA_COMMON_IOADDR(desc_area
);
252 tx_desc_dma_handle
= (hxge_os_dma_handle_t
)DMA_COMMON_HANDLE(desc_area
);
253 tx_msg_ring
= tx_ring_p
->tx_msg_ring
;
255 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "==> hxge_start: wr_index %d i %d",
259 msgsize
= msgdsize(nmp
);
260 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
261 "==> hxge_start(1): wr_index %d i %d msgdsize %d",
262 sop_index
, i
, msgsize
));
265 * The first 16 bytes of the premapped buffer are reserved
266 * for header. No padding will be used.
268 pkt_len
= pack_len
= boff
= TX_PKT_HEADER_SIZE
;
269 if (hxge_tx_use_bcopy
) {
270 bcopy_thresh
= (hxge_bcopy_thresh
- TX_PKT_HEADER_SIZE
);
272 bcopy_thresh
= (TX_BCOPY_SIZE
- TX_PKT_HEADER_SIZE
);
275 good_packet
= B_TRUE
;
276 b_rptr
= nmp
->b_rptr
;
284 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "==> hxge_start(1): nmblks %d "
285 "len %d pkt_len %d pack_len %d",
286 nmblks
, len
, pkt_len
, pack_len
));
288 * Hardware limits the transfer length to 4K.
289 * If len is more than 4K, we need to break
290 * nmp into two chunks: Make first chunk smaller
291 * than 4K. The second chunk will be broken into
292 * less than 4K (if needed) during the next pass.
294 if (len
> (TX_MAX_TRANSFER_LENGTH
- TX_PKT_HEADER_SIZE
)) {
295 if ((t_mp
= dupb(nmp
)) != NULL
) {
296 nmp
->b_wptr
= nmp
->b_rptr
+
297 (TX_MAX_TRANSFER_LENGTH
-
299 t_mp
->b_rptr
= nmp
->b_wptr
;
300 t_mp
->b_cont
= nmp
->b_cont
;
304 good_packet
= B_FALSE
;
305 goto hxge_start_fail2
;
309 tx_desc_p
= &tx_desc_ring_vp
[i
];
311 tx_desc_pp
= &tx_desc_ring_pp
[i
];
313 tx_msg_p
= &tx_msg_ring
[i
];
315 hpi_desc_handle
.regp
= (uint32_t)tx_desc_p
;
317 hpi_desc_handle
.regp
= (uint64_t)tx_desc_p
;
320 ((!hxge_tx_use_bcopy
&& (len
> TX_BCOPY_SIZE
)) ||
321 (len
>= bcopy_thresh
))) {
323 bcopy_thresh
+= TX_PKT_HEADER_SIZE
;
326 kaddr
= (caddr_t
)DMA_COMMON_VPTR(tx_msg_p
->buf_dma
);
327 hdrp
= (p_tx_pkt_header_t
)kaddr
;
329 dma_handle
= tx_msg_p
->buf_dma_handle
;
330 dma_ioaddr
= DMA_COMMON_IOADDR(tx_msg_p
->buf_dma
);
331 offset
= tx_msg_p
->offset_index
* hxge_bcopy_thresh
;
332 (void) ddi_dma_sync(dma_handle
,
333 offset
, hxge_bcopy_thresh
, DDI_DMA_SYNC_FORDEV
);
335 tx_msg_p
->flags
.dma_type
= USE_BCOPY
;
336 goto hxge_start_control_header_only
;
342 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
343 "==> hxge_start(3): desc entry %d DESC IOADDR $%p "
344 "desc_vp $%p tx_desc_p $%p desc_pp $%p tx_desc_pp $%p "
345 "len %d pkt_len %d pack_len %d",
347 DMA_COMMON_IOADDR(desc_area
),
348 tx_desc_ring_vp
, tx_desc_p
,
349 tx_desc_ring_pp
, tx_desc_pp
,
350 len
, pkt_len
, pack_len
));
352 if (len
< bcopy_thresh
) {
353 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
354 "==> hxge_start(4): USE BCOPY: "));
355 if (hxge_tx_tiny_pack
) {
356 uint32_t blst
= TXDMA_DESC_NEXT_INDEX(i
, -1,
357 tx_ring_p
->tx_wrap_mask
);
358 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
359 "==> hxge_start(5): pack"));
360 if ((pack_len
<= bcopy_thresh
) &&
361 (last_bidx
== blst
)) {
362 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
363 "==> hxge_start: pack(6) "
364 "(pkt_len %d pack_len %d)",
367 tx_desc_p
= &tx_desc_ring_vp
[i
];
369 tx_desc_pp
= &tx_desc_ring_pp
[i
];
371 tx_msg_p
= &tx_msg_ring
[i
];
372 boff
= pack_len
- len
;
374 } else if (pack_len
> bcopy_thresh
&&
378 bcopy_thresh
= hxge_bcopy_thresh
;
379 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
380 "==> hxge_start(7): > max NEW "
382 "pkt_len %d pack_len %d(next)",
383 bcopy_thresh
, pkt_len
, pack_len
));
387 kaddr
= (caddr_t
)DMA_COMMON_VPTR(tx_msg_p
->buf_dma
);
388 if ((boff
== TX_PKT_HEADER_SIZE
) && (nmblks
== 1)) {
389 hdrp
= (p_tx_pkt_header_t
)kaddr
;
391 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
392 "==> hxge_start(7_x2): "
393 "pkt_len %d pack_len %d (new hdrp $%p)",
394 pkt_len
, pack_len
, hdrp
));
396 tx_msg_p
->flags
.dma_type
= USE_BCOPY
;
398 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
399 "==> hxge_start(8): USE BCOPY: before bcopy "
400 "DESC IOADDR $%p entry %d bcopy packets %d "
401 "bcopy kaddr $%p bcopy ioaddr (SAD) $%p "
402 "bcopy clen %d bcopy boff %d",
403 DMA_COMMON_IOADDR(desc_area
), i
,
404 tdc_stats
->tx_hdr_pkts
, kaddr
, dma_ioaddr
,
406 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
407 "==> hxge_start: 1USE BCOPY: "));
408 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
409 "==> hxge_start: 2USE BCOPY: "));
410 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "==> hxge_start: "
411 "last USE BCOPY: copy from b_rptr $%p "
412 "to KADDR $%p (len %d offset %d",
413 b_rptr
, kaddr
, len
, boff
));
414 bcopy(b_rptr
, kaddr
, len
);
416 dump_len
= (len
> 128) ? 128: len
;
417 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
418 "==> hxge_start: dump packets "
419 "(After BCOPY len %d)"
420 "(b_rptr $%p): %s", len
, nmp
->b_rptr
,
421 hxge_dump_packet((char *)nmp
->b_rptr
,
424 dma_handle
= tx_msg_p
->buf_dma_handle
;
425 dma_ioaddr
= DMA_COMMON_IOADDR(tx_msg_p
->buf_dma
);
426 offset
= tx_msg_p
->offset_index
* hxge_bcopy_thresh
;
427 (void) ddi_dma_sync(dma_handle
,
428 offset
, hxge_bcopy_thresh
, DDI_DMA_SYNC_FORDEV
);
430 tdc_stats
->tx_hdr_pkts
++;
431 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "==> hxge_start(9): "
432 "USE BCOPY: DESC IOADDR $%p entry %d "
433 "bcopy packets %d bcopy kaddr $%p "
434 "bcopy ioaddr (SAD) $%p bcopy clen %d "
436 DMA_COMMON_IOADDR(desc_area
), i
,
437 tdc_stats
->tx_hdr_pkts
, kaddr
, dma_ioaddr
,
440 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
441 "==> hxge_start(12): USE DVMA: len %d", len
));
442 tx_msg_p
->flags
.dma_type
= USE_DMA
;
443 dma_flags
= DDI_DMA_WRITE
;
444 if (len
< hxge_dma_stream_thresh
) {
445 dma_flags
|= DDI_DMA_CONSISTENT
;
447 dma_flags
|= DDI_DMA_STREAMING
;
450 dma_handle
= tx_msg_p
->dma_handle
;
451 dma_status
= ddi_dma_addr_bind_handle(dma_handle
, NULL
,
452 (caddr_t
)b_rptr
, len
, dma_flags
,
453 DDI_DMA_DONTWAIT
, NULL
,
454 &dma_cookie
, &ncookies
);
455 if (dma_status
== DDI_DMA_MAPPED
) {
456 dma_ioaddr
= dma_cookie
.dmac_laddress
;
457 len
= (int)dma_cookie
.dmac_size
;
458 clen
= (uint32_t)dma_cookie
.dmac_size
;
459 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
460 "==> hxge_start(12_1): "
461 "USE DVMA: len %d clen %d ngathers %d",
462 len
, clen
, ngathers
));
464 hpi_desc_handle
.regp
= (uint32_t)tx_desc_p
;
466 hpi_desc_handle
.regp
= (uint64_t)tx_desc_p
;
468 while (ncookies
> 1) {
471 * this is the fix for multiple
472 * cookies, which are basically
473 * a descriptor entry, we don't set
474 * SOP bit as well as related fields
477 (void) hpi_txdma_desc_gather_set(
478 hpi_desc_handle
, &tx_desc
,
479 (ngathers
-1), mark_mode
,
480 ngathers
, dma_ioaddr
, clen
);
481 tx_msg_p
->tx_msg_size
= clen
;
482 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
483 "==> hxge_start: DMA "
484 "ncookie %d ngathers %d "
485 "dma_ioaddr $%p len %d"
486 "desc $%p descp $%p (%d)",
489 *tx_desc_p
, tx_desc_p
, i
));
491 ddi_dma_nextcookie(dma_handle
,
493 dma_ioaddr
= dma_cookie
.dmac_laddress
;
495 len
= (int)dma_cookie
.dmac_size
;
496 clen
= (uint32_t)dma_cookie
.dmac_size
;
497 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
498 "==> hxge_start(12_2): "
499 "USE DVMA: len %d clen %d ",
502 i
= TXDMA_DESC_NEXT_INDEX(i
, 1,
503 tx_ring_p
->tx_wrap_mask
);
504 tx_desc_p
= &tx_desc_ring_vp
[i
];
506 hpi_desc_handle
.regp
=
512 tx_msg_p
= &tx_msg_ring
[i
];
513 tx_msg_p
->flags
.dma_type
= USE_NONE
;
517 tdc_stats
->tx_ddi_pkts
++;
518 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
519 "==> hxge_start: DMA: ddi packets %d",
520 tdc_stats
->tx_ddi_pkts
));
522 HXGE_ERROR_MSG((hxgep
, HXGE_ERR_CTL
,
523 "dma mapping failed for %d "
524 "bytes addr $%p flags %x (%d)",
525 len
, b_rptr
, status
, status
));
526 good_packet
= B_FALSE
;
527 tdc_stats
->tx_dma_bind_fail
++;
528 tx_msg_p
->flags
.dma_type
= USE_NONE
;
530 goto hxge_start_fail2
;
535 hxge_start_control_header_only
:
537 hpi_desc_handle
.regp
= (uint32_t)tx_desc_p
;
539 hpi_desc_handle
.regp
= (uint64_t)tx_desc_p
;
545 save_desc_p
= &sop_tx_desc
;
547 sop_tx_desc_p
= &sop_tx_desc
;
548 sop_tx_desc_p
->value
= 0;
549 sop_tx_desc_p
->bits
.tr_len
= clen
;
550 sop_tx_desc_p
->bits
.sad
= dma_ioaddr
>> 32;
551 sop_tx_desc_p
->bits
.sad_l
= dma_ioaddr
& 0xffffffff;
554 save_desc_p
= &tx_desc
;
556 tmp_desc_p
= &tx_desc
;
557 tmp_desc_p
->value
= 0;
558 tmp_desc_p
->bits
.tr_len
= clen
;
559 tmp_desc_p
->bits
.sad
= dma_ioaddr
>> 32;
560 tmp_desc_p
->bits
.sad_l
= dma_ioaddr
& 0xffffffff;
562 tx_desc_p
->value
= tmp_desc_p
->value
;
565 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
566 "==> hxge_start(13): Desc_entry %d ngathers %d "
567 "desc_vp $%p tx_desc_p $%p "
568 "len %d clen %d pkt_len %d pack_len %d nmblks %d "
569 "dma_ioaddr (SAD) $%p mark %d",
570 i
, ngathers
, tx_desc_ring_vp
, tx_desc_p
,
571 len
, clen
, pkt_len
, pack_len
, nmblks
,
572 dma_ioaddr
, mark_mode
));
575 hpi_desc_handle
.hxgep
= hxgep
;
576 hpi_desc_handle
.function
.function
= 0;
577 hpi_desc_handle
.function
.instance
= hxgep
->instance
;
578 sad
= save_desc_p
->bits
.sad
;
579 sad
= (sad
<< 32) | save_desc_p
->bits
.sad_l
;
580 xfer_len
= save_desc_p
->bits
.tr_len
;
582 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "\n\t: value 0x%llx\n"
583 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t"
585 save_desc_p
->value
, sad
, save_desc_p
->bits
.tr_len
,
586 xfer_len
, save_desc_p
->bits
.num_ptr
,
587 save_desc_p
->bits
.mark
, save_desc_p
->bits
.sop
));
589 hpi_txdma_dump_desc_one(hpi_desc_handle
, NULL
, i
);
592 tx_msg_p
->tx_msg_size
= clen
;
593 i
= TXDMA_DESC_NEXT_INDEX(i
, 1, tx_ring_p
->tx_wrap_mask
);
594 if (ngathers
> hxge_tx_max_gathers
) {
595 good_packet
= B_FALSE
;
596 mac_hcksum_get(mp
, &start_offset
, &stuff_offset
,
597 &end_offset
, &value
, &cksum_flags
);
599 HXGE_DEBUG_MSG((NULL
, TX_CTL
,
600 "==> hxge_start(14): pull msg - "
601 "len %d pkt_len %d ngathers %d",
602 len
, pkt_len
, ngathers
));
603 goto hxge_start_fail2
;
607 tx_msg_p
->tx_message
= mp
;
608 tx_desc_p
= &tx_desc_ring_vp
[sop_index
];
610 hpi_desc_handle
.regp
= (uint32_t)tx_desc_p
;
612 hpi_desc_handle
.regp
= (uint64_t)tx_desc_p
;
615 pkthdrp
= (p_tx_pkt_hdr_all_t
)hdrp
;
616 pkthdrp
->reserved
= 0;
618 (void) hxge_fill_tx_hdr(mp
, B_FALSE
, cksum_on
,
619 (pkt_len
- TX_PKT_HEADER_SIZE
), npads
, pkthdrp
);
622 * Hardware header should not be counted as part of the frame
623 * when determining the frame size
625 if ((pkt_len
- TX_PKT_HEADER_SIZE
) > (STD_FRAME_SIZE
- ETHERFCSL
)) {
626 tdc_stats
->tx_jumbo_pkts
++;
629 min_len
= (hxgep
->msg_min
+ TX_PKT_HEADER_SIZE
+ (npads
* 2));
630 if (pkt_len
< min_len
) {
631 /* Assume we use bcopy to premapped buffers */
632 kaddr
= (caddr_t
)DMA_COMMON_VPTR(tx_msg_p
->buf_dma
);
633 HXGE_DEBUG_MSG((NULL
, TX_CTL
,
634 "==> hxge_start(14-1): < (msg_min + 16)"
635 "len %d pkt_len %d min_len %d bzero %d ngathers %d",
636 len
, pkt_len
, min_len
, (min_len
- pkt_len
), ngathers
));
637 bzero((kaddr
+ pkt_len
), (min_len
- pkt_len
));
638 pkt_len
= tx_msg_p
->tx_msg_size
= min_len
;
640 sop_tx_desc_p
->bits
.tr_len
= min_len
;
642 HXGE_MEM_PIO_WRITE64(hpi_desc_handle
, sop_tx_desc_p
->value
);
643 tx_desc_p
->value
= sop_tx_desc_p
->value
;
645 HXGE_DEBUG_MSG((NULL
, TX_CTL
,
646 "==> hxge_start(14-2): < msg_min - "
647 "len %d pkt_len %d min_len %d ngathers %d",
648 len
, pkt_len
, min_len
, ngathers
));
651 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "==> hxge_start: cksum_flags 0x%x ",
653 if (cksum_flags
& HCK_PARTIALCKSUM
) {
654 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
655 "==> hxge_start: cksum_flags 0x%x (partial checksum) ",
658 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
659 "==> hxge_start: from IP cksum_flags 0x%x "
660 "(partial checksum) "
661 "start_offset %d stuff_offset %d",
662 cksum_flags
, start_offset
, stuff_offset
));
663 tmp_len
= (uint64_t)(start_offset
>> 1);
664 hdrp
->value
|= (tmp_len
<< TX_PKT_HEADER_L4START_SHIFT
);
665 tmp_len
= (uint64_t)(stuff_offset
>> 1);
666 hdrp
->value
|= (tmp_len
<< TX_PKT_HEADER_L4STUFF_SHIFT
);
668 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
669 "==> hxge_start: from IP cksum_flags 0x%x "
670 "(partial checksum) "
671 "after SHIFT start_offset %d stuff_offset %d",
672 cksum_flags
, start_offset
, stuff_offset
));
676 * pkt_len already includes 16 + paddings!!
677 * Update the control header length
681 * Note that Hydra is different from Neptune where
682 * tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE);
684 tot_xfer_len
= pkt_len
;
685 tmp_len
= hdrp
->value
|
686 (tot_xfer_len
<< TX_PKT_HEADER_TOT_XFER_LEN_SHIFT
);
688 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
689 "==> hxge_start(15_x1): setting SOP "
690 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len "
691 "0x%llx hdrp->value 0x%llx",
692 tot_xfer_len
, tot_xfer_len
, pkt_len
, tmp_len
, hdrp
->value
));
693 #if defined(_BIG_ENDIAN)
694 hdrp
->value
= ddi_swap64(tmp_len
);
696 hdrp
->value
= tmp_len
;
698 HXGE_DEBUG_MSG((hxgep
,
699 TX_CTL
, "==> hxge_start(15_x2): setting SOP "
700 "after SWAP: tot_xfer_len 0x%llx pkt_len %d "
701 "tmp_len 0x%llx hdrp->value 0x%llx",
702 tot_xfer_len
, pkt_len
, tmp_len
, hdrp
->value
));
704 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "==> hxge_start(15): setting SOP "
705 "wr_index %d tot_xfer_len (%d) pkt_len %d npads %d",
706 sop_index
, tot_xfer_len
, pkt_len
, npads
));
708 sop_tx_desc_p
->bits
.sop
= 1;
709 sop_tx_desc_p
->bits
.mark
= mark_mode
;
710 sop_tx_desc_p
->bits
.num_ptr
= ngathers
;
713 tdc_stats
->tx_marks
++;
715 HXGE_MEM_PIO_WRITE64(hpi_desc_handle
, sop_tx_desc_p
->value
);
716 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "==> hxge_start(16): set SOP done"));
719 hpi_desc_handle
.hxgep
= hxgep
;
720 hpi_desc_handle
.function
.function
= 0;
721 hpi_desc_handle
.function
.instance
= hxgep
->instance
;
723 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "\n\t: value 0x%llx\n"
724 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n",
725 save_desc_p
->value
, sad
, save_desc_p
->bits
.tr_len
,
726 xfer_len
, save_desc_p
->bits
.num_ptr
, save_desc_p
->bits
.mark
,
727 save_desc_p
->bits
.sop
));
728 (void) hpi_txdma_dump_desc_one(hpi_desc_handle
, NULL
, sop_index
);
730 dump_len
= (pkt_len
> 128) ? 128: pkt_len
;
731 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
732 "==> hxge_start: dump packets(17) (after sop set, len "
733 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n"
734 "ptr $%p: %s", len
, dump_len
, pkt_len
, tot_xfer_len
,
735 (char *)hdrp
, hxge_dump_packet((char *)hdrp
, dump_len
)));
736 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
737 "==> hxge_start(18): TX desc sync: sop_index %d", sop_index
));
740 if ((ngathers
== 1) || tx_ring_p
->wr_index
< i
) {
741 (void) ddi_dma_sync(tx_desc_dma_handle
,
742 sop_index
* sizeof (tx_desc_t
),
743 ngathers
* sizeof (tx_desc_t
), DDI_DMA_SYNC_FORDEV
);
745 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "hxge_start(19): sync 1 "
746 "cs_off = 0x%02X cs_s_off = 0x%02X "
747 "pkt_len %d ngathers %d sop_index %d\n",
748 stuff_offset
, start_offset
,
749 pkt_len
, ngathers
, sop_index
));
750 } else { /* more than one descriptor and wrap around */
751 uint32_t nsdescs
= tx_ring_p
->tx_ring_size
- sop_index
;
752 (void) ddi_dma_sync(tx_desc_dma_handle
,
753 sop_index
* sizeof (tx_desc_t
),
754 nsdescs
* sizeof (tx_desc_t
), DDI_DMA_SYNC_FORDEV
);
755 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "hxge_start(20): sync 1 "
756 "cs_off = 0x%02X cs_s_off = 0x%02X "
757 "pkt_len %d ngathers %d sop_index %d\n",
758 stuff_offset
, start_offset
, pkt_len
, ngathers
, sop_index
));
760 (void) ddi_dma_sync(tx_desc_dma_handle
, 0,
761 (ngathers
- nsdescs
) * sizeof (tx_desc_t
),
762 DDI_DMA_SYNC_FORDEV
);
763 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "hxge_start(21): sync 2 "
764 "cs_off = 0x%02X cs_s_off = 0x%02X "
765 "pkt_len %d ngathers %d sop_index %d\n",
766 stuff_offset
, start_offset
,
767 pkt_len
, ngathers
, sop_index
));
770 tail_index
= tx_ring_p
->wr_index
;
771 tail_wrap
= tx_ring_p
->wr_index_wrap
;
773 tx_ring_p
->wr_index
= i
;
774 if (tx_ring_p
->wr_index
<= tail_index
) {
775 tx_ring_p
->wr_index_wrap
= ((tail_wrap
== B_TRUE
) ?
779 tx_ring_p
->descs_pending
+= ngathers
;
780 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "==> hxge_start: TX kick: "
781 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d",
782 tx_ring_p
->tdc
, tx_ring_p
->wr_index
, tx_ring_p
->wr_index_wrap
,
783 ngathers
, tx_ring_p
->descs_pending
));
784 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "==> hxge_start: TX KICKING: "));
787 kick
.bits
.wrap
= tx_ring_p
->wr_index_wrap
;
788 kick
.bits
.tail
= (uint16_t)tx_ring_p
->wr_index
;
790 /* Kick start the Transmit kick register */
791 TXDMA_REG_WRITE64(HXGE_DEV_HPI_HANDLE(hxgep
),
792 TDC_TDR_KICK
, (uint8_t)tx_ring_p
->tdc
, kick
.value
);
793 tdc_stats
->tx_starts
++;
794 MUTEX_EXIT(&tx_ring_p
->lock
);
795 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "<== hxge_start"));
799 if (good_packet
== B_FALSE
) {
800 cur_index
= sop_index
;
801 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "==> hxge_start: clean up"));
802 for (i
= 0; i
< ngathers
; i
++) {
803 tx_desc_p
= &tx_desc_ring_vp
[cur_index
];
805 hpi_handle
.regp
= (uint32_t)tx_desc_p
;
807 hpi_handle
.regp
= (uint64_t)tx_desc_p
;
809 tx_msg_p
= &tx_msg_ring
[cur_index
];
810 (void) hpi_txdma_desc_set_zero(hpi_handle
, 1);
811 if (tx_msg_p
->flags
.dma_type
== USE_DVMA
) {
812 HXGE_DEBUG_MSG((hxgep
, TX_CTL
,
813 "tx_desc_p = %X index = %d",
814 tx_desc_p
, tx_ring_p
->rd_index
));
815 (void) dvma_unload(tx_msg_p
->dvma_handle
,
817 tx_msg_p
->dvma_handle
= NULL
;
818 if (tx_ring_p
->dvma_wr_index
==
819 tx_ring_p
->dvma_wrap_mask
)
820 tx_ring_p
->dvma_wr_index
= 0;
822 tx_ring_p
->dvma_wr_index
++;
823 tx_ring_p
->dvma_pending
--;
824 } else if (tx_msg_p
->flags
.dma_type
== USE_DMA
) {
825 if (ddi_dma_unbind_handle(
826 tx_msg_p
->dma_handle
)) {
827 cmn_err(CE_WARN
, "hxge_start: "
828 "ddi_dma_unbind_handle failed");
831 tx_msg_p
->flags
.dma_type
= USE_NONE
;
832 cur_index
= TXDMA_DESC_NEXT_INDEX(cur_index
, 1,
833 tx_ring_p
->tx_wrap_mask
);
838 MUTEX_EXIT(&tx_ring_p
->lock
);
841 /* Add FMA to check the access handle hxge_hregh */
842 HXGE_DEBUG_MSG((hxgep
, TX_CTL
, "<== hxge_start"));