2 * Copyright (c) 2010 Broadcom Corporation
4 * Permission to use, copy, modify, and/or distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
11 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
13 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
14 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
21 #include <bcmendian.h>
31 #define DMA_ERROR(args) if (!(*di->msg_level & 1)); else printf args
32 #define DMA_TRACE(args) if (!(*di->msg_level & 2)); else printf args
34 #define DMA_ERROR(args)
35 #define DMA_TRACE(args)
38 #define DMA_NONE(args)
40 #define d32txregs dregs.d32_u.txregs_32
41 #define d32rxregs dregs.d32_u.rxregs_32
42 #define txd32 dregs.d32_u.txd_32
43 #define rxd32 dregs.d32_u.rxd_32
45 #define d64txregs dregs.d64_u.txregs_64
46 #define d64rxregs dregs.d64_u.rxregs_64
47 #define txd64 dregs.d64_u.txd_64
48 #define rxd64 dregs.d64_u.rxd_64
50 /* default dma message level (if input msg_level pointer is null in dma_attach()) */
51 static uint dma_msg_level
= 0;
53 #define MAXNAMEL 8 /* 8 char names */
55 #define DI_INFO(dmah) ((dma_info_t *)dmah)
57 /* dma engine software state */
58 typedef struct dma_info
{
59 struct hnddma_pub hnddma
; /* exported structure, don't use hnddma_t,
60 * which could be const
62 uint
*msg_level
; /* message level pointer */
63 char name
[MAXNAMEL
]; /* callers name for diag msgs */
65 void *osh
; /* os handle */
66 si_t
*sih
; /* sb handle */
68 bool dma64
; /* this dma engine is operating in 64-bit mode */
69 bool addrext
; /* this dma engine supports DmaExtendedAddrChanges */
73 dma32regs_t
*txregs_32
; /* 32-bit dma tx engine registers */
74 dma32regs_t
*rxregs_32
; /* 32-bit dma rx engine registers */
75 dma32dd_t
*txd_32
; /* pointer to dma32 tx descriptor ring */
76 dma32dd_t
*rxd_32
; /* pointer to dma32 rx descriptor ring */
79 dma64regs_t
*txregs_64
; /* 64-bit dma tx engine registers */
80 dma64regs_t
*rxregs_64
; /* 64-bit dma rx engine registers */
81 dma64dd_t
*txd_64
; /* pointer to dma64 tx descriptor ring */
82 dma64dd_t
*rxd_64
; /* pointer to dma64 rx descriptor ring */
86 uint16 dmadesc_align
; /* alignment requirement for dma descriptors */
88 uint16 ntxd
; /* # tx descriptors tunable */
89 uint16 txin
; /* index of next descriptor to reclaim */
90 uint16 txout
; /* index of next descriptor to post */
91 void **txp
; /* pointer to parallel array of pointers to packets */
92 osldma_t
*tx_dmah
; /* DMA TX descriptor ring handle */
93 hnddma_seg_map_t
*txp_dmah
; /* DMA MAP meta-data handle */
94 dmaaddr_t txdpa
; /* Aligned physical address of descriptor ring */
95 dmaaddr_t txdpaorig
; /* Original physical address of descriptor ring */
96 uint16 txdalign
; /* #bytes added to alloc'd mem to align txd */
97 uint32 txdalloc
; /* #bytes allocated for the ring */
98 uint32 xmtptrbase
; /* When using unaligned descriptors, the ptr register
99 * is not just an index, it needs all 13 bits to be
100 * an offset from the addr register.
103 uint16 nrxd
; /* # rx descriptors tunable */
104 uint16 rxin
; /* index of next descriptor to reclaim */
105 uint16 rxout
; /* index of next descriptor to post */
106 void **rxp
; /* pointer to parallel array of pointers to packets */
107 osldma_t
*rx_dmah
; /* DMA RX descriptor ring handle */
108 hnddma_seg_map_t
*rxp_dmah
; /* DMA MAP meta-data handle */
109 dmaaddr_t rxdpa
; /* Aligned physical address of descriptor ring */
110 dmaaddr_t rxdpaorig
; /* Original physical address of descriptor ring */
111 uint16 rxdalign
; /* #bytes added to alloc'd mem to align rxd */
112 uint32 rxdalloc
; /* #bytes allocated for the ring */
113 uint32 rcvptrbase
; /* Base for ptr reg when using unaligned descriptors */
116 uint16 rxbufsize
; /* rx buffer size in bytes,
117 * not including the extra headroom
119 uint rxextrahdrroom
; /* extra rx headroom, reverseved to assist upper stack
120 * e.g. some rx pkt buffers will be bridged to tx side
121 * without byte copying. The extra headroom needs to be
122 * large enough to fit txheader needs.
123 * Some dongle driver may not need it.
125 uint nrxpost
; /* # rx buffers to keep posted */
126 uint rxoffset
; /* rxcontrol offset */
127 uint ddoffsetlow
; /* add to get dma address of descriptor ring, low 32 bits */
128 uint ddoffsethigh
; /* high 32 bits */
129 uint dataoffsetlow
; /* add to get dma address of data buffer, low 32 bits */
130 uint dataoffsethigh
; /* high 32 bits */
131 bool aligndesc_4k
; /* descriptor base need to be aligned or not */
135 * If BCMDMA32 is defined, hnddma will support both 32-bit and 64-bit DMA engines.
136 * Otherwise it will support only 64-bit.
138 * DMA32_ENAB indicates whether hnddma is compiled with support for 32-bit DMA engines.
139 * DMA64_ENAB indicates whether hnddma is compiled with support for 64-bit DMA engines.
141 * DMA64_MODE indicates whether the current DMA engine is running as 64-bit.
144 #define DMA32_ENAB(di) 1
145 #define DMA64_ENAB(di) 1
146 #define DMA64_MODE(di) ((di)->dma64)
147 #else /* !BCMDMA32 */
148 #define DMA32_ENAB(di) 0
149 #define DMA64_ENAB(di) 1
150 #define DMA64_MODE(di) 1
151 #endif /* !BCMDMA32 */
153 /* DMA Scatter-gather list is supported. Note this is limited to TX direction only */
154 #ifdef BCMDMASGLISTOSL
155 #define DMASGLIST_ENAB TRUE
157 #define DMASGLIST_ENAB FALSE
158 #endif /* BCMDMASGLISTOSL */
160 /* descriptor bumping macros */
161 #define XXD(x, n) ((x) & ((n) - 1)) /* faster than %, but n must be power of 2 */
162 #define TXD(x) XXD((x), di->ntxd)
163 #define RXD(x) XXD((x), di->nrxd)
164 #define NEXTTXD(i) TXD((i) + 1)
165 #define PREVTXD(i) TXD((i) - 1)
166 #define NEXTRXD(i) RXD((i) + 1)
167 #define PREVRXD(i) RXD((i) - 1)
169 #define NTXDACTIVE(h, t) TXD((t) - (h))
170 #define NRXDACTIVE(h, t) RXD((t) - (h))
172 /* macros to convert between byte offsets and indexes */
173 #define B2I(bytes, type) ((bytes) / sizeof(type))
174 #define I2B(index, type) ((index) * sizeof(type))
176 #define PCI32ADDR_HIGH 0xc0000000 /* address[31:30] */
177 #define PCI32ADDR_HIGH_SHIFT 30 /* address[31:30] */
179 #define PCI64ADDR_HIGH 0x80000000 /* address[63] */
180 #define PCI64ADDR_HIGH_SHIFT 31 /* address[63] */
182 /* Common prototypes */
183 static bool _dma_isaddrext(dma_info_t
* di
);
184 static bool _dma_descriptor_align(dma_info_t
* di
);
185 static bool _dma_alloc(dma_info_t
* di
, uint direction
);
186 static void _dma_detach(dma_info_t
* di
);
187 static void _dma_ddtable_init(dma_info_t
* di
, uint direction
, dmaaddr_t pa
);
188 static void _dma_rxinit(dma_info_t
* di
);
189 static void *_dma_rx(dma_info_t
* di
);
190 static bool _dma_rxfill(dma_info_t
* di
);
191 static void _dma_rxreclaim(dma_info_t
* di
);
192 static void _dma_rxenable(dma_info_t
* di
);
193 static void *_dma_getnextrxp(dma_info_t
* di
, bool forceall
);
194 static void _dma_rx_param_get(dma_info_t
* di
, uint16
* rxoffset
,
197 static void _dma_txblock(dma_info_t
* di
);
198 static void _dma_txunblock(dma_info_t
* di
);
199 static uint
_dma_txactive(dma_info_t
* di
);
200 static uint
_dma_rxactive(dma_info_t
* di
);
201 static uint
_dma_txpending(dma_info_t
* di
);
202 static uint
_dma_txcommitted(dma_info_t
* di
);
204 static void *_dma_peeknexttxp(dma_info_t
* di
);
205 static void *_dma_peeknextrxp(dma_info_t
* di
);
206 static uintptr
_dma_getvar(dma_info_t
* di
, const char *name
);
207 static void _dma_counterreset(dma_info_t
* di
);
208 static void _dma_fifoloopbackenable(dma_info_t
* di
);
209 static uint
_dma_ctrlflags(dma_info_t
* di
, uint mask
, uint flags
);
210 static uint8
dma_align_sizetobits(uint size
);
211 static void *dma_ringalloc(osl_t
* osh
, uint32 boundary
, uint size
,
212 uint16
* alignbits
, uint
* alloced
,
213 dmaaddr_t
* descpa
, osldma_t
** dmah
);
215 /* Prototypes for 32-bit routines */
216 static bool dma32_alloc(dma_info_t
* di
, uint direction
);
217 static bool dma32_txreset(dma_info_t
* di
);
218 static bool dma32_rxreset(dma_info_t
* di
);
219 static bool dma32_txsuspendedidle(dma_info_t
* di
);
220 static int dma32_txfast(dma_info_t
* di
, void *p0
, bool commit
);
221 static void *dma32_getnexttxp(dma_info_t
* di
, txd_range_t range
);
222 static void *dma32_getnextrxp(dma_info_t
* di
, bool forceall
);
223 static void dma32_txrotate(dma_info_t
* di
);
224 static bool dma32_rxidle(dma_info_t
* di
);
225 static void dma32_txinit(dma_info_t
* di
);
226 static bool dma32_txenabled(dma_info_t
* di
);
227 static void dma32_txsuspend(dma_info_t
* di
);
228 static void dma32_txresume(dma_info_t
* di
);
229 static bool dma32_txsuspended(dma_info_t
* di
);
230 static void dma32_txreclaim(dma_info_t
* di
, txd_range_t range
);
231 static bool dma32_txstopped(dma_info_t
* di
);
232 static bool dma32_rxstopped(dma_info_t
* di
);
233 static bool dma32_rxenabled(dma_info_t
* di
);
235 static bool _dma32_addrext(osl_t
* osh
, dma32regs_t
* dma32regs
);
237 /* Prototypes for 64-bit routines */
238 static bool dma64_alloc(dma_info_t
* di
, uint direction
);
239 static bool dma64_txreset(dma_info_t
* di
);
240 static bool dma64_rxreset(dma_info_t
* di
);
241 static bool dma64_txsuspendedidle(dma_info_t
* di
);
242 static int dma64_txfast(dma_info_t
* di
, void *p0
, bool commit
);
243 static int dma64_txunframed(dma_info_t
* di
, void *p0
, uint len
, bool commit
);
244 static void *dma64_getpos(dma_info_t
* di
, bool direction
);
245 static void *dma64_getnexttxp(dma_info_t
* di
, txd_range_t range
);
246 static void *dma64_getnextrxp(dma_info_t
* di
, bool forceall
);
247 static void dma64_txrotate(dma_info_t
* di
);
249 static bool dma64_rxidle(dma_info_t
* di
);
250 static void dma64_txinit(dma_info_t
* di
);
251 static bool dma64_txenabled(dma_info_t
* di
);
252 static void dma64_txsuspend(dma_info_t
* di
);
253 static void dma64_txresume(dma_info_t
* di
);
254 static bool dma64_txsuspended(dma_info_t
* di
);
255 static void dma64_txreclaim(dma_info_t
* di
, txd_range_t range
);
256 static bool dma64_txstopped(dma_info_t
* di
);
257 static bool dma64_rxstopped(dma_info_t
* di
);
258 static bool dma64_rxenabled(dma_info_t
* di
);
259 static bool _dma64_addrext(osl_t
* osh
, dma64regs_t
* dma64regs
);
261 STATIC INLINE uint32
parity32(uint32 data
);
263 const di_fcn_t dma64proc
= {
264 (di_detach_t
) _dma_detach
,
265 (di_txinit_t
) dma64_txinit
,
266 (di_txreset_t
) dma64_txreset
,
267 (di_txenabled_t
) dma64_txenabled
,
268 (di_txsuspend_t
) dma64_txsuspend
,
269 (di_txresume_t
) dma64_txresume
,
270 (di_txsuspended_t
) dma64_txsuspended
,
271 (di_txsuspendedidle_t
) dma64_txsuspendedidle
,
272 (di_txfast_t
) dma64_txfast
,
273 (di_txunframed_t
) dma64_txunframed
,
274 (di_getpos_t
) dma64_getpos
,
275 (di_txstopped_t
) dma64_txstopped
,
276 (di_txreclaim_t
) dma64_txreclaim
,
277 (di_getnexttxp_t
) dma64_getnexttxp
,
278 (di_peeknexttxp_t
) _dma_peeknexttxp
,
279 (di_txblock_t
) _dma_txblock
,
280 (di_txunblock_t
) _dma_txunblock
,
281 (di_txactive_t
) _dma_txactive
,
282 (di_txrotate_t
) dma64_txrotate
,
284 (di_rxinit_t
) _dma_rxinit
,
285 (di_rxreset_t
) dma64_rxreset
,
286 (di_rxidle_t
) dma64_rxidle
,
287 (di_rxstopped_t
) dma64_rxstopped
,
288 (di_rxenable_t
) _dma_rxenable
,
289 (di_rxenabled_t
) dma64_rxenabled
,
291 (di_rxfill_t
) _dma_rxfill
,
292 (di_rxreclaim_t
) _dma_rxreclaim
,
293 (di_getnextrxp_t
) _dma_getnextrxp
,
294 (di_peeknextrxp_t
) _dma_peeknextrxp
,
295 (di_rxparam_get_t
) _dma_rx_param_get
,
297 (di_fifoloopbackenable_t
) _dma_fifoloopbackenable
,
298 (di_getvar_t
) _dma_getvar
,
299 (di_counterreset_t
) _dma_counterreset
,
300 (di_ctrlflags_t
) _dma_ctrlflags
,
304 (di_rxactive_t
) _dma_rxactive
,
305 (di_txpending_t
) _dma_txpending
,
306 (di_txcommitted_t
) _dma_txcommitted
,
310 static const di_fcn_t dma32proc
= {
311 (di_detach_t
) _dma_detach
,
312 (di_txinit_t
) dma32_txinit
,
313 (di_txreset_t
) dma32_txreset
,
314 (di_txenabled_t
) dma32_txenabled
,
315 (di_txsuspend_t
) dma32_txsuspend
,
316 (di_txresume_t
) dma32_txresume
,
317 (di_txsuspended_t
) dma32_txsuspended
,
318 (di_txsuspendedidle_t
) dma32_txsuspendedidle
,
319 (di_txfast_t
) dma32_txfast
,
322 (di_txstopped_t
) dma32_txstopped
,
323 (di_txreclaim_t
) dma32_txreclaim
,
324 (di_getnexttxp_t
) dma32_getnexttxp
,
325 (di_peeknexttxp_t
) _dma_peeknexttxp
,
326 (di_txblock_t
) _dma_txblock
,
327 (di_txunblock_t
) _dma_txunblock
,
328 (di_txactive_t
) _dma_txactive
,
329 (di_txrotate_t
) dma32_txrotate
,
331 (di_rxinit_t
) _dma_rxinit
,
332 (di_rxreset_t
) dma32_rxreset
,
333 (di_rxidle_t
) dma32_rxidle
,
334 (di_rxstopped_t
) dma32_rxstopped
,
335 (di_rxenable_t
) _dma_rxenable
,
336 (di_rxenabled_t
) dma32_rxenabled
,
338 (di_rxfill_t
) _dma_rxfill
,
339 (di_rxreclaim_t
) _dma_rxreclaim
,
340 (di_getnextrxp_t
) _dma_getnextrxp
,
341 (di_peeknextrxp_t
) _dma_peeknextrxp
,
342 (di_rxparam_get_t
) _dma_rx_param_get
,
344 (di_fifoloopbackenable_t
) _dma_fifoloopbackenable
,
345 (di_getvar_t
) _dma_getvar
,
346 (di_counterreset_t
) _dma_counterreset
,
347 (di_ctrlflags_t
) _dma_ctrlflags
,
351 (di_rxactive_t
) _dma_rxactive
,
352 (di_txpending_t
) _dma_txpending
,
353 (di_txcommitted_t
) _dma_txcommitted
,
357 hnddma_t
*dma_attach(osl_t
* osh
, char *name
, si_t
* sih
, void *dmaregstx
,
358 void *dmaregsrx
, uint ntxd
, uint nrxd
, uint rxbufsize
,
359 int rxextheadroom
, uint nrxpost
, uint rxoffset
,
365 /* allocate private info structure */
366 if ((di
= MALLOC(osh
, sizeof(dma_info_t
))) == NULL
) {
368 printf("dma_attach: out of memory, malloced %d bytes\n",
374 bzero((char *)di
, sizeof(dma_info_t
));
376 di
->msg_level
= msg_level
? msg_level
: &dma_msg_level
;
378 /* old chips w/o sb is no longer supported */
383 ((si_core_sflags(sih
, 0, 0) & SISF_DMA64
) == SISF_DMA64
);
387 /* check arguments */
388 ASSERT(ISPOWEROF2(ntxd
));
389 ASSERT(ISPOWEROF2(nrxd
));
392 ASSERT(dmaregsrx
== NULL
);
394 ASSERT(dmaregstx
== NULL
);
396 /* init dma reg pointer */
397 if (DMA64_ENAB(di
) && DMA64_MODE(di
)) {
398 ASSERT(ntxd
<= D64MAXDD
);
399 ASSERT(nrxd
<= D64MAXDD
);
400 di
->d64txregs
= (dma64regs_t
*) dmaregstx
;
401 di
->d64rxregs
= (dma64regs_t
*) dmaregsrx
;
402 di
->hnddma
.di_fn
= (const di_fcn_t
*)&dma64proc
;
403 } else if (DMA32_ENAB(di
)) {
404 ASSERT(ntxd
<= D32MAXDD
);
405 ASSERT(nrxd
<= D32MAXDD
);
406 di
->d32txregs
= (dma32regs_t
*) dmaregstx
;
407 di
->d32rxregs
= (dma32regs_t
*) dmaregsrx
;
408 di
->hnddma
.di_fn
= (const di_fcn_t
*)&dma32proc
;
410 DMA_ERROR(("dma_attach: driver doesn't support 32-bit DMA\n"));
415 /* Default flags (which can be changed by the driver calling dma_ctrlflags
416 * before enable): For backwards compatibility both Rx Overflow Continue
417 * and Parity are DISABLED.
420 di
->hnddma
.di_fn
->ctrlflags(&di
->hnddma
, DMA_CTRL_ROC
| DMA_CTRL_PEN
,
423 DMA_TRACE(("%s: dma_attach: %s osh %p flags 0x%x ntxd %d nrxd %d rxbufsize %d " "rxextheadroom %d nrxpost %d rxoffset %d dmaregstx %p dmaregsrx %p\n", name
, (DMA64_MODE(di
) ? "DMA64" : "DMA32"), osh
, di
->hnddma
.dmactrlflags
, ntxd
, nrxd
, rxbufsize
, rxextheadroom
, nrxpost
, rxoffset
, dmaregstx
, dmaregsrx
));
425 /* make a private copy of our callers name */
426 strncpy(di
->name
, name
, MAXNAMEL
);
427 di
->name
[MAXNAMEL
- 1] = '\0';
433 di
->ntxd
= (uint16
) ntxd
;
434 di
->nrxd
= (uint16
) nrxd
;
436 /* the actual dma size doesn't include the extra headroom */
438 (rxextheadroom
== -1) ? BCMEXTRAHDROOM
: rxextheadroom
;
439 if (rxbufsize
> BCMEXTRAHDROOM
)
440 di
->rxbufsize
= (uint16
) (rxbufsize
- di
->rxextrahdrroom
);
442 di
->rxbufsize
= (uint16
) rxbufsize
;
444 di
->nrxpost
= (uint16
) nrxpost
;
445 di
->rxoffset
= (uint8
) rxoffset
;
448 * figure out the DMA physical address offset for dd and data
449 * PCI/PCIE: they map silicon backplace address to zero based memory, need offset
450 * Other bus: use zero
451 * SI_BUS BIGENDIAN kludge: use sdram swapped region for data buffer, not descriptor
454 di
->dataoffsetlow
= 0;
455 /* for pci bus, add offset */
456 if (sih
->bustype
== PCI_BUS
) {
457 if ((sih
->buscoretype
== PCIE_CORE_ID
) && DMA64_MODE(di
)) {
458 /* pcie with DMA64 */
460 di
->ddoffsethigh
= SI_PCIE_DMA_H32
;
462 /* pci(DMA32/DMA64) or pcie with DMA32 */
463 di
->ddoffsetlow
= SI_PCI_DMA
;
464 di
->ddoffsethigh
= 0;
466 di
->dataoffsetlow
= di
->ddoffsetlow
;
467 di
->dataoffsethigh
= di
->ddoffsethigh
;
469 #if defined(__mips__) && defined(IL_BIGENDIAN)
470 di
->dataoffsetlow
= di
->dataoffsetlow
+ SI_SDRAM_SWAPPED
;
471 #endif /* defined(__mips__) && defined(IL_BIGENDIAN) */
472 /* WAR64450 : DMACtl.Addr ext fields are not supported in SDIOD core. */
473 if ((si_coreid(sih
) == SDIOD_CORE_ID
)
474 && ((si_corerev(sih
) > 0) && (si_corerev(sih
) <= 2)))
476 else if ((si_coreid(sih
) == I2S_CORE_ID
) &&
477 ((si_corerev(sih
) == 0) || (si_corerev(sih
) == 1)))
480 di
->addrext
= _dma_isaddrext(di
);
482 /* does the descriptors need to be aligned and if yes, on 4K/8K or not */
483 di
->aligndesc_4k
= _dma_descriptor_align(di
);
484 if (di
->aligndesc_4k
) {
485 if (DMA64_MODE(di
)) {
486 di
->dmadesc_align
= D64RINGALIGN_BITS
;
487 if ((ntxd
< D64MAXDD
/ 2) && (nrxd
< D64MAXDD
/ 2)) {
488 /* for smaller dd table, HW relax the alignment requirement */
489 di
->dmadesc_align
= D64RINGALIGN_BITS
- 1;
492 di
->dmadesc_align
= D32RINGALIGN_BITS
;
494 di
->dmadesc_align
= 4; /* 16 byte alignment */
496 DMA_NONE(("DMA descriptor align_needed %d, align %d\n",
497 di
->aligndesc_4k
, di
->dmadesc_align
));
499 /* allocate tx packet pointer vector */
501 size
= ntxd
* sizeof(void *);
502 if ((di
->txp
= MALLOC(osh
, size
)) == NULL
) {
503 DMA_ERROR(("%s: dma_attach: out of tx memory, malloced %d bytes\n", di
->name
, MALLOCED(osh
)));
506 bzero((char *)di
->txp
, size
);
509 /* allocate rx packet pointer vector */
511 size
= nrxd
* sizeof(void *);
512 if ((di
->rxp
= MALLOC(osh
, size
)) == NULL
) {
513 DMA_ERROR(("%s: dma_attach: out of rx memory, malloced %d bytes\n", di
->name
, MALLOCED(osh
)));
516 bzero((char *)di
->rxp
, size
);
519 /* allocate transmit descriptor ring, only need ntxd descriptors but it must be aligned */
521 if (!_dma_alloc(di
, DMA_TX
))
525 /* allocate receive descriptor ring, only need nrxd descriptors but it must be aligned */
527 if (!_dma_alloc(di
, DMA_RX
))
531 if ((di
->ddoffsetlow
!= 0) && !di
->addrext
) {
532 if (PHYSADDRLO(di
->txdpa
) > SI_PCI_DMA_SZ
) {
533 DMA_ERROR(("%s: dma_attach: txdpa 0x%x: addrext not supported\n", di
->name
, (uint32
) PHYSADDRLO(di
->txdpa
)));
536 if (PHYSADDRLO(di
->rxdpa
) > SI_PCI_DMA_SZ
) {
537 DMA_ERROR(("%s: dma_attach: rxdpa 0x%x: addrext not supported\n", di
->name
, (uint32
) PHYSADDRLO(di
->rxdpa
)));
542 DMA_TRACE(("ddoffsetlow 0x%x ddoffsethigh 0x%x dataoffsetlow 0x%x dataoffsethigh " "0x%x addrext %d\n", di
->ddoffsetlow
, di
->ddoffsethigh
, di
->dataoffsetlow
, di
->dataoffsethigh
, di
->addrext
));
544 /* allocate DMA mapping vectors */
545 if (DMASGLIST_ENAB
) {
547 size
= ntxd
* sizeof(hnddma_seg_map_t
);
549 (hnddma_seg_map_t
*) MALLOC(osh
, size
)) == NULL
)
551 bzero((char *)di
->txp_dmah
, size
);
555 size
= nrxd
* sizeof(hnddma_seg_map_t
);
557 (hnddma_seg_map_t
*) MALLOC(osh
, size
)) == NULL
)
559 bzero((char *)di
->rxp_dmah
, size
);
563 return ((hnddma_t
*) di
);
570 /* init the tx or rx descriptor */
572 dma32_dd_upd(dma_info_t
* di
, dma32dd_t
* ddring
, dmaaddr_t pa
, uint outidx
,
573 uint32
* flags
, uint32 bufcount
)
575 /* dma32 uses 32-bit control to fit both flags and bufcounter */
576 *flags
= *flags
| (bufcount
& CTRL_BC_MASK
);
578 if ((di
->dataoffsetlow
== 0) || !(PHYSADDRLO(pa
) & PCI32ADDR_HIGH
)) {
579 W_SM(&ddring
[outidx
].addr
,
580 BUS_SWAP32(PHYSADDRLO(pa
) + di
->dataoffsetlow
));
581 W_SM(&ddring
[outidx
].ctrl
, BUS_SWAP32(*flags
));
583 /* address extension */
586 ae
= (PHYSADDRLO(pa
) & PCI32ADDR_HIGH
) >> PCI32ADDR_HIGH_SHIFT
;
587 PHYSADDRLO(pa
) &= ~PCI32ADDR_HIGH
;
589 *flags
|= (ae
<< CTRL_AE_SHIFT
);
590 W_SM(&ddring
[outidx
].addr
,
591 BUS_SWAP32(PHYSADDRLO(pa
) + di
->dataoffsetlow
));
592 W_SM(&ddring
[outidx
].ctrl
, BUS_SWAP32(*flags
));
596 /* Check for odd number of 1's */
597 STATIC INLINE uint32
parity32(uint32 data
)
608 #define DMA64_DD_PARITY(dd) parity32((dd)->addrlow ^ (dd)->addrhigh ^ (dd)->ctrl1 ^ (dd)->ctrl2)
611 dma64_dd_upd(dma_info_t
* di
, dma64dd_t
* ddring
, dmaaddr_t pa
, uint outidx
,
612 uint32
* flags
, uint32 bufcount
)
614 uint32 ctrl2
= bufcount
& D64_CTRL2_BC_MASK
;
616 /* PCI bus with big(>1G) physical address, use address extension */
617 #if defined(__mips__) && defined(IL_BIGENDIAN)
618 if ((di
->dataoffsetlow
== SI_SDRAM_SWAPPED
)
619 || !(PHYSADDRLO(pa
) & PCI32ADDR_HIGH
)) {
621 if ((di
->dataoffsetlow
== 0) || !(PHYSADDRLO(pa
) & PCI32ADDR_HIGH
)) {
622 #endif /* defined(__mips__) && defined(IL_BIGENDIAN) */
623 ASSERT((PHYSADDRHI(pa
) & PCI64ADDR_HIGH
) == 0);
625 W_SM(&ddring
[outidx
].addrlow
,
626 BUS_SWAP32(PHYSADDRLO(pa
) + di
->dataoffsetlow
));
627 W_SM(&ddring
[outidx
].addrhigh
,
628 BUS_SWAP32(PHYSADDRHI(pa
) + di
->dataoffsethigh
));
629 W_SM(&ddring
[outidx
].ctrl1
, BUS_SWAP32(*flags
));
630 W_SM(&ddring
[outidx
].ctrl2
, BUS_SWAP32(ctrl2
));
632 /* address extension for 32-bit PCI */
636 ae
= (PHYSADDRLO(pa
) & PCI32ADDR_HIGH
) >> PCI32ADDR_HIGH_SHIFT
;
637 PHYSADDRLO(pa
) &= ~PCI32ADDR_HIGH
;
638 ASSERT(PHYSADDRHI(pa
) == 0);
640 ctrl2
|= (ae
<< D64_CTRL2_AE_SHIFT
) & D64_CTRL2_AE
;
641 W_SM(&ddring
[outidx
].addrlow
,
642 BUS_SWAP32(PHYSADDRLO(pa
) + di
->dataoffsetlow
));
643 W_SM(&ddring
[outidx
].addrhigh
,
644 BUS_SWAP32(0 + di
->dataoffsethigh
));
645 W_SM(&ddring
[outidx
].ctrl1
, BUS_SWAP32(*flags
));
646 W_SM(&ddring
[outidx
].ctrl2
, BUS_SWAP32(ctrl2
));
648 if (di
->hnddma
.dmactrlflags
& DMA_CTRL_PEN
) {
649 if (DMA64_DD_PARITY(&ddring
[outidx
])) {
650 W_SM(&ddring
[outidx
].ctrl2
,
651 BUS_SWAP32(ctrl2
| D64_CTRL2_PARITY
));
656 static bool _dma32_addrext(osl_t
* osh
, dma32regs_t
* dma32regs
)
660 OR_REG(osh
, &dma32regs
->control
, XC_AE
);
661 w
= R_REG(osh
, &dma32regs
->control
);
662 AND_REG(osh
, &dma32regs
->control
, ~XC_AE
);
663 return ((w
& XC_AE
) == XC_AE
);
666 static bool _dma_alloc(dma_info_t
* di
, uint direction
)
668 if (DMA64_ENAB(di
) && DMA64_MODE(di
)) {
669 return dma64_alloc(di
, direction
);
670 } else if (DMA32_ENAB(di
)) {
671 return dma32_alloc(di
, direction
);
676 /* !! may be called with core in reset */
677 static void _dma_detach(dma_info_t
* di
)
680 DMA_TRACE(("%s: dma_detach\n", di
->name
));
682 /* shouldn't be here if descriptors are unreclaimed */
683 ASSERT(di
->txin
== di
->txout
);
684 ASSERT(di
->rxin
== di
->rxout
);
686 /* free dma descriptor rings */
687 if (DMA64_ENAB(di
) && DMA64_MODE(di
)) {
689 DMA_FREE_CONSISTENT(di
->osh
,
690 ((int8
*) (uintptr
) di
->txd64
-
691 di
->txdalign
), di
->txdalloc
,
692 (di
->txdpaorig
), &di
->tx_dmah
);
694 DMA_FREE_CONSISTENT(di
->osh
,
695 ((int8
*) (uintptr
) di
->rxd64
-
696 di
->rxdalign
), di
->rxdalloc
,
697 (di
->rxdpaorig
), &di
->rx_dmah
);
698 } else if (DMA32_ENAB(di
)) {
700 DMA_FREE_CONSISTENT(di
->osh
,
701 ((int8
*) (uintptr
) di
->txd32
-
702 di
->txdalign
), di
->txdalloc
,
703 (di
->txdpaorig
), &di
->tx_dmah
);
705 DMA_FREE_CONSISTENT(di
->osh
,
706 ((int8
*) (uintptr
) di
->rxd32
-
707 di
->rxdalign
), di
->rxdalloc
,
708 (di
->rxdpaorig
), &di
->rx_dmah
);
712 /* free packet pointer vectors */
714 MFREE(di
->osh
, (void *)di
->txp
, (di
->ntxd
* sizeof(void *)));
716 MFREE(di
->osh
, (void *)di
->rxp
, (di
->nrxd
* sizeof(void *)));
718 /* free tx packet DMA handles */
720 MFREE(di
->osh
, (void *)di
->txp_dmah
,
721 di
->ntxd
* sizeof(hnddma_seg_map_t
));
723 /* free rx packet DMA handles */
725 MFREE(di
->osh
, (void *)di
->rxp_dmah
,
726 di
->nrxd
* sizeof(hnddma_seg_map_t
));
728 /* free our private info structure */
729 MFREE(di
->osh
, (void *)di
, sizeof(dma_info_t
));
733 static bool _dma_descriptor_align(dma_info_t
* di
)
735 if (DMA64_ENAB(di
) && DMA64_MODE(di
)) {
738 /* Check to see if the descriptors need to be aligned on 4K/8K or not */
739 if (di
->d64txregs
!= NULL
) {
740 W_REG(di
->osh
, &di
->d64txregs
->addrlow
, 0xff0);
741 addrl
= R_REG(di
->osh
, &di
->d64txregs
->addrlow
);
744 } else if (di
->d64rxregs
!= NULL
) {
745 W_REG(di
->osh
, &di
->d64rxregs
->addrlow
, 0xff0);
746 addrl
= R_REG(di
->osh
, &di
->d64rxregs
->addrlow
);
754 /* return TRUE if this dma engine supports DmaExtendedAddrChanges, otherwise FALSE */
755 static bool _dma_isaddrext(dma_info_t
* di
)
757 if (DMA64_ENAB(di
) && DMA64_MODE(di
)) {
758 /* DMA64 supports full 32- or 64-bit operation. AE is always valid */
760 /* not all tx or rx channel are available */
761 if (di
->d64txregs
!= NULL
) {
762 if (!_dma64_addrext(di
->osh
, di
->d64txregs
)) {
763 DMA_ERROR(("%s: _dma_isaddrext: DMA64 tx doesn't have AE set\n", di
->name
));
767 } else if (di
->d64rxregs
!= NULL
) {
768 if (!_dma64_addrext(di
->osh
, di
->d64rxregs
)) {
769 DMA_ERROR(("%s: _dma_isaddrext: DMA64 rx doesn't have AE set\n", di
->name
));
775 } else if (DMA32_ENAB(di
)) {
777 return (_dma32_addrext(di
->osh
, di
->d32txregs
));
778 else if (di
->d32rxregs
)
779 return (_dma32_addrext(di
->osh
, di
->d32rxregs
));
786 /* initialize descriptor table base address */
787 static void _dma_ddtable_init(dma_info_t
* di
, uint direction
, dmaaddr_t pa
)
789 if (DMA64_ENAB(di
) && DMA64_MODE(di
)) {
790 if (!di
->aligndesc_4k
) {
791 if (direction
== DMA_TX
)
792 di
->xmtptrbase
= PHYSADDRLO(pa
);
794 di
->rcvptrbase
= PHYSADDRLO(pa
);
797 if ((di
->ddoffsetlow
== 0)
798 || !(PHYSADDRLO(pa
) & PCI32ADDR_HIGH
)) {
799 if (direction
== DMA_TX
) {
800 W_REG(di
->osh
, &di
->d64txregs
->addrlow
,
801 (PHYSADDRLO(pa
) + di
->ddoffsetlow
));
802 W_REG(di
->osh
, &di
->d64txregs
->addrhigh
,
803 (PHYSADDRHI(pa
) + di
->ddoffsethigh
));
805 W_REG(di
->osh
, &di
->d64rxregs
->addrlow
,
806 (PHYSADDRLO(pa
) + di
->ddoffsetlow
));
807 W_REG(di
->osh
, &di
->d64rxregs
->addrhigh
,
808 (PHYSADDRHI(pa
) + di
->ddoffsethigh
));
811 /* DMA64 32bits address extension */
814 ASSERT(PHYSADDRHI(pa
) == 0);
816 /* shift the high bit(s) from pa to ae */
817 ae
= (PHYSADDRLO(pa
) & PCI32ADDR_HIGH
) >>
818 PCI32ADDR_HIGH_SHIFT
;
819 PHYSADDRLO(pa
) &= ~PCI32ADDR_HIGH
;
821 if (direction
== DMA_TX
) {
822 W_REG(di
->osh
, &di
->d64txregs
->addrlow
,
823 (PHYSADDRLO(pa
) + di
->ddoffsetlow
));
824 W_REG(di
->osh
, &di
->d64txregs
->addrhigh
,
826 SET_REG(di
->osh
, &di
->d64txregs
->control
,
827 D64_XC_AE
, (ae
<< D64_XC_AE_SHIFT
));
829 W_REG(di
->osh
, &di
->d64rxregs
->addrlow
,
830 (PHYSADDRLO(pa
) + di
->ddoffsetlow
));
831 W_REG(di
->osh
, &di
->d64rxregs
->addrhigh
,
833 SET_REG(di
->osh
, &di
->d64rxregs
->control
,
834 D64_RC_AE
, (ae
<< D64_RC_AE_SHIFT
));
838 } else if (DMA32_ENAB(di
)) {
839 ASSERT(PHYSADDRHI(pa
) == 0);
840 if ((di
->ddoffsetlow
== 0)
841 || !(PHYSADDRLO(pa
) & PCI32ADDR_HIGH
)) {
842 if (direction
== DMA_TX
)
843 W_REG(di
->osh
, &di
->d32txregs
->addr
,
844 (PHYSADDRLO(pa
) + di
->ddoffsetlow
));
846 W_REG(di
->osh
, &di
->d32rxregs
->addr
,
847 (PHYSADDRLO(pa
) + di
->ddoffsetlow
));
849 /* dma32 address extension */
853 /* shift the high bit(s) from pa to ae */
854 ae
= (PHYSADDRLO(pa
) & PCI32ADDR_HIGH
) >>
855 PCI32ADDR_HIGH_SHIFT
;
856 PHYSADDRLO(pa
) &= ~PCI32ADDR_HIGH
;
858 if (direction
== DMA_TX
) {
859 W_REG(di
->osh
, &di
->d32txregs
->addr
,
860 (PHYSADDRLO(pa
) + di
->ddoffsetlow
));
861 SET_REG(di
->osh
, &di
->d32txregs
->control
, XC_AE
,
864 W_REG(di
->osh
, &di
->d32rxregs
->addr
,
865 (PHYSADDRLO(pa
) + di
->ddoffsetlow
));
866 SET_REG(di
->osh
, &di
->d32rxregs
->control
, RC_AE
,
874 static void _dma_fifoloopbackenable(dma_info_t
* di
)
876 DMA_TRACE(("%s: dma_fifoloopbackenable\n", di
->name
));
878 if (DMA64_ENAB(di
) && DMA64_MODE(di
))
879 OR_REG(di
->osh
, &di
->d64txregs
->control
, D64_XC_LE
);
880 else if (DMA32_ENAB(di
))
881 OR_REG(di
->osh
, &di
->d32txregs
->control
, XC_LE
);
886 static void _dma_rxinit(dma_info_t
* di
)
888 DMA_TRACE(("%s: dma_rxinit\n", di
->name
));
893 di
->rxin
= di
->rxout
= 0;
895 /* clear rx descriptor ring */
896 if (DMA64_ENAB(di
) && DMA64_MODE(di
)) {
897 BZERO_SM((void *)(uintptr
) di
->rxd64
,
898 (di
->nrxd
* sizeof(dma64dd_t
)));
900 /* DMA engine with out alignment requirement requires table to be inited
901 * before enabling the engine
903 if (!di
->aligndesc_4k
)
904 _dma_ddtable_init(di
, DMA_RX
, di
->rxdpa
);
908 if (di
->aligndesc_4k
)
909 _dma_ddtable_init(di
, DMA_RX
, di
->rxdpa
);
910 } else if (DMA32_ENAB(di
)) {
911 BZERO_SM((void *)(uintptr
) di
->rxd32
,
912 (di
->nrxd
* sizeof(dma32dd_t
)));
914 _dma_ddtable_init(di
, DMA_RX
, di
->rxdpa
);
919 static void _dma_rxenable(dma_info_t
* di
)
921 uint dmactrlflags
= di
->hnddma
.dmactrlflags
;
923 DMA_TRACE(("%s: dma_rxenable\n", di
->name
));
925 if (DMA64_ENAB(di
) && DMA64_MODE(di
)) {
927 (R_REG(di
->osh
, &di
->d64rxregs
->control
) & D64_RC_AE
) |
930 if ((dmactrlflags
& DMA_CTRL_PEN
) == 0)
931 control
|= D64_RC_PD
;
933 if (dmactrlflags
& DMA_CTRL_ROC
)
934 control
|= D64_RC_OC
;
936 W_REG(di
->osh
, &di
->d64rxregs
->control
,
937 ((di
->rxoffset
<< D64_RC_RO_SHIFT
) | control
));
938 } else if (DMA32_ENAB(di
)) {
940 (R_REG(di
->osh
, &di
->d32rxregs
->control
) & RC_AE
) | RC_RE
;
942 if ((dmactrlflags
& DMA_CTRL_PEN
) == 0)
945 if (dmactrlflags
& DMA_CTRL_ROC
)
948 W_REG(di
->osh
, &di
->d32rxregs
->control
,
949 ((di
->rxoffset
<< RC_RO_SHIFT
) | control
));
955 _dma_rx_param_get(dma_info_t
* di
, uint16
* rxoffset
, uint16
* rxbufsize
)
957 /* the normal values fit into 16 bits */
958 *rxoffset
= (uint16
) di
->rxoffset
;
959 *rxbufsize
= (uint16
) di
->rxbufsize
;
962 /* !! rx entry routine
963 * returns a pointer to the next frame received, or NULL if there are no more
964 * if DMA_CTRL_RXMULTI is defined, DMA scattering(multiple buffers) is supported
966 * otherwise, it's treated as giant pkt and will be tossed.
967 * The DMA scattering starts with normal DMA header, followed by first buffer data.
968 * After it reaches the max size of buffer, the data continues in next DMA descriptor
969 * buffer WITHOUT DMA header
971 static void *BCMFASTPATH
_dma_rx(dma_info_t
* di
)
973 void *p
, *head
, *tail
;
979 head
= _dma_getnextrxp(di
, FALSE
);
983 len
= ltoh16(*(uint16
*) (PKTDATA(head
)));
984 DMA_TRACE(("%s: dma_rx len %d\n", di
->name
, len
));
986 #if defined(__mips__)
988 while (!(len
= *(uint16
*) OSL_UNCACHED(PKTDATA(head
))))
991 *(uint16
*) PKTDATA(head
) = htol16((uint16
) len
);
993 #endif /* defined(__mips__) */
995 /* set actual length */
996 pkt_len
= MIN((di
->rxoffset
+ len
), di
->rxbufsize
);
997 PKTSETLEN(head
, pkt_len
);
998 resid
= len
- (di
->rxbufsize
- di
->rxoffset
);
1000 /* check for single or multi-buffer rx */
1003 while ((resid
> 0) && (p
= _dma_getnextrxp(di
, FALSE
))) {
1004 PKTSETNEXT(tail
, p
);
1005 pkt_len
= MIN(resid
, (int)di
->rxbufsize
);
1006 PKTSETLEN(p
, pkt_len
);
1009 resid
-= di
->rxbufsize
;
1016 cur
= (DMA64_ENAB(di
) && DMA64_MODE(di
)) ?
1017 B2I(((R_REG(di
->osh
, &di
->d64rxregs
->status0
) &
1019 di
->rcvptrbase
) & D64_RS0_CD_MASK
,
1020 dma64dd_t
) : B2I(R_REG(di
->osh
,
1022 status
) & RS_CD_MASK
,
1024 DMA_ERROR(("_dma_rx, rxin %d rxout %d, hw_curr %d\n",
1025 di
->rxin
, di
->rxout
, cur
));
1029 if ((di
->hnddma
.dmactrlflags
& DMA_CTRL_RXMULTI
) == 0) {
1030 DMA_ERROR(("%s: dma_rx: bad frame length (%d)\n",
1032 PKTFREE(di
->osh
, head
, FALSE
);
1033 di
->hnddma
.rxgiants
++;
1041 /* post receive buffers
1042 * return FALSE is refill failed completely and ring is empty
1043 * this will stall the rx dma and user might want to call rxfill again asap
1044 * This unlikely happens on memory-rich NIC, but often on memory-constrained dongle
1046 static bool BCMFASTPATH
_dma_rxfill(dma_info_t
* di
)
1054 uint extra_offset
= 0;
1060 * Determine how many receive buffers we're lacking
1061 * from the full complement, allocate, initialize,
1062 * and post them, then update the chip rx lastdscr.
1068 n
= di
->nrxpost
- NRXDACTIVE(rxin
, rxout
);
1070 DMA_TRACE(("%s: dma_rxfill: post %d\n", di
->name
, n
));
1072 if (di
->rxbufsize
> BCMEXTRAHDROOM
)
1073 extra_offset
= di
->rxextrahdrroom
;
1075 for (i
= 0; i
< n
; i
++) {
1076 /* the di->rxbufsize doesn't include the extra headroom, we need to add it to the
1077 size to be allocated
1080 p
= osl_pktget(di
->osh
, di
->rxbufsize
+ extra_offset
);
1083 DMA_ERROR(("%s: dma_rxfill: out of rxbufs\n",
1086 if (DMA64_ENAB(di
) && DMA64_MODE(di
)) {
1087 if (dma64_rxidle(di
)) {
1088 DMA_ERROR(("%s: rxfill64: ring is empty !\n", di
->name
));
1091 } else if (DMA32_ENAB(di
)) {
1092 if (dma32_rxidle(di
)) {
1093 DMA_ERROR(("%s: rxfill32: ring is empty !\n", di
->name
));
1099 di
->hnddma
.rxnobuf
++;
1102 /* reserve an extra headroom, if applicable */
1104 PKTPULL(p
, extra_offset
);
1106 /* Do a cached write instead of uncached write since DMA_MAP
1107 * will flush the cache.
1109 *(uint32
*) (PKTDATA(p
)) = 0;
1112 bzero(&di
->rxp_dmah
[rxout
], sizeof(hnddma_seg_map_t
));
1114 pa
= DMA_MAP(di
->osh
, PKTDATA(p
),
1115 di
->rxbufsize
, DMA_RX
, p
, &di
->rxp_dmah
[rxout
]);
1117 ASSERT(ISALIGNED(PHYSADDRLO(pa
), 4));
1119 /* save the free packet pointer */
1120 ASSERT(di
->rxp
[rxout
] == NULL
);
1123 /* reset flags for each descriptor */
1125 if (DMA64_ENAB(di
) && DMA64_MODE(di
)) {
1126 if (rxout
== (di
->nrxd
- 1))
1127 flags
= D64_CTRL1_EOT
;
1129 dma64_dd_upd(di
, di
->rxd64
, pa
, rxout
, &flags
,
1131 } else if (DMA32_ENAB(di
)) {
1132 if (rxout
== (di
->nrxd
- 1))
1135 ASSERT(PHYSADDRHI(pa
) == 0);
1136 dma32_dd_upd(di
, di
->rxd32
, pa
, rxout
, &flags
,
1140 rxout
= NEXTRXD(rxout
);
1145 /* update the chip lastdscr pointer */
1146 if (DMA64_ENAB(di
) && DMA64_MODE(di
)) {
1147 W_REG(di
->osh
, &di
->d64rxregs
->ptr
,
1148 di
->rcvptrbase
+ I2B(rxout
, dma64dd_t
));
1149 } else if (DMA32_ENAB(di
)) {
1150 W_REG(di
->osh
, &di
->d32rxregs
->ptr
, I2B(rxout
, dma32dd_t
));
1157 /* like getnexttxp but no reclaim */
1158 static void *_dma_peeknexttxp(dma_info_t
* di
)
1165 if (DMA64_ENAB(di
) && DMA64_MODE(di
)) {
1167 B2I(((R_REG(di
->osh
, &di
->d64txregs
->status0
) &
1168 D64_XS0_CD_MASK
) - di
->xmtptrbase
) & D64_XS0_CD_MASK
,
1170 } else if (DMA32_ENAB(di
)) {
1172 B2I(R_REG(di
->osh
, &di
->d32txregs
->status
) & XS_CD_MASK
,
1177 for (i
= di
->txin
; i
!= end
; i
= NEXTTXD(i
))
1179 return (di
->txp
[i
]);
1184 /* like getnextrxp but not take off the ring */
1185 static void *_dma_peeknextrxp(dma_info_t
* di
)
1192 if (DMA64_ENAB(di
) && DMA64_MODE(di
)) {
1194 B2I(((R_REG(di
->osh
, &di
->d64rxregs
->status0
) &
1195 D64_RS0_CD_MASK
) - di
->rcvptrbase
) & D64_RS0_CD_MASK
,
1197 } else if (DMA32_ENAB(di
)) {
1199 B2I(R_REG(di
->osh
, &di
->d32rxregs
->status
) & RS_CD_MASK
,
1204 for (i
= di
->rxin
; i
!= end
; i
= NEXTRXD(i
))
1206 return (di
->rxp
[i
]);
1211 static void _dma_rxreclaim(dma_info_t
* di
)
1215 /* "unused local" warning suppression for OSLs that
1216 * define PKTFREE() without using the di->osh arg
1220 DMA_TRACE(("%s: dma_rxreclaim\n", di
->name
));
1222 while ((p
= _dma_getnextrxp(di
, TRUE
)))
1223 PKTFREE(di
->osh
, p
, FALSE
);
1226 static void *BCMFASTPATH
_dma_getnextrxp(dma_info_t
* di
, bool forceall
)
1231 if (DMA64_ENAB(di
) && DMA64_MODE(di
)) {
1232 return dma64_getnextrxp(di
, forceall
);
1233 } else if (DMA32_ENAB(di
)) {
1234 return dma32_getnextrxp(di
, forceall
);
1239 static void _dma_txblock(dma_info_t
* di
)
1241 di
->hnddma
.txavail
= 0;
1244 static void _dma_txunblock(dma_info_t
* di
)
1246 di
->hnddma
.txavail
= di
->ntxd
- NTXDACTIVE(di
->txin
, di
->txout
) - 1;
1249 static uint
_dma_txactive(dma_info_t
* di
)
1251 return NTXDACTIVE(di
->txin
, di
->txout
);
1254 static uint
_dma_txpending(dma_info_t
* di
)
1258 if (DMA64_ENAB(di
) && DMA64_MODE(di
)) {
1260 B2I(((R_REG(di
->osh
, &di
->d64txregs
->status0
) &
1261 D64_XS0_CD_MASK
) - di
->xmtptrbase
) & D64_XS0_CD_MASK
,
1263 } else if (DMA32_ENAB(di
)) {
1265 B2I(R_REG(di
->osh
, &di
->d32txregs
->status
) & XS_CD_MASK
,
1270 return NTXDACTIVE(curr
, di
->txout
);
1273 static uint
_dma_txcommitted(dma_info_t
* di
)
1276 uint txin
= di
->txin
;
1278 if (txin
== di
->txout
)
1281 if (DMA64_ENAB(di
) && DMA64_MODE(di
)) {
1282 ptr
= B2I(R_REG(di
->osh
, &di
->d64txregs
->ptr
), dma64dd_t
);
1283 } else if (DMA32_ENAB(di
)) {
1284 ptr
= B2I(R_REG(di
->osh
, &di
->d32txregs
->ptr
), dma32dd_t
);
1288 return NTXDACTIVE(di
->txin
, ptr
);
1291 static uint
_dma_rxactive(dma_info_t
* di
)
1293 return NRXDACTIVE(di
->rxin
, di
->rxout
);
1296 static void _dma_counterreset(dma_info_t
* di
)
1298 /* reset all software counter */
1299 di
->hnddma
.rxgiants
= 0;
1300 di
->hnddma
.rxnobuf
= 0;
1301 di
->hnddma
.txnobuf
= 0;
1304 static uint
_dma_ctrlflags(dma_info_t
* di
, uint mask
, uint flags
)
1306 uint dmactrlflags
= di
->hnddma
.dmactrlflags
;
1309 DMA_ERROR(("%s: _dma_ctrlflags: NULL dma handle\n", di
->name
));
1313 ASSERT((flags
& ~mask
) == 0);
1315 dmactrlflags
&= ~mask
;
1316 dmactrlflags
|= flags
;
1318 /* If trying to enable parity, check if parity is actually supported */
1319 if (dmactrlflags
& DMA_CTRL_PEN
) {
1322 if (DMA64_ENAB(di
) && DMA64_MODE(di
)) {
1323 control
= R_REG(di
->osh
, &di
->d64txregs
->control
);
1324 W_REG(di
->osh
, &di
->d64txregs
->control
,
1325 control
| D64_XC_PD
);
1326 if (R_REG(di
->osh
, &di
->d64txregs
->control
) & D64_XC_PD
) {
1327 /* We *can* disable it so it is supported,
1328 * restore control register
1330 W_REG(di
->osh
, &di
->d64txregs
->control
,
1333 /* Not supported, don't allow it to be enabled */
1334 dmactrlflags
&= ~DMA_CTRL_PEN
;
1336 } else if (DMA32_ENAB(di
)) {
1337 control
= R_REG(di
->osh
, &di
->d32txregs
->control
);
1338 W_REG(di
->osh
, &di
->d32txregs
->control
,
1340 if (R_REG(di
->osh
, &di
->d32txregs
->control
) & XC_PD
) {
1341 W_REG(di
->osh
, &di
->d32txregs
->control
,
1344 /* Not supported, don't allow it to be enabled */
1345 dmactrlflags
&= ~DMA_CTRL_PEN
;
1351 di
->hnddma
.dmactrlflags
= dmactrlflags
;
1353 return (dmactrlflags
);
1356 /* get the address of the var in order to change later */
1357 static uintptr
_dma_getvar(dma_info_t
* di
, const char *name
)
1359 if (!strcmp(name
, "&txavail"))
1360 return ((uintptr
) & (di
->hnddma
.txavail
));
1367 void dma_txpioloopback(osl_t
* osh
, dma32regs_t
* regs
)
1369 OR_REG(osh
, ®s
->control
, XC_LE
);
1373 uint8
dma_align_sizetobits(uint size
)
1377 ASSERT(!(size
& (size
- 1)));
1378 while (size
>>= 1) {
1384 /* This function ensures that the DMA descriptor ring will not get allocated
1385 * across Page boundary. If the allocation is done across the page boundary
1386 * at the first time, then it is freed and the allocation is done at
1387 * descriptor ring size aligned location. This will ensure that the ring will
1388 * not cross page boundary
1390 static void *dma_ringalloc(osl_t
* osh
, uint32 boundary
, uint size
,
1391 uint16
* alignbits
, uint
* alloced
,
1392 dmaaddr_t
* descpa
, osldma_t
** dmah
)
1395 uint32 desc_strtaddr
;
1396 uint32 alignbytes
= 1 << *alignbits
;
1400 DMA_ALLOC_CONSISTENT(osh
, size
, *alignbits
, alloced
, descpa
,
1404 desc_strtaddr
= (uint32
) ROUNDUP((uintptr
) va
, alignbytes
);
1405 if (((desc_strtaddr
+ size
- 1) & boundary
) != (desc_strtaddr
1407 *alignbits
= dma_align_sizetobits(size
);
1408 DMA_FREE_CONSISTENT(osh
, va
, size
, *descpa
, dmah
);
1409 va
= DMA_ALLOC_CONSISTENT(osh
, size
, *alignbits
, alloced
,
1415 /* 32-bit DMA functions */
1417 static void dma32_txinit(dma_info_t
* di
)
1419 uint32 control
= XC_XE
;
1421 DMA_TRACE(("%s: dma_txinit\n", di
->name
));
1426 di
->txin
= di
->txout
= 0;
1427 di
->hnddma
.txavail
= di
->ntxd
- 1;
1429 /* clear tx descriptor ring */
1430 BZERO_SM((void *)(uintptr
) di
->txd32
, (di
->ntxd
* sizeof(dma32dd_t
)));
1432 if ((di
->hnddma
.dmactrlflags
& DMA_CTRL_PEN
) == 0)
1434 W_REG(di
->osh
, &di
->d32txregs
->control
, control
);
1435 _dma_ddtable_init(di
, DMA_TX
, di
->txdpa
);
1438 static bool dma32_txenabled(dma_info_t
* di
)
1442 /* If the chip is dead, it is not enabled :-) */
1443 xc
= R_REG(di
->osh
, &di
->d32txregs
->control
);
1444 return ((xc
!= 0xffffffff) && (xc
& XC_XE
));
1447 static void dma32_txsuspend(dma_info_t
* di
)
1449 DMA_TRACE(("%s: dma_txsuspend\n", di
->name
));
1454 OR_REG(di
->osh
, &di
->d32txregs
->control
, XC_SE
);
1457 static void dma32_txresume(dma_info_t
* di
)
1459 DMA_TRACE(("%s: dma_txresume\n", di
->name
));
1464 AND_REG(di
->osh
, &di
->d32txregs
->control
, ~XC_SE
);
1467 static bool dma32_txsuspended(dma_info_t
* di
)
1469 return (di
->ntxd
== 0)
1470 || ((R_REG(di
->osh
, &di
->d32txregs
->control
) & XC_SE
) == XC_SE
);
1473 static void dma32_txreclaim(dma_info_t
* di
, txd_range_t range
)
1477 DMA_TRACE(("%s: dma_txreclaim %s\n", di
->name
,
1478 (range
== HNDDMA_RANGE_ALL
) ? "all" :
1480 HNDDMA_RANGE_TRANSMITTED
) ? "transmitted" :
1483 if (di
->txin
== di
->txout
)
1486 while ((p
= dma32_getnexttxp(di
, range
)))
1487 PKTFREE(di
->osh
, p
, TRUE
);
1490 static bool dma32_txstopped(dma_info_t
* di
)
1492 return ((R_REG(di
->osh
, &di
->d32txregs
->status
) & XS_XS_MASK
) ==
1496 static bool dma32_rxstopped(dma_info_t
* di
)
1498 return ((R_REG(di
->osh
, &di
->d32rxregs
->status
) & RS_RS_MASK
) ==
1502 static bool dma32_alloc(dma_info_t
* di
, uint direction
)
1511 ddlen
= sizeof(dma32dd_t
);
1513 size
= (direction
== DMA_TX
) ? (di
->ntxd
* ddlen
) : (di
->nrxd
* ddlen
);
1516 align_bits
= di
->dmadesc_align
;
1517 align
= (1 << align_bits
);
1519 if (direction
== DMA_TX
) {
1521 dma_ringalloc(di
->osh
, D32RINGALIGN
, size
, &align_bits
,
1522 &alloced
, &di
->txdpaorig
,
1523 &di
->tx_dmah
)) == NULL
) {
1524 DMA_ERROR(("%s: dma_alloc: DMA_ALLOC_CONSISTENT(ntxd) failed\n", di
->name
));
1528 PHYSADDRHISET(di
->txdpa
, 0);
1529 ASSERT(PHYSADDRHI(di
->txdpaorig
) == 0);
1530 di
->txd32
= (dma32dd_t
*) ROUNDUP((uintptr
) va
, align
);
1532 (uint
) ((int8
*) (uintptr
) di
->txd32
- (int8
*) va
);
1534 PHYSADDRLOSET(di
->txdpa
,
1535 PHYSADDRLO(di
->txdpaorig
) + di
->txdalign
);
1536 /* Make sure that alignment didn't overflow */
1537 ASSERT(PHYSADDRLO(di
->txdpa
) >= PHYSADDRLO(di
->txdpaorig
));
1539 di
->txdalloc
= alloced
;
1540 ASSERT(ISALIGNED((uintptr
) di
->txd32
, align
));
1543 dma_ringalloc(di
->osh
, D32RINGALIGN
, size
, &align_bits
,
1544 &alloced
, &di
->rxdpaorig
,
1545 &di
->rx_dmah
)) == NULL
) {
1546 DMA_ERROR(("%s: dma_alloc: DMA_ALLOC_CONSISTENT(nrxd) failed\n", di
->name
));
1550 PHYSADDRHISET(di
->rxdpa
, 0);
1551 ASSERT(PHYSADDRHI(di
->rxdpaorig
) == 0);
1552 di
->rxd32
= (dma32dd_t
*) ROUNDUP((uintptr
) va
, align
);
1554 (uint
) ((int8
*) (uintptr
) di
->rxd32
- (int8
*) va
);
1556 PHYSADDRLOSET(di
->rxdpa
,
1557 PHYSADDRLO(di
->rxdpaorig
) + di
->rxdalign
);
1558 /* Make sure that alignment didn't overflow */
1559 ASSERT(PHYSADDRLO(di
->rxdpa
) >= PHYSADDRLO(di
->rxdpaorig
));
1560 di
->rxdalloc
= alloced
;
1561 ASSERT(ISALIGNED((uintptr
) di
->rxd32
, align
));
1567 static bool dma32_txreset(dma_info_t
* di
)
1574 /* suspend tx DMA first */
1575 W_REG(di
->osh
, &di
->d32txregs
->control
, XC_SE
);
1577 (R_REG(di
->osh
, &di
->d32txregs
->status
) & XS_XS_MASK
))
1578 != XS_XS_DISABLED
) && (status
!= XS_XS_IDLE
)
1579 && (status
!= XS_XS_STOPPED
), (10000));
1581 W_REG(di
->osh
, &di
->d32txregs
->control
, 0);
1582 SPINWAIT(((status
= (R_REG(di
->osh
,
1583 &di
->d32txregs
->status
) & XS_XS_MASK
)) !=
1584 XS_XS_DISABLED
), 10000);
1586 /* wait for the last transaction to complete */
1589 return (status
== XS_XS_DISABLED
);
1592 static bool dma32_rxidle(dma_info_t
* di
)
1594 DMA_TRACE(("%s: dma_rxidle\n", di
->name
));
1599 return ((R_REG(di
->osh
, &di
->d32rxregs
->status
) & RS_CD_MASK
) ==
1600 R_REG(di
->osh
, &di
->d32rxregs
->ptr
));
1603 static bool dma32_rxreset(dma_info_t
* di
)
1610 W_REG(di
->osh
, &di
->d32rxregs
->control
, 0);
1611 SPINWAIT(((status
= (R_REG(di
->osh
,
1612 &di
->d32rxregs
->status
) & RS_RS_MASK
)) !=
1613 RS_RS_DISABLED
), 10000);
1615 return (status
== RS_RS_DISABLED
);
1618 static bool dma32_rxenabled(dma_info_t
* di
)
1622 rc
= R_REG(di
->osh
, &di
->d32rxregs
->control
);
1623 return ((rc
!= 0xffffffff) && (rc
& RC_RE
));
1626 static bool dma32_txsuspendedidle(dma_info_t
* di
)
1631 if (!(R_REG(di
->osh
, &di
->d32txregs
->control
) & XC_SE
))
1634 if ((R_REG(di
->osh
, &di
->d32txregs
->status
) & XS_XS_MASK
) != XS_XS_IDLE
)
1638 return ((R_REG(di
->osh
, &di
->d32txregs
->status
) & XS_XS_MASK
) ==
1642 /* !! tx entry routine
1643 * supports full 32bit dma engine buffer addressing so
1644 * dma buffers can cross 4 Kbyte page boundaries.
1646 * WARNING: call must check the return value for error.
1647 * the error(toss frames) could be fatal and cause many subsequent hard to debug problems
1649 static int dma32_txfast(dma_info_t
* di
, void *p0
, bool commit
)
1658 DMA_TRACE(("%s: dma_txfast\n", di
->name
));
1663 * Walk the chain of packet buffers
1664 * allocating and initializing transmit descriptor entries.
1666 for (p
= p0
; p
; p
= next
) {
1668 hnddma_seg_map_t
*map
;
1673 len
+= PKTDMAPAD(di
->osh
, p
);
1677 /* return nonzero if out of tx descriptors */
1678 if (NEXTTXD(txout
) == di
->txin
)
1685 bzero(&di
->txp_dmah
[txout
], sizeof(hnddma_seg_map_t
));
1687 /* get physical address of buffer start */
1688 pa
= DMA_MAP(di
->osh
, data
, len
, DMA_TX
, p
,
1689 &di
->txp_dmah
[txout
]);
1691 if (DMASGLIST_ENAB
) {
1692 map
= &di
->txp_dmah
[txout
];
1694 /* See if all the segments can be accounted for */
1696 (uint
) (di
->ntxd
- NTXDACTIVE(di
->txin
, di
->txout
) -
1704 for (j
= 1; j
<= nsegs
; j
++) {
1706 if (p
== p0
&& j
== 1)
1709 /* With a DMA segment list, Descriptor table is filled
1710 * using the segment list instead of looping over
1711 * buffers in multi-chain DMA. Therefore, EOF for SGLIST is when
1712 * end of segment list is reached.
1714 if ((!DMASGLIST_ENAB
&& next
== NULL
) ||
1715 (DMASGLIST_ENAB
&& j
== nsegs
))
1716 flags
|= (CTRL_IOC
| CTRL_EOF
);
1717 if (txout
== (di
->ntxd
- 1))
1720 if (DMASGLIST_ENAB
) {
1721 len
= map
->segs
[j
- 1].length
;
1722 pa
= map
->segs
[j
- 1].addr
;
1724 ASSERT(PHYSADDRHI(pa
) == 0);
1726 dma32_dd_upd(di
, di
->txd32
, pa
, txout
, &flags
, len
);
1727 ASSERT(di
->txp
[txout
] == NULL
);
1729 txout
= NEXTTXD(txout
);
1732 /* See above. No need to loop over individual buffers */
1737 /* if last txd eof not set, fix it */
1738 if (!(flags
& CTRL_EOF
))
1739 W_SM(&di
->txd32
[PREVTXD(txout
)].ctrl
,
1740 BUS_SWAP32(flags
| CTRL_IOC
| CTRL_EOF
));
1742 /* save the packet */
1743 di
->txp
[PREVTXD(txout
)] = p0
;
1745 /* bump the tx descriptor index */
1750 W_REG(di
->osh
, &di
->d32txregs
->ptr
, I2B(txout
, dma32dd_t
));
1752 /* tx flow control */
1753 di
->hnddma
.txavail
= di
->ntxd
- NTXDACTIVE(di
->txin
, di
->txout
) - 1;
1758 DMA_ERROR(("%s: dma_txfast: out of txds\n", di
->name
));
1759 PKTFREE(di
->osh
, p0
, TRUE
);
1760 di
->hnddma
.txavail
= 0;
1761 di
->hnddma
.txnobuf
++;
1766 * Reclaim next completed txd (txds if using chained buffers) in the range
1767 * specified and return associated packet.
1768 * If range is HNDDMA_RANGE_TRANSMITTED, reclaim descriptors that have be
1769 * transmitted as noted by the hardware "CurrDescr" pointer.
1770 * If range is HNDDMA_RANGE_TRANSFERED, reclaim descriptors that have be
1771 * transfered by the DMA as noted by the hardware "ActiveDescr" pointer.
1772 * If range is HNDDMA_RANGE_ALL, reclaim all txd(s) posted to the ring and
1773 * return associated packet regardless of the value of hardware pointers.
1775 static void *dma32_getnexttxp(dma_info_t
* di
, txd_range_t range
)
1777 uint16 start
, end
, i
;
1781 DMA_TRACE(("%s: dma_getnexttxp %s\n", di
->name
,
1782 (range
== HNDDMA_RANGE_ALL
) ? "all" :
1784 HNDDMA_RANGE_TRANSMITTED
) ? "transmitted" :
1793 if (range
== HNDDMA_RANGE_ALL
)
1796 dma32regs_t
*dregs
= di
->d32txregs
;
1799 (uint16
) B2I(R_REG(di
->osh
, &dregs
->status
) & XS_CD_MASK
,
1802 if (range
== HNDDMA_RANGE_TRANSFERED
) {
1804 (uint16
) ((R_REG(di
->osh
, &dregs
->status
) &
1805 XS_AD_MASK
) >> XS_AD_SHIFT
);
1806 active_desc
= (uint16
) B2I(active_desc
, dma32dd_t
);
1807 if (end
!= active_desc
)
1808 end
= PREVTXD(active_desc
);
1812 if ((start
== 0) && (end
> di
->txout
))
1815 for (i
= start
; i
!= end
&& !txp
; i
= NEXTTXD(i
)) {
1817 hnddma_seg_map_t
*map
= NULL
;
1818 uint size
, j
, nsegs
;
1821 (BUS_SWAP32(R_SM(&di
->txd32
[i
].addr
)) -
1822 di
->dataoffsetlow
));
1823 PHYSADDRHISET(pa
, 0);
1825 if (DMASGLIST_ENAB
) {
1826 map
= &di
->txp_dmah
[i
];
1827 size
= map
->origsize
;
1831 (BUS_SWAP32(R_SM(&di
->txd32
[i
].ctrl
)) &
1836 for (j
= nsegs
; j
> 0; j
--) {
1837 W_SM(&di
->txd32
[i
].addr
, 0xdeadbeef);
1845 DMA_UNMAP(di
->osh
, pa
, size
, DMA_TX
, txp
, map
);
1850 /* tx flow control */
1851 di
->hnddma
.txavail
= di
->ntxd
- NTXDACTIVE(di
->txin
, di
->txout
) - 1;
1856 DMA_NONE(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n", start
, end
, di
->txout
, forceall
));
1860 static void *dma32_getnextrxp(dma_info_t
* di
, bool forceall
)
1865 /* if forcing, dma engine must be disabled */
1866 ASSERT(!forceall
|| !dma32_rxenabled(di
));
1870 /* return if no packets posted */
1875 B2I(R_REG(di
->osh
, &di
->d32rxregs
->status
) & RS_CD_MASK
, dma32dd_t
);
1877 /* ignore curr if forceall */
1878 if (!forceall
&& (i
== curr
))
1881 /* get the packet pointer that corresponds to the rx descriptor */
1887 (BUS_SWAP32(R_SM(&di
->rxd32
[i
].addr
)) -
1888 di
->dataoffsetlow
));
1889 PHYSADDRHISET(pa
, 0);
1891 /* clear this packet from the descriptor ring */
1892 DMA_UNMAP(di
->osh
, pa
, di
->rxbufsize
, DMA_RX
, rxp
, &di
->rxp_dmah
[i
]);
1894 W_SM(&di
->rxd32
[i
].addr
, 0xdeadbeef);
1896 di
->rxin
= NEXTRXD(i
);
1902 * Rotate all active tx dma ring entries "forward" by (ActiveDescriptor - txin).
1904 static void dma32_txrotate(dma_info_t
* di
)
1913 ASSERT(dma32_txsuspendedidle(di
));
1915 nactive
= _dma_txactive(di
);
1917 (((R_REG(di
->osh
, &di
->d32txregs
->status
) & XS_AD_MASK
)
1918 >> XS_AD_SHIFT
), dma32dd_t
));
1919 rot
= TXD(ad
- di
->txin
);
1921 ASSERT(rot
< di
->ntxd
);
1923 /* full-ring case is a lot harder - don't worry about this */
1924 if (rot
>= (di
->ntxd
- nactive
)) {
1925 DMA_ERROR(("%s: dma_txrotate: ring full - punt\n", di
->name
));
1930 last
= PREVTXD(di
->txout
);
1932 /* move entries starting at last and moving backwards to first */
1933 for (old
= last
; old
!= PREVTXD(first
); old
= PREVTXD(old
)) {
1934 new = TXD(old
+ rot
);
1937 * Move the tx dma descriptor.
1938 * EOT is set only in the last entry in the ring.
1940 w
= BUS_SWAP32(R_SM(&di
->txd32
[old
].ctrl
)) & ~CTRL_EOT
;
1941 if (new == (di
->ntxd
- 1))
1943 W_SM(&di
->txd32
[new].ctrl
, BUS_SWAP32(w
));
1944 W_SM(&di
->txd32
[new].addr
, R_SM(&di
->txd32
[old
].addr
));
1946 /* zap the old tx dma descriptor address field */
1947 W_SM(&di
->txd32
[old
].addr
, BUS_SWAP32(0xdeadbeef));
1949 /* move the corresponding txp[] entry */
1950 ASSERT(di
->txp
[new] == NULL
);
1951 di
->txp
[new] = di
->txp
[old
];
1953 /* Move the segment map as well */
1954 if (DMASGLIST_ENAB
) {
1955 bcopy(&di
->txp_dmah
[old
], &di
->txp_dmah
[new],
1956 sizeof(hnddma_seg_map_t
));
1957 bzero(&di
->txp_dmah
[old
], sizeof(hnddma_seg_map_t
));
1960 di
->txp
[old
] = NULL
;
1963 /* update txin and txout */
1965 di
->txout
= TXD(di
->txout
+ rot
);
1966 di
->hnddma
.txavail
= di
->ntxd
- NTXDACTIVE(di
->txin
, di
->txout
) - 1;
1969 W_REG(di
->osh
, &di
->d32txregs
->ptr
, I2B(di
->txout
, dma32dd_t
));
1972 /* 64-bit DMA functions */
1974 static void dma64_txinit(dma_info_t
* di
)
1976 uint32 control
= D64_XC_XE
;
1978 DMA_TRACE(("%s: dma_txinit\n", di
->name
));
1983 di
->txin
= di
->txout
= 0;
1984 di
->hnddma
.txavail
= di
->ntxd
- 1;
1986 /* clear tx descriptor ring */
1987 BZERO_SM((void *)(uintptr
) di
->txd64
, (di
->ntxd
* sizeof(dma64dd_t
)));
1989 /* DMA engine with out alignment requirement requires table to be inited
1990 * before enabling the engine
1992 if (!di
->aligndesc_4k
)
1993 _dma_ddtable_init(di
, DMA_TX
, di
->txdpa
);
1995 if ((di
->hnddma
.dmactrlflags
& DMA_CTRL_PEN
) == 0)
1996 control
|= D64_XC_PD
;
1997 OR_REG(di
->osh
, &di
->d64txregs
->control
, control
);
1999 /* DMA engine with alignment requirement requires table to be inited
2000 * before enabling the engine
2002 if (di
->aligndesc_4k
)
2003 _dma_ddtable_init(di
, DMA_TX
, di
->txdpa
);
2006 static bool dma64_txenabled(dma_info_t
* di
)
2010 /* If the chip is dead, it is not enabled :-) */
2011 xc
= R_REG(di
->osh
, &di
->d64txregs
->control
);
2012 return ((xc
!= 0xffffffff) && (xc
& D64_XC_XE
));
2015 static void dma64_txsuspend(dma_info_t
* di
)
2017 DMA_TRACE(("%s: dma_txsuspend\n", di
->name
));
2022 OR_REG(di
->osh
, &di
->d64txregs
->control
, D64_XC_SE
);
2025 static void dma64_txresume(dma_info_t
* di
)
2027 DMA_TRACE(("%s: dma_txresume\n", di
->name
));
2032 AND_REG(di
->osh
, &di
->d64txregs
->control
, ~D64_XC_SE
);
2035 static bool dma64_txsuspended(dma_info_t
* di
)
2037 return (di
->ntxd
== 0) ||
2038 ((R_REG(di
->osh
, &di
->d64txregs
->control
) & D64_XC_SE
) ==
2042 static void BCMFASTPATH
dma64_txreclaim(dma_info_t
* di
, txd_range_t range
)
2046 DMA_TRACE(("%s: dma_txreclaim %s\n", di
->name
,
2047 (range
== HNDDMA_RANGE_ALL
) ? "all" :
2049 HNDDMA_RANGE_TRANSMITTED
) ? "transmitted" :
2052 if (di
->txin
== di
->txout
)
2055 while ((p
= dma64_getnexttxp(di
, range
))) {
2056 /* For unframed data, we don't have any packets to free */
2057 if (!(di
->hnddma
.dmactrlflags
& DMA_CTRL_UNFRAMED
))
2058 PKTFREE(di
->osh
, p
, TRUE
);
2062 static bool dma64_txstopped(dma_info_t
* di
)
2064 return ((R_REG(di
->osh
, &di
->d64txregs
->status0
) & D64_XS0_XS_MASK
) ==
2065 D64_XS0_XS_STOPPED
);
2068 static bool dma64_rxstopped(dma_info_t
* di
)
2070 return ((R_REG(di
->osh
, &di
->d64rxregs
->status0
) & D64_RS0_RS_MASK
) ==
2071 D64_RS0_RS_STOPPED
);
2074 static bool dma64_alloc(dma_info_t
* di
, uint direction
)
2083 ddlen
= sizeof(dma64dd_t
);
2085 size
= (direction
== DMA_TX
) ? (di
->ntxd
* ddlen
) : (di
->nrxd
* ddlen
);
2086 align_bits
= di
->dmadesc_align
;
2087 align
= (1 << align_bits
);
2089 if (direction
== DMA_TX
) {
2091 dma_ringalloc(di
->osh
, D64RINGALIGN
, size
, &align_bits
,
2092 &alloced
, &di
->txdpaorig
,
2093 &di
->tx_dmah
)) == NULL
) {
2094 DMA_ERROR(("%s: dma64_alloc: DMA_ALLOC_CONSISTENT(ntxd) failed\n", di
->name
));
2097 align
= (1 << align_bits
);
2098 di
->txd64
= (dma64dd_t
*) ROUNDUP((uintptr
) va
, align
);
2100 (uint
) ((int8
*) (uintptr
) di
->txd64
- (int8
*) va
);
2101 PHYSADDRLOSET(di
->txdpa
,
2102 PHYSADDRLO(di
->txdpaorig
) + di
->txdalign
);
2103 /* Make sure that alignment didn't overflow */
2104 ASSERT(PHYSADDRLO(di
->txdpa
) >= PHYSADDRLO(di
->txdpaorig
));
2106 PHYSADDRHISET(di
->txdpa
, PHYSADDRHI(di
->txdpaorig
));
2107 di
->txdalloc
= alloced
;
2108 ASSERT(ISALIGNED((uintptr
) di
->txd64
, align
));
2111 dma_ringalloc(di
->osh
, D64RINGALIGN
, size
, &align_bits
,
2112 &alloced
, &di
->rxdpaorig
,
2113 &di
->rx_dmah
)) == NULL
) {
2114 DMA_ERROR(("%s: dma64_alloc: DMA_ALLOC_CONSISTENT(nrxd) failed\n", di
->name
));
2117 align
= (1 << align_bits
);
2118 di
->rxd64
= (dma64dd_t
*) ROUNDUP((uintptr
) va
, align
);
2120 (uint
) ((int8
*) (uintptr
) di
->rxd64
- (int8
*) va
);
2121 PHYSADDRLOSET(di
->rxdpa
,
2122 PHYSADDRLO(di
->rxdpaorig
) + di
->rxdalign
);
2123 /* Make sure that alignment didn't overflow */
2124 ASSERT(PHYSADDRLO(di
->rxdpa
) >= PHYSADDRLO(di
->rxdpaorig
));
2126 PHYSADDRHISET(di
->rxdpa
, PHYSADDRHI(di
->rxdpaorig
));
2127 di
->rxdalloc
= alloced
;
2128 ASSERT(ISALIGNED((uintptr
) di
->rxd64
, align
));
2134 static bool dma64_txreset(dma_info_t
* di
)
2141 /* suspend tx DMA first */
2142 W_REG(di
->osh
, &di
->d64txregs
->control
, D64_XC_SE
);
2144 (R_REG(di
->osh
, &di
->d64txregs
->status0
) & D64_XS0_XS_MASK
))
2145 != D64_XS0_XS_DISABLED
) && (status
!= D64_XS0_XS_IDLE
)
2146 && (status
!= D64_XS0_XS_STOPPED
), 10000);
2148 W_REG(di
->osh
, &di
->d64txregs
->control
, 0);
2150 (R_REG(di
->osh
, &di
->d64txregs
->status0
) & D64_XS0_XS_MASK
))
2151 != D64_XS0_XS_DISABLED
), 10000);
2153 /* wait for the last transaction to complete */
2156 return (status
== D64_XS0_XS_DISABLED
);
2159 static bool dma64_rxidle(dma_info_t
* di
)
2161 DMA_TRACE(("%s: dma_rxidle\n", di
->name
));
2166 return ((R_REG(di
->osh
, &di
->d64rxregs
->status0
) & D64_RS0_CD_MASK
) ==
2167 (R_REG(di
->osh
, &di
->d64rxregs
->ptr
) & D64_RS0_CD_MASK
));
2170 static bool dma64_rxreset(dma_info_t
* di
)
2177 W_REG(di
->osh
, &di
->d64rxregs
->control
, 0);
2179 (R_REG(di
->osh
, &di
->d64rxregs
->status0
) & D64_RS0_RS_MASK
))
2180 != D64_RS0_RS_DISABLED
), 10000);
2182 return (status
== D64_RS0_RS_DISABLED
);
2185 static bool dma64_rxenabled(dma_info_t
* di
)
2189 rc
= R_REG(di
->osh
, &di
->d64rxregs
->control
);
2190 return ((rc
!= 0xffffffff) && (rc
& D64_RC_RE
));
2193 static bool dma64_txsuspendedidle(dma_info_t
* di
)
2199 if (!(R_REG(di
->osh
, &di
->d64txregs
->control
) & D64_XC_SE
))
2202 if ((R_REG(di
->osh
, &di
->d64txregs
->status0
) & D64_XS0_XS_MASK
) ==
2209 /* Useful when sending unframed data. This allows us to get a progress report from the DMA.
2210 * We return a pointer to the beginning of the DATA buffer of the current descriptor.
2211 * If DMA is idle, we return NULL.
2213 static void *dma64_getpos(dma_info_t
* di
, bool direction
)
2219 if (direction
== DMA_TX
) {
2221 R_REG(di
->osh
, &di
->d64txregs
->status0
) & D64_XS0_CD_MASK
;
2222 idle
= !NTXDACTIVE(di
->txin
, di
->txout
);
2223 va
= di
->txp
[B2I(cd_offset
, dma64dd_t
)];
2226 R_REG(di
->osh
, &di
->d64rxregs
->status0
) & D64_XS0_CD_MASK
;
2227 idle
= !NRXDACTIVE(di
->rxin
, di
->rxout
);
2228 va
= di
->rxp
[B2I(cd_offset
, dma64dd_t
)];
2231 /* If DMA is IDLE, return NULL */
2233 DMA_TRACE(("%s: DMA idle, return NULL\n", __func__
));
2240 /* TX of unframed data
2242 * Adds a DMA ring descriptor for the data pointed to by "buf".
2243 * This is for DMA of a buffer of data and is unlike other hnddma TX functions
2244 * that take a pointer to a "packet"
2245 * Each call to this is results in a single descriptor being added for "len" bytes of
2246 * data starting at "buf", it doesn't handle chained buffers.
2248 static int dma64_txunframed(dma_info_t
* di
, void *buf
, uint len
, bool commit
)
2252 dmaaddr_t pa
; /* phys addr */
2256 /* return nonzero if out of tx descriptors */
2257 if (NEXTTXD(txout
) == di
->txin
)
2263 pa
= DMA_MAP(di
->osh
, buf
, len
, DMA_TX
, NULL
, &di
->txp_dmah
[txout
]);
2265 flags
= (D64_CTRL1_SOF
| D64_CTRL1_IOC
| D64_CTRL1_EOF
);
2267 if (txout
== (di
->ntxd
- 1))
2268 flags
|= D64_CTRL1_EOT
;
2270 dma64_dd_upd(di
, di
->txd64
, pa
, txout
, &flags
, len
);
2271 ASSERT(di
->txp
[txout
] == NULL
);
2273 /* save the buffer pointer - used by dma_getpos */
2274 di
->txp
[txout
] = buf
;
2276 txout
= NEXTTXD(txout
);
2277 /* bump the tx descriptor index */
2282 W_REG(di
->osh
, &di
->d64txregs
->ptr
,
2283 di
->xmtptrbase
+ I2B(txout
, dma64dd_t
));
2286 /* tx flow control */
2287 di
->hnddma
.txavail
= di
->ntxd
- NTXDACTIVE(di
->txin
, di
->txout
) - 1;
2292 DMA_ERROR(("%s: %s: out of txds !!!\n", di
->name
, __func__
));
2293 di
->hnddma
.txavail
= 0;
2294 di
->hnddma
.txnobuf
++;
2298 /* !! tx entry routine
2299 * WARNING: call must check the return value for error.
2300 * the error(toss frames) could be fatal and cause many subsequent hard to debug problems
2302 static int BCMFASTPATH
dma64_txfast(dma_info_t
* di
, void *p0
, bool commit
)
2311 DMA_TRACE(("%s: dma_txfast\n", di
->name
));
2316 * Walk the chain of packet buffers
2317 * allocating and initializing transmit descriptor entries.
2319 for (p
= p0
; p
; p
= next
) {
2321 hnddma_seg_map_t
*map
;
2326 len
+= PKTDMAPAD(di
->osh
, p
);
2327 #endif /* BCM_DMAPAD */
2330 /* return nonzero if out of tx descriptors */
2331 if (NEXTTXD(txout
) == di
->txin
)
2337 /* get physical address of buffer start */
2339 bzero(&di
->txp_dmah
[txout
], sizeof(hnddma_seg_map_t
));
2341 pa
= DMA_MAP(di
->osh
, data
, len
, DMA_TX
, p
,
2342 &di
->txp_dmah
[txout
]);
2344 if (DMASGLIST_ENAB
) {
2345 map
= &di
->txp_dmah
[txout
];
2347 /* See if all the segments can be accounted for */
2349 (uint
) (di
->ntxd
- NTXDACTIVE(di
->txin
, di
->txout
) -
2357 for (j
= 1; j
<= nsegs
; j
++) {
2359 if (p
== p0
&& j
== 1)
2360 flags
|= D64_CTRL1_SOF
;
2362 /* With a DMA segment list, Descriptor table is filled
2363 * using the segment list instead of looping over
2364 * buffers in multi-chain DMA. Therefore, EOF for SGLIST is when
2365 * end of segment list is reached.
2367 if ((!DMASGLIST_ENAB
&& next
== NULL
) ||
2368 (DMASGLIST_ENAB
&& j
== nsegs
))
2369 flags
|= (D64_CTRL1_IOC
| D64_CTRL1_EOF
);
2370 if (txout
== (di
->ntxd
- 1))
2371 flags
|= D64_CTRL1_EOT
;
2373 if (DMASGLIST_ENAB
) {
2374 len
= map
->segs
[j
- 1].length
;
2375 pa
= map
->segs
[j
- 1].addr
;
2377 dma64_dd_upd(di
, di
->txd64
, pa
, txout
, &flags
, len
);
2378 ASSERT(di
->txp
[txout
] == NULL
);
2380 txout
= NEXTTXD(txout
);
2383 /* See above. No need to loop over individual buffers */
2388 /* if last txd eof not set, fix it */
2389 if (!(flags
& D64_CTRL1_EOF
))
2390 W_SM(&di
->txd64
[PREVTXD(txout
)].ctrl1
,
2391 BUS_SWAP32(flags
| D64_CTRL1_IOC
| D64_CTRL1_EOF
));
2393 /* save the packet */
2394 di
->txp
[PREVTXD(txout
)] = p0
;
2396 /* bump the tx descriptor index */
2401 W_REG(di
->osh
, &di
->d64txregs
->ptr
,
2402 di
->xmtptrbase
+ I2B(txout
, dma64dd_t
));
2404 /* tx flow control */
2405 di
->hnddma
.txavail
= di
->ntxd
- NTXDACTIVE(di
->txin
, di
->txout
) - 1;
2410 DMA_ERROR(("%s: dma_txfast: out of txds !!!\n", di
->name
));
2411 PKTFREE(di
->osh
, p0
, TRUE
);
2412 di
->hnddma
.txavail
= 0;
2413 di
->hnddma
.txnobuf
++;
2418 * Reclaim next completed txd (txds if using chained buffers) in the range
2419 * specified and return associated packet.
2420 * If range is HNDDMA_RANGE_TRANSMITTED, reclaim descriptors that have be
2421 * transmitted as noted by the hardware "CurrDescr" pointer.
2422 * If range is HNDDMA_RANGE_TRANSFERED, reclaim descriptors that have be
2423 * transfered by the DMA as noted by the hardware "ActiveDescr" pointer.
2424 * If range is HNDDMA_RANGE_ALL, reclaim all txd(s) posted to the ring and
2425 * return associated packet regardless of the value of hardware pointers.
2427 static void *BCMFASTPATH
dma64_getnexttxp(dma_info_t
* di
, txd_range_t range
)
2429 uint16 start
, end
, i
;
2433 DMA_TRACE(("%s: dma_getnexttxp %s\n", di
->name
,
2434 (range
== HNDDMA_RANGE_ALL
) ? "all" :
2436 HNDDMA_RANGE_TRANSMITTED
) ? "transmitted" :
2445 if (range
== HNDDMA_RANGE_ALL
)
2448 dma64regs_t
*dregs
= di
->d64txregs
;
2452 (((R_REG(di
->osh
, &dregs
->status0
) &
2454 di
->xmtptrbase
) & D64_XS0_CD_MASK
, dma64dd_t
));
2456 if (range
== HNDDMA_RANGE_TRANSFERED
) {
2458 (uint16
) (R_REG(di
->osh
, &dregs
->status1
) &
2461 (active_desc
- di
->xmtptrbase
) & D64_XS0_CD_MASK
;
2462 active_desc
= B2I(active_desc
, dma64dd_t
);
2463 if (end
!= active_desc
)
2464 end
= PREVTXD(active_desc
);
2468 if ((start
== 0) && (end
> di
->txout
))
2471 for (i
= start
; i
!= end
&& !txp
; i
= NEXTTXD(i
)) {
2473 hnddma_seg_map_t
*map
= NULL
;
2474 uint size
, j
, nsegs
;
2477 (BUS_SWAP32(R_SM(&di
->txd64
[i
].addrlow
)) -
2478 di
->dataoffsetlow
));
2480 (BUS_SWAP32(R_SM(&di
->txd64
[i
].addrhigh
)) -
2481 di
->dataoffsethigh
));
2483 if (DMASGLIST_ENAB
) {
2484 map
= &di
->txp_dmah
[i
];
2485 size
= map
->origsize
;
2489 (BUS_SWAP32(R_SM(&di
->txd64
[i
].ctrl2
)) &
2494 for (j
= nsegs
; j
> 0; j
--) {
2495 W_SM(&di
->txd64
[i
].addrlow
, 0xdeadbeef);
2496 W_SM(&di
->txd64
[i
].addrhigh
, 0xdeadbeef);
2504 DMA_UNMAP(di
->osh
, pa
, size
, DMA_TX
, txp
, map
);
2509 /* tx flow control */
2510 di
->hnddma
.txavail
= di
->ntxd
- NTXDACTIVE(di
->txin
, di
->txout
) - 1;
2515 DMA_NONE(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n", start
, end
, di
->txout
, forceall
));
2519 static void *BCMFASTPATH
dma64_getnextrxp(dma_info_t
* di
, bool forceall
)
2525 /* if forcing, dma engine must be disabled */
2526 ASSERT(!forceall
|| !dma64_rxenabled(di
));
2530 /* return if no packets posted */
2535 B2I(((R_REG(di
->osh
, &di
->d64rxregs
->status0
) & D64_RS0_CD_MASK
) -
2536 di
->rcvptrbase
) & D64_RS0_CD_MASK
, dma64dd_t
);
2538 /* ignore curr if forceall */
2539 if (!forceall
&& (i
== curr
))
2542 /* get the packet pointer that corresponds to the rx descriptor */
2548 (BUS_SWAP32(R_SM(&di
->rxd64
[i
].addrlow
)) -
2549 di
->dataoffsetlow
));
2551 (BUS_SWAP32(R_SM(&di
->rxd64
[i
].addrhigh
)) -
2552 di
->dataoffsethigh
));
2554 /* clear this packet from the descriptor ring */
2555 DMA_UNMAP(di
->osh
, pa
, di
->rxbufsize
, DMA_RX
, rxp
, &di
->rxp_dmah
[i
]);
2557 W_SM(&di
->rxd64
[i
].addrlow
, 0xdeadbeef);
2558 W_SM(&di
->rxd64
[i
].addrhigh
, 0xdeadbeef);
2560 di
->rxin
= NEXTRXD(i
);
2565 static bool _dma64_addrext(osl_t
* osh
, dma64regs_t
* dma64regs
)
2568 OR_REG(osh
, &dma64regs
->control
, D64_XC_AE
);
2569 w
= R_REG(osh
, &dma64regs
->control
);
2570 AND_REG(osh
, &dma64regs
->control
, ~D64_XC_AE
);
2571 return ((w
& D64_XC_AE
) == D64_XC_AE
);
2575 * Rotate all active tx dma ring entries "forward" by (ActiveDescriptor - txin).
2577 static void dma64_txrotate(dma_info_t
* di
)
2586 ASSERT(dma64_txsuspendedidle(di
));
2588 nactive
= _dma_txactive(di
);
2590 ((((R_REG(di
->osh
, &di
->d64txregs
->status1
) &
2592 - di
->xmtptrbase
) & D64_XS1_AD_MASK
), dma64dd_t
));
2593 rot
= TXD(ad
- di
->txin
);
2595 ASSERT(rot
< di
->ntxd
);
2597 /* full-ring case is a lot harder - don't worry about this */
2598 if (rot
>= (di
->ntxd
- nactive
)) {
2599 DMA_ERROR(("%s: dma_txrotate: ring full - punt\n", di
->name
));
2604 last
= PREVTXD(di
->txout
);
2606 /* move entries starting at last and moving backwards to first */
2607 for (old
= last
; old
!= PREVTXD(first
); old
= PREVTXD(old
)) {
2608 new = TXD(old
+ rot
);
2611 * Move the tx dma descriptor.
2612 * EOT is set only in the last entry in the ring.
2614 w
= BUS_SWAP32(R_SM(&di
->txd64
[old
].ctrl1
)) & ~D64_CTRL1_EOT
;
2615 if (new == (di
->ntxd
- 1))
2617 W_SM(&di
->txd64
[new].ctrl1
, BUS_SWAP32(w
));
2619 w
= BUS_SWAP32(R_SM(&di
->txd64
[old
].ctrl2
));
2620 W_SM(&di
->txd64
[new].ctrl2
, BUS_SWAP32(w
));
2622 W_SM(&di
->txd64
[new].addrlow
, R_SM(&di
->txd64
[old
].addrlow
));
2623 W_SM(&di
->txd64
[new].addrhigh
, R_SM(&di
->txd64
[old
].addrhigh
));
2625 /* zap the old tx dma descriptor address field */
2626 W_SM(&di
->txd64
[old
].addrlow
, BUS_SWAP32(0xdeadbeef));
2627 W_SM(&di
->txd64
[old
].addrhigh
, BUS_SWAP32(0xdeadbeef));
2629 /* move the corresponding txp[] entry */
2630 ASSERT(di
->txp
[new] == NULL
);
2631 di
->txp
[new] = di
->txp
[old
];
2634 if (DMASGLIST_ENAB
) {
2635 bcopy(&di
->txp_dmah
[old
], &di
->txp_dmah
[new],
2636 sizeof(hnddma_seg_map_t
));
2637 bzero(&di
->txp_dmah
[old
], sizeof(hnddma_seg_map_t
));
2640 di
->txp
[old
] = NULL
;
2643 /* update txin and txout */
2645 di
->txout
= TXD(di
->txout
+ rot
);
2646 di
->hnddma
.txavail
= di
->ntxd
- NTXDACTIVE(di
->txin
, di
->txout
) - 1;
2649 W_REG(di
->osh
, &di
->d64txregs
->ptr
,
2650 di
->xmtptrbase
+ I2B(di
->txout
, dma64dd_t
));
2653 uint
dma_addrwidth(si_t
* sih
, void *dmaregs
)
2655 dma32regs_t
*dma32regs
;
2660 /* Perform 64-bit checks only if we want to advertise 64-bit (> 32bit) capability) */
2661 /* DMA engine is 64-bit capable */
2662 if ((si_core_sflags(sih
, 0, 0) & SISF_DMA64
) == SISF_DMA64
) {
2663 /* backplane are 64-bit capable */
2664 if (si_backplane64(sih
))
2665 /* If bus is System Backplane or PCIE then we can access 64-bits */
2666 if ((BUSTYPE(sih
->bustype
) == SI_BUS
) ||
2667 ((BUSTYPE(sih
->bustype
) == PCI_BUS
) &&
2668 (sih
->buscoretype
== PCIE_CORE_ID
)))
2669 return (DMADDRWIDTH_64
);
2671 /* DMA64 is always 32-bit capable, AE is always TRUE */
2672 ASSERT(_dma64_addrext(osh
, (dma64regs_t
*) dmaregs
));
2674 return (DMADDRWIDTH_32
);
2677 /* Start checking for 32-bit / 30-bit addressing */
2678 dma32regs
= (dma32regs_t
*) dmaregs
;
2680 /* For System Backplane, PCIE bus or addrext feature, 32-bits ok */
2681 if ((BUSTYPE(sih
->bustype
) == SI_BUS
) ||
2682 ((BUSTYPE(sih
->bustype
) == PCI_BUS
)
2683 && sih
->buscoretype
== PCIE_CORE_ID
)
2684 || (_dma32_addrext(osh
, dma32regs
)))
2685 return (DMADDRWIDTH_32
);
2688 return (DMADDRWIDTH_30
);