?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Updates to Tomato RAF including NGINX && PHP
[tomato.git] / release / src / shared / hnddma.c
blob42bdfca34b5745ed741ddeb6378281fc8e765c57
1 /*
2 * Generic Broadcom Home Networking Division (HND) DMA module.
3 * This supports the following chips: BCM42xx, 44xx, 47xx .
4 *
5 * Copyright 2004, Broadcom Corporation
6 * All Rights Reserved.
7 *
8 * THIS SOFTWARE IS OFFERED "AS IS", AND BROADCOM GRANTS NO WARRANTIES OF ANY
9 * KIND, EXPRESS OR IMPLIED, BY STATUTE, COMMUNICATION OR OTHERWISE. BROADCOM
10 * SPECIFICALLY DISCLAIMS ANY IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS
11 * FOR A SPECIFIC PURPOSE OR NONINFRINGEMENT CONCERNING THIS SOFTWARE.
12 *
13 * $Id$
14 */
15
16 #include <typedefs.h>
17 #include <bcmdefs.h>
18 #include <osl.h>
19 #include <bcmendian.h>
20 #include <sbconfig.h>
21 #include <bcmutils.h>
22 #include <bcmdevs.h>
23 #include <sbutils.h>
24
25 #include <sbhnddma.h>
26 #include <hnddma.h>
27
28 /* debug/trace */
29 #define DMA_ERROR(args)
30 #define DMA_TRACE(args)
31
32 /* default dma message level (if input msg_level pointer is null in dma_attach()) */
33 static uint dma_msg_level =
34 0;
35
36 #define MAXNAMEL 8 /* 8 char names */
37
38 #define DI_INFO(dmah) (dma_info_t *)dmah
39
40 /* dma engine software state */
41 typedef struct dma_info {
42 struct hnddma_pub hnddma; /* exported structure, don't use hnddma_t,
43 * which could be const
44 */
45 uint *msg_level; /* message level pointer */
46 char name[MAXNAMEL]; /* callers name for diag msgs */
47
48 void *osh; /* os handle */
49 sb_t *sbh; /* sb handle */
50
51 bool dma64; /* dma64 enabled */
52 bool addrext; /* this dma engine supports DmaExtendedAddrChanges */
53
54 dma32regs_t *d32txregs; /* 32 bits dma tx engine registers */
55 dma32regs_t *d32rxregs; /* 32 bits dma rx engine registers */
56 dma64regs_t *d64txregs; /* 64 bits dma tx engine registers */
57 dma64regs_t *d64rxregs; /* 64 bits dma rx engine registers */
58
59 uint32 dma64align; /* either 8k or 4k depends on number of dd */
60 dma32dd_t *txd32; /* pointer to dma32 tx descriptor ring */
61 dma64dd_t *txd64; /* pointer to dma64 tx descriptor ring */
62 uint ntxd; /* # tx descriptors tunable */
63 uint txin; /* index of next descriptor to reclaim */
64 uint txout; /* index of next descriptor to post */
65 void **txp; /* pointer to parallel array of pointers to packets */
66 osldma_t *tx_dmah; /* DMA TX descriptor ring handle */
67 osldma_t **txp_dmah; /* DMA TX packet data handle */
68 ulong txdpa; /* physical address of descriptor ring */
69 uint txdalign; /* #bytes added to alloc'd mem to align txd */
70 uint txdalloc; /* #bytes allocated for the ring */
71
72 dma32dd_t *rxd32; /* pointer to dma32 rx descriptor ring */
73 dma64dd_t *rxd64; /* pointer to dma64 rx descriptor ring */
74 uint nrxd; /* # rx descriptors tunable */
75 uint rxin; /* index of next descriptor to reclaim */
76 uint rxout; /* index of next descriptor to post */
77 void **rxp; /* pointer to parallel array of pointers to packets */
78 osldma_t *rx_dmah; /* DMA RX descriptor ring handle */
79 osldma_t **rxp_dmah; /* DMA RX packet data handle */
80 ulong rxdpa; /* physical address of descriptor ring */
81 uint rxdalign; /* #bytes added to alloc'd mem to align rxd */
82 uint rxdalloc; /* #bytes allocated for the ring */
83
84 /* tunables */
85 uint rxbufsize; /* rx buffer size in bytes,
86 not including the extra headroom
87 */
88 uint nrxpost; /* # rx buffers to keep posted */
89 uint rxoffset; /* rxcontrol offset */
90 uint ddoffsetlow; /* add to get dma address of descriptor ring, low 32 bits */
91 uint ddoffsethigh; /* high 32 bits */
92 uint dataoffsetlow; /* add to get dma address of data buffer, low 32 bits */
93 uint dataoffsethigh; /* high 32 bits */
94 } dma_info_t;
95
96 #ifdef BCMDMA64
97 #define DMA64_ENAB(di) ((di)->dma64)
98 #define DMA64_CAP TRUE
99 #else
100 #define DMA64_ENAB(di) (0)
101 #define DMA64_CAP FALSE
102 #endif
103
104 /* descriptor bumping macros */
105 #define XXD(x, n) ((x) & ((n) - 1)) /* faster than %, but n must be power of 2 */
106 #define TXD(x) XXD((x), di->ntxd)
107 #define RXD(x) XXD((x), di->nrxd)
108 #define NEXTTXD(i) TXD(i + 1)
109 #define PREVTXD(i) TXD(i - 1)
110 #define NEXTRXD(i) RXD(i + 1)
111 #define NTXDACTIVE(h, t) TXD(t - h)
112 #define NRXDACTIVE(h, t) RXD(t - h)
113
114 /* macros to convert between byte offsets and indexes */
115 #define B2I(bytes, type) ((bytes) / sizeof(type))
116 #define I2B(index, type) ((index) * sizeof(type))
117
118 #define PCI32ADDR_HIGH 0xc0000000 /* address[31:30] */
119 #define PCI32ADDR_HIGH_SHIFT 30 /* address[31:30] */
120
121
122 /* common prototypes */
123 static bool _dma_isaddrext(dma_info_t *di);
124 static bool _dma_alloc(dma_info_t *di, uint direction);
125 static void _dma_detach(dma_info_t *di);
126 static void _dma_ddtable_init(dma_info_t *di, uint direction, ulong pa);
127 static void _dma_rxinit(dma_info_t *di);
128 static void *_dma_rx(dma_info_t *di);
129 static void _dma_rxfill(dma_info_t *di);
130 static void _dma_rxreclaim(dma_info_t *di);
131 static void _dma_rxenable(dma_info_t *di);
132 static void * _dma_getnextrxp(dma_info_t *di, bool forceall);
133
134 static void _dma_txblock(dma_info_t *di);
135 static void _dma_txunblock(dma_info_t *di);
136 static uint _dma_txactive(dma_info_t *di);
137
138 static void* _dma_peeknexttxp(dma_info_t *di);
139 static uintptr _dma_getvar(dma_info_t *di, const char *name);
140 static void _dma_counterreset(dma_info_t *di);
141 static void _dma_fifoloopbackenable(dma_info_t *di);
142
143 /* ** 32 bit DMA prototypes */
144 static bool dma32_alloc(dma_info_t *di, uint direction);
145 static bool dma32_txreset(dma_info_t *di);
146 static bool dma32_rxreset(dma_info_t *di);
147 static bool dma32_txsuspendedidle(dma_info_t *di);
148 static int dma32_txfast(dma_info_t *di, void *p0, bool commit);
149 static void *dma32_getnexttxp(dma_info_t *di, bool forceall);
150 static void *dma32_getnextrxp(dma_info_t *di, bool forceall);
151 static void dma32_txrotate(dma_info_t *di);
152 static bool dma32_rxidle(dma_info_t *di);
153 static void dma32_txinit(dma_info_t *di);
154 static bool dma32_txenabled(dma_info_t *di);
155 static void dma32_txsuspend(dma_info_t *di);
156 static void dma32_txresume(dma_info_t *di);
157 static bool dma32_txsuspended(dma_info_t *di);
158 static void dma32_txreclaim(dma_info_t *di, bool forceall);
159 static bool dma32_txstopped(dma_info_t *di);
160 static bool dma32_rxstopped(dma_info_t *di);
161 static bool dma32_rxenabled(dma_info_t *di);
162 static bool _dma32_addrext(osl_t *osh, dma32regs_t *dma32regs);
163
164 /* ** 64 bit DMA prototypes and stubs */
165 #ifdef BCMDMA64
166 static bool dma64_alloc(dma_info_t *di, uint direction);
167 static bool dma64_txreset(dma_info_t *di);
168 static bool dma64_rxreset(dma_info_t *di);
169 static bool dma64_txsuspendedidle(dma_info_t *di);
170 static int dma64_txfast(dma_info_t *di, void *p0, bool commit);
171 static void *dma64_getnexttxp(dma_info_t *di, bool forceall);
172 static void *dma64_getnextrxp(dma_info_t *di, bool forceall);
173 static void dma64_txrotate(dma_info_t *di);
174
175 static bool dma64_rxidle(dma_info_t *di);
176 static void dma64_txinit(dma_info_t *di);
177 static bool dma64_txenabled(dma_info_t *di);
178 static void dma64_txsuspend(dma_info_t *di);
179 static void dma64_txresume(dma_info_t *di);
180 static bool dma64_txsuspended(dma_info_t *di);
181 static void dma64_txreclaim(dma_info_t *di, bool forceall);
182 static bool dma64_txstopped(dma_info_t *di);
183 static bool dma64_rxstopped(dma_info_t *di);
184 static bool dma64_rxenabled(dma_info_t *di);
185 static bool _dma64_addrext(osl_t *osh, dma64regs_t *dma64regs);
186
187 #else
188 static bool dma64_alloc(dma_info_t *di, uint direction) { return FALSE; }
189 static bool dma64_txreset(dma_info_t *di) { return FALSE; }
190 static bool dma64_rxreset(dma_info_t *di) { return FALSE; }
191 static bool dma64_txsuspendedidle(dma_info_t *di) { return FALSE;}
192 static int dma64_txfast(dma_info_t *di, void *p0, bool commit) { return 0; }
193 static void *dma64_getnexttxp(dma_info_t *di, bool forceall) { return NULL; }
194 static void *dma64_getnextrxp(dma_info_t *di, bool forceall) { return NULL; }
195 static void dma64_txrotate(dma_info_t *di) { return; }
196
197 static bool dma64_rxidle(dma_info_t *di) { return FALSE; }
198 static void dma64_txinit(dma_info_t *di) { return; }
199 static bool dma64_txenabled(dma_info_t *di) { return FALSE; }
200 static void dma64_txsuspend(dma_info_t *di) { return; }
201 static void dma64_txresume(dma_info_t *di) { return; }
202 static bool dma64_txsuspended(dma_info_t *di) {return FALSE; }
203 static void dma64_txreclaim(dma_info_t *di, bool forceall) { return; }
204 static bool dma64_txstopped(dma_info_t *di) { return FALSE; }
205 static bool dma64_rxstopped(dma_info_t *di) { return FALSE; }
206 static bool dma64_rxenabled(dma_info_t *di) { return FALSE; }
207 static bool _dma64_addrext(osl_t *osh, dma64regs_t *dma64regs) { return FALSE; }
208
209 #endif /* BCMDMA64 */
210
211
212
213 static di_fcn_t dma64proc = {
214 (di_detach_t)_dma_detach,
215 (di_txinit_t)dma64_txinit,
216 (di_txreset_t)dma64_txreset,
217 (di_txenabled_t)dma64_txenabled,
218 (di_txsuspend_t)dma64_txsuspend,
219 (di_txresume_t)dma64_txresume,
220 (di_txsuspended_t)dma64_txsuspended,
221 (di_txsuspendedidle_t)dma64_txsuspendedidle,
222 (di_txfast_t)dma64_txfast,
223 (di_txstopped_t)dma64_txstopped,
224 (di_txreclaim_t)dma64_txreclaim,
225 (di_getnexttxp_t)dma64_getnexttxp,
226 (di_peeknexttxp_t)_dma_peeknexttxp,
227 (di_txblock_t)_dma_txblock,
228 (di_txunblock_t)_dma_txunblock,
229 (di_txactive_t)_dma_txactive,
230 (di_txrotate_t)dma64_txrotate,
231
232 (di_rxinit_t)_dma_rxinit,
233 (di_rxreset_t)dma64_rxreset,
234 (di_rxidle_t)dma64_rxidle,
235 (di_rxstopped_t)dma64_rxstopped,
236 (di_rxenable_t)_dma_rxenable,
237 (di_rxenabled_t)dma64_rxenabled,
238 (di_rx_t)_dma_rx,
239 (di_rxfill_t)_dma_rxfill,
240 (di_rxreclaim_t)_dma_rxreclaim,
241 (di_getnextrxp_t)_dma_getnextrxp,
242
243 (di_fifoloopbackenable_t)_dma_fifoloopbackenable,
244 (di_getvar_t)_dma_getvar,
245 (di_counterreset_t)_dma_counterreset,
246
247 NULL,
248 NULL,
249 NULL,
250 34
251 };
252
253 static di_fcn_t dma32proc = {
254 (di_detach_t)_dma_detach,
255 (di_txinit_t)dma32_txinit,
256 (di_txreset_t)dma32_txreset,
257 (di_txenabled_t)dma32_txenabled,
258 (di_txsuspend_t)dma32_txsuspend,
259 (di_txresume_t)dma32_txresume,
260 (di_txsuspended_t)dma32_txsuspended,
261 (di_txsuspendedidle_t)dma32_txsuspendedidle,
262 (di_txfast_t)dma32_txfast,
263 (di_txstopped_t)dma32_txstopped,
264 (di_txreclaim_t)dma32_txreclaim,
265 (di_getnexttxp_t)dma32_getnexttxp,
266 (di_peeknexttxp_t)_dma_peeknexttxp,
267 (di_txblock_t)_dma_txblock,
268 (di_txunblock_t)_dma_txunblock,
269 (di_txactive_t)_dma_txactive,
270 (di_txrotate_t)dma32_txrotate,
271
272 (di_rxinit_t)_dma_rxinit,
273 (di_rxreset_t)dma32_rxreset,
274 (di_rxidle_t)dma32_rxidle,
275 (di_rxstopped_t)dma32_rxstopped,
276 (di_rxenable_t)_dma_rxenable,
277 (di_rxenabled_t)dma32_rxenabled,
278 (di_rx_t)_dma_rx,
279 (di_rxfill_t)_dma_rxfill,
280 (di_rxreclaim_t)_dma_rxreclaim,
281 (di_getnextrxp_t)_dma_getnextrxp,
282
283 (di_fifoloopbackenable_t)_dma_fifoloopbackenable,
284 (di_getvar_t)_dma_getvar,
285 (di_counterreset_t)_dma_counterreset,
286
287 NULL,
288 NULL,
289 NULL,
290 34
291 };
292
293 hnddma_t *
294 dma_attach(osl_t *osh, char *name, sb_t *sbh, void *dmaregstx, void *dmaregsrx,
295 uint ntxd, uint nrxd, uint rxbufsize, uint nrxpost, uint rxoffset, uint *msg_level)
296 {
297 dma_info_t *di;
298 uint size;
299
300 /* allocate private info structure */
301 if ((di = MALLOC(osh, sizeof (dma_info_t))) == NULL) {
302 return (NULL);
303 }
304 bzero((char *)di, sizeof(dma_info_t));
305
306 di->msg_level = msg_level ? msg_level : &dma_msg_level;
307
308 /* old chips w/o sb is no longer supported */
309 ASSERT(sbh != NULL);
310
311 di->dma64 = ((sb_coreflagshi(sbh, 0, 0) & SBTMH_DMA64) == SBTMH_DMA64);
312
313 #ifndef BCMDMA64
314 if (di->dma64) {
315 DMA_ERROR(("dma_attach: driver doesn't have the capability to support "
316 "64 bits DMA\n"));
317 goto fail;
318 }
319 #endif
320
321 /* check arguments */
322 ASSERT(ISPOWEROF2(ntxd));
323 ASSERT(ISPOWEROF2(nrxd));
324 if (nrxd == 0)
325 ASSERT(dmaregsrx == NULL);
326 if (ntxd == 0)
327 ASSERT(dmaregstx == NULL);
328
329
330 /* init dma reg pointer */
331 if (di->dma64) {
332 ASSERT(ntxd <= D64MAXDD);
333 ASSERT(nrxd <= D64MAXDD);
334 di->d64txregs = (dma64regs_t *)dmaregstx;
335 di->d64rxregs = (dma64regs_t *)dmaregsrx;
336
337 di->dma64align = D64RINGALIGN;
338 if ((ntxd < D64MAXDD / 2) && (nrxd < D64MAXDD / 2)) {
339 /* for smaller dd table, HW relax the alignment requirement */
340 di->dma64align = D64RINGALIGN / 2;
341 }
342 } else {
343 ASSERT(ntxd <= D32MAXDD);
344 ASSERT(nrxd <= D32MAXDD);
345 di->d32txregs = (dma32regs_t *)dmaregstx;
346 di->d32rxregs = (dma32regs_t *)dmaregsrx;
347 }
348
349 DMA_TRACE(("%s: dma_attach: %s osh %p ntxd %d nrxd %d rxbufsize %d nrxpost %d "
350 "rxoffset %d dmaregstx %p dmaregsrx %p\n",
351 name, (di->dma64 ? "DMA64" : "DMA32"), osh, ntxd, nrxd, rxbufsize,
352 nrxpost, rxoffset, dmaregstx, dmaregsrx));
353
354 /* make a private copy of our callers name */
355 strncpy(di->name, name, MAXNAMEL);
356 di->name[MAXNAMEL-1] = '\0';
357
358 di->osh = osh;
359 di->sbh = sbh;
360
361 /* save tunables */
362 di->ntxd = ntxd;
363 di->nrxd = nrxd;
364
365 /* the actual dma size doesn't include the extra headroom */
366 if (rxbufsize > BCMEXTRAHDROOM)
367 di->rxbufsize = rxbufsize - BCMEXTRAHDROOM;
368 else
369 di->rxbufsize = rxbufsize;
370
371 di->nrxpost = nrxpost;
372 di->rxoffset = rxoffset;
373
374 /*
375 * figure out the DMA physical address offset for dd and data
376 * for old chips w/o sb, use zero
377 * for new chips w sb,
378 * PCI/PCIE: they map silicon backplace address to zero based memory, need offset
379 * Other bus: use zero
380 * SB_BUS BIGENDIAN kludge: use sdram swapped region for data buffer, not descriptor
381 */
382 di->ddoffsetlow = 0;
383 di->dataoffsetlow = 0;
384 /* for pci bus, add offset */
385 if (sbh->bustype == PCI_BUS) {
386 if ((sbh->buscoretype == SB_PCIE) && di->dma64) {
387 /* pcie with DMA64 */
388 di->ddoffsetlow = 0;
389 di->ddoffsethigh = SB_PCIE_DMA_H32;
390 } else {
391 /* pci(DMA32/DMA64) or pcie with DMA32 */
392 di->ddoffsetlow = SB_PCI_DMA;
393 di->ddoffsethigh = 0;
394 }
395 di->dataoffsetlow = di->ddoffsetlow;
396 di->dataoffsethigh = di->ddoffsethigh;
397 }
398
399 #if defined(__mips__) && defined(IL_BIGENDIAN)
400 di->dataoffsetlow = di->dataoffsetlow + SB_SDRAM_SWAPPED;
401 #endif
402
403 di->addrext = _dma_isaddrext(di);
404
405 /* allocate tx packet pointer vector */
406 if (ntxd) {
407 size = ntxd * sizeof(void *);
408 if ((di->txp = MALLOC(osh, size)) == NULL) {
409 DMA_ERROR(("%s: dma_attach: out of tx memory, malloced %d bytes\n",
410 di->name, MALLOCED(osh)));
411 goto fail;
412 }
413 bzero((char *)di->txp, size);
414 }
415
416 /* allocate rx packet pointer vector */
417 if (nrxd) {
418 size = nrxd * sizeof(void *);
419 if ((di->rxp = MALLOC(osh, size)) == NULL) {
420 DMA_ERROR(("%s: dma_attach: out of rx memory, malloced %d bytes\n",
421 di->name, MALLOCED(osh)));
422 goto fail;
423 }
424 bzero((char *)di->rxp, size);
425 }
426
427 /* allocate transmit descriptor ring, only need ntxd descriptors but it must be aligned */
428 if (ntxd) {
429 if (!_dma_alloc(di, DMA_TX))
430 goto fail;
431 }
432
433 /* allocate receive descriptor ring, only need nrxd descriptors but it must be aligned */
434 if (nrxd) {
435 if (!_dma_alloc(di, DMA_RX))
436 goto fail;
437 }
438
439 if ((di->ddoffsetlow == SB_PCI_DMA) && (di->txdpa > SB_PCI_DMA_SZ) && !di->addrext) {
440 DMA_ERROR(("%s: dma_attach: txdpa 0x%lx: addrext not supported\n",
441 di->name, di->txdpa));
442 goto fail;
443 }
444 if ((di->ddoffsetlow == SB_PCI_DMA) && (di->rxdpa > SB_PCI_DMA_SZ) && !di->addrext) {
445 DMA_ERROR(("%s: dma_attach: rxdpa 0x%lx: addrext not supported\n",
446 di->name, di->rxdpa));
447 goto fail;
448 }
449
450 DMA_TRACE(("ddoffsetlow 0x%x ddoffsethigh 0x%x dataoffsetlow 0x%x dataoffsethigh "
451 "0x%x addrext %d\n", di->ddoffsetlow, di->ddoffsethigh, di->dataoffsetlow,
452 di->dataoffsethigh, di->addrext));
453
454 /* allocate tx packet pointer vector and DMA mapping vectors */
455 if (ntxd) {
456
457 size = ntxd * sizeof(osldma_t **);
458 if ((di->txp_dmah = (osldma_t **)MALLOC(osh, size)) == NULL)
459 goto fail;
460 bzero((char*)di->txp_dmah, size);
461 }else
462 di->txp_dmah = NULL;
463
464 /* allocate rx packet pointer vector and DMA mapping vectors */
465 if (nrxd) {
466
467 size = nrxd * sizeof(osldma_t **);
468 if ((di->rxp_dmah = (osldma_t **)MALLOC(osh, size)) == NULL)
469 goto fail;
470 bzero((char*)di->rxp_dmah, size);
471
472 }else
473 di->rxp_dmah = NULL;
474
475 /* initialize opsvec of function pointers */
476 di->hnddma.di_fn = DMA64_ENAB(di) ? dma64proc : dma32proc;
477
478 return ((hnddma_t *)di);
479
480 fail:
481 _dma_detach(di);
482 return (NULL);
483 }
484
485 /* init the tx or rx descriptor */
486 static INLINE void
487 dma32_dd_upd(dma_info_t *di, dma32dd_t *ddring, ulong pa, uint outidx, uint32 *flags,
488 uint32 bufcount)
489 {
490 /* dma32 uses 32 bits control to fit both flags and bufcounter */
491 *flags = *flags | (bufcount & CTRL_BC_MASK);
492
493 if ((di->dataoffsetlow != SB_PCI_DMA) || !(pa & PCI32ADDR_HIGH)) {
494 W_SM(&ddring[outidx].addr, BUS_SWAP32(pa + di->dataoffsetlow));
495 W_SM(&ddring[outidx].ctrl, BUS_SWAP32(*flags));
496 } else {
497 /* address extension */
498 uint32 ae;
499 ASSERT(di->addrext);
500 ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
501 pa &= ~PCI32ADDR_HIGH;
502
503 *flags |= (ae << CTRL_AE_SHIFT);
504 W_SM(&ddring[outidx].addr, BUS_SWAP32(pa + di->dataoffsetlow));
505 W_SM(&ddring[outidx].ctrl, BUS_SWAP32(*flags));
506 }
507 }
508
509 static INLINE void
510 dma64_dd_upd(dma_info_t *di, dma64dd_t *ddring, ulong pa, uint outidx, uint32 *flags,
511 uint32 bufcount)
512 {
513 uint32 ctrl2 = bufcount & D64_CTRL2_BC_MASK;
514
515 /* PCI bus with big(>1G) physical address, use address extension */
516 if ((di->dataoffsetlow != SB_PCI_DMA) || !(pa & PCI32ADDR_HIGH)) {
517 W_SM(&ddring[outidx].addrlow, BUS_SWAP32(pa + di->dataoffsetlow));
518 W_SM(&ddring[outidx].addrhigh, BUS_SWAP32(0 + di->dataoffsethigh));
519 W_SM(&ddring[outidx].ctrl1, BUS_SWAP32(*flags));
520 W_SM(&ddring[outidx].ctrl2, BUS_SWAP32(ctrl2));
521 } else {
522 /* address extension */
523 uint32 ae;
524 ASSERT(di->addrext);
525
526 ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
527 pa &= ~PCI32ADDR_HIGH;
528
529 ctrl2 |= (ae << D64_CTRL2_AE_SHIFT) & D64_CTRL2_AE;
530 W_SM(&ddring[outidx].addrlow, BUS_SWAP32(pa + di->dataoffsetlow));
531 W_SM(&ddring[outidx].addrhigh, BUS_SWAP32(0 + di->dataoffsethigh));
532 W_SM(&ddring[outidx].ctrl1, BUS_SWAP32(*flags));
533 W_SM(&ddring[outidx].ctrl2, BUS_SWAP32(ctrl2));
534 }
535 }
536
537 static bool
538 _dma32_addrext(osl_t *osh, dma32regs_t *dma32regs)
539 {
540 uint32 w;
541
542 OR_REG(osh, &dma32regs->control, XC_AE);
543 w = R_REG(osh, &dma32regs->control);
544 AND_REG(osh, &dma32regs->control, ~XC_AE);
545 return ((w & XC_AE) == XC_AE);
546 }
547
548 static bool
549 _dma_alloc(dma_info_t *di, uint direction)
550 {
551 if (DMA64_ENAB(di)) {
552 return dma64_alloc(di, direction);
553 } else {
554 return dma32_alloc(di, direction);
555 }
556 }
557
558 /* !! may be called with core in reset */
559 static void
560 _dma_detach(dma_info_t *di)
561 {
562 if (di == NULL)
563 return;
564
565 DMA_TRACE(("%s: dma_detach\n", di->name));
566
567 /* shouldn't be here if descriptors are unreclaimed */
568 ASSERT(di->txin == di->txout);
569 ASSERT(di->rxin == di->rxout);
570
571 /* free dma descriptor rings */
572 if (DMA64_ENAB(di)) {
573 if (di->txd64)
574 DMA_FREE_CONSISTENT(di->osh, ((int8*)(uintptr)di->txd64 - di->txdalign),
575 di->txdalloc, (di->txdpa - di->txdalign), &di->tx_dmah);
576 if (di->rxd64)
577 DMA_FREE_CONSISTENT(di->osh, ((int8*)(uintptr)di->rxd64 - di->rxdalign),
578 di->rxdalloc, (di->rxdpa - di->rxdalign), &di->rx_dmah);
579 } else {
580 if (di->txd32)
581 DMA_FREE_CONSISTENT(di->osh, ((int8*)(uintptr)di->txd32 - di->txdalign),
582 di->txdalloc, (di->txdpa - di->txdalign), &di->tx_dmah);
583 if (di->rxd32)
584 DMA_FREE_CONSISTENT(di->osh, ((int8*)(uintptr)di->rxd32 - di->rxdalign),
585 di->rxdalloc, (di->rxdpa - di->rxdalign), &di->rx_dmah);
586 }
587
588 /* free packet pointer vectors */
589 if (di->txp)
590 MFREE(di->osh, (void *)di->txp, (di->ntxd * sizeof(void *)));
591 if (di->rxp)
592 MFREE(di->osh, (void *)di->rxp, (di->nrxd * sizeof(void *)));
593
594 /* free tx packet DMA handles */
595 if (di->txp_dmah)
596 MFREE(di->osh, (void *)di->txp_dmah, di->ntxd * sizeof(osldma_t **));
597
598 /* free rx packet DMA handles */
599 if (di->rxp_dmah)
600 MFREE(di->osh, (void *)di->rxp_dmah, di->nrxd * sizeof(osldma_t **));
601
602 /* free our private info structure */
603 MFREE(di->osh, (void *)di, sizeof(dma_info_t));
604
605 }
606
607 /* return TRUE if this dma engine supports DmaExtendedAddrChanges, otherwise FALSE */
608 static bool
609 _dma_isaddrext(dma_info_t *di)
610 {
611 if (DMA64_ENAB(di)) {
612 /* DMA64 supports full 32 bits or 64 bits. AE is always valid */
613
614 /* not all tx or rx channel are available */
615 if (di->d64txregs != NULL) {
616 if (!_dma64_addrext(di->osh, di->d64txregs)) {
617 DMA_ERROR(("%s: _dma_isaddrext: DMA64 tx doesn't have AE set\n",
618 di->name));
619 ASSERT(0);
620 }
621 return TRUE;
622 } else if (di->d64rxregs != NULL) {
623 if (!_dma64_addrext(di->osh, di->d64rxregs)) {
624 DMA_ERROR(("%s: _dma_isaddrext: DMA64 rx doesn't have AE set\n",
625 di->name));
626 ASSERT(0);
627 }
628 return TRUE;
629 }
630 return FALSE;
631 } else if (di->d32txregs)
632 return (_dma32_addrext(di->osh, di->d32txregs));
633 else if (di->d32rxregs)
634 return (_dma32_addrext(di->osh, di->d32rxregs));
635 return FALSE;
636 }
637
638 /* initialize descriptor table base address */
639 static void
640 _dma_ddtable_init(dma_info_t *di, uint direction, ulong pa)
641 {
642 if (DMA64_ENAB(di)) {
643
644 if ((di->ddoffsetlow != SB_PCI_DMA) || !(pa & PCI32ADDR_HIGH)) {
645 if (direction == DMA_TX) {
646 W_REG(di->osh, &di->d64txregs->addrlow, (pa + di->ddoffsetlow));
647 W_REG(di->osh, &di->d64txregs->addrhigh, di->ddoffsethigh);
648 } else {
649 W_REG(di->osh, &di->d64rxregs->addrlow, (pa + di->ddoffsetlow));
650 W_REG(di->osh, &di->d64rxregs->addrhigh, di->ddoffsethigh);
651 }
652 } else {
653 /* DMA64 32bits address extension */
654 uint32 ae;
655 ASSERT(di->addrext);
656
657 /* shift the high bit(s) from pa to ae */
658 ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
659 pa &= ~PCI32ADDR_HIGH;
660
661 if (direction == DMA_TX) {
662 W_REG(di->osh, &di->d64txregs->addrlow, (pa + di->ddoffsetlow));
663 W_REG(di->osh, &di->d64txregs->addrhigh, di->ddoffsethigh);
664 SET_REG(di->osh, &di->d64txregs->control, D64_XC_AE,
665 (ae << D64_XC_AE_SHIFT));
666 } else {
667 W_REG(di->osh, &di->d64rxregs->addrlow, (pa + di->ddoffsetlow));
668 W_REG(di->osh, &di->d64rxregs->addrhigh, di->ddoffsethigh);
669 SET_REG(di->osh, &di->d64rxregs->control, D64_RC_AE,
670 (ae << D64_RC_AE_SHIFT));
671 }
672 }
673
674 } else {
675 if ((di->ddoffsetlow != SB_PCI_DMA) || !(pa & PCI32ADDR_HIGH)) {
676 if (direction == DMA_TX)
677 W_REG(di->osh, &di->d32txregs->addr, (pa + di->ddoffsetlow));
678 else
679 W_REG(di->osh, &di->d32rxregs->addr, (pa + di->ddoffsetlow));
680 } else {
681 /* dma32 address extension */
682 uint32 ae;
683 ASSERT(di->addrext);
684
685 /* shift the high bit(s) from pa to ae */
686 ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
687 pa &= ~PCI32ADDR_HIGH;
688
689 if (direction == DMA_TX) {
690 W_REG(di->osh, &di->d32txregs->addr, (pa + di->ddoffsetlow));
691 SET_REG(di->osh, &di->d32txregs->control, XC_AE, ae <<XC_AE_SHIFT);
692 } else {
693 W_REG(di->osh, &di->d32rxregs->addr, (pa + di->ddoffsetlow));
694 SET_REG(di->osh, &di->d32rxregs->control, RC_AE, ae <<RC_AE_SHIFT);
695 }
696 }
697 }
698 }
699
700 static void
701 _dma_fifoloopbackenable(dma_info_t *di)
702 {
703 DMA_TRACE(("%s: dma_fifoloopbackenable\n", di->name));
704 if (DMA64_ENAB(di))
705 OR_REG(di->osh, &di->d64txregs->control, D64_XC_LE);
706 else
707 OR_REG(di->osh, &di->d32txregs->control, XC_LE);
708 }
709
710 static void
711 _dma_rxinit(dma_info_t *di)
712 {
713 DMA_TRACE(("%s: dma_rxinit\n", di->name));
714
715 if (di->nrxd == 0)
716 return;
717
718 di->rxin = di->rxout = 0;
719
720 /* clear rx descriptor ring */
721 if (DMA64_ENAB(di))
722 BZERO_SM((void *)(uintptr)di->rxd64, (di->nrxd * sizeof(dma64dd_t)));
723 else
724 BZERO_SM((void *)(uintptr)di->rxd32, (di->nrxd * sizeof(dma32dd_t)));
725
726 _dma_rxenable(di);
727 _dma_ddtable_init(di, DMA_RX, di->rxdpa);
728 }
729
730 static void
731 _dma_rxenable(dma_info_t *di)
732 {
733 DMA_TRACE(("%s: dma_rxenable\n", di->name));
734
735 if (DMA64_ENAB(di))
736 W_REG(di->osh, &di->d64rxregs->control,
737 ((di->rxoffset << D64_RC_RO_SHIFT) | D64_RC_RE));
738 else
739 W_REG(di->osh, &di->d32rxregs->control, ((di->rxoffset << RC_RO_SHIFT) | RC_RE));
740 }
741
742 /* !! rx entry routine, returns a pointer to the next frame received,
743 * or NULL if there are no more
744 */
745 static void *
746 _dma_rx(dma_info_t *di)
747 {
748 void *p;
749 uint len;
750 int skiplen = 0;
751
752 while ((p = _dma_getnextrxp(di, FALSE))) {
753 /* skip giant packets which span multiple rx descriptors */
754 if (skiplen > 0) {
755 skiplen -= di->rxbufsize;
756 if (skiplen < 0)
757 skiplen = 0;
758 PKTFREE(di->osh, p, FALSE);
759 continue;
760 }
761
762 len = ltoh16(*(uint16*)(PKTDATA(di->osh, p)));
763 DMA_TRACE(("%s: dma_rx len %d\n", di->name, len));
764
765 /* bad frame length check */
766 if (len > (di->rxbufsize - di->rxoffset)) {
767 DMA_ERROR(("%s: dma_rx: bad frame length (%d)\n", di->name, len));
768 if (len > 0)
769 skiplen = len - (di->rxbufsize - di->rxoffset);
770 PKTFREE(di->osh, p, FALSE);
771 di->hnddma.rxgiants++;
772 continue;
773 }
774
775 /* set actual length */
776 PKTSETLEN(di->osh, p, (di->rxoffset + len));
777
778 break;
779 }
780
781 return (p);
782 }
783
784 /* post receive buffers */
785 static void
786 _dma_rxfill(dma_info_t *di)
787 {
788 void *p;
789 uint rxin, rxout;
790 uint32 flags = 0;
791 uint n;
792 uint i;
793 uint32 pa;
794 uint extra_offset = 0;
795
796 /*
797 * Determine how many receive buffers we're lacking
798 * from the full complement, allocate, initialize,
799 * and post them, then update the chip rx lastdscr.
800 */
801
802 rxin = di->rxin;
803 rxout = di->rxout;
804
805 n = di->nrxpost - NRXDACTIVE(rxin, rxout);
806
807 DMA_TRACE(("%s: dma_rxfill: post %d\n", di->name, n));
808
809 if (di->rxbufsize > BCMEXTRAHDROOM)
810 extra_offset = BCMEXTRAHDROOM;
811
812 for (i = 0; i < n; i++) {
813 /* the di->rxbufsize doesn't include the extra headroom, we need to add it to the
814 size to be allocated
815 */
816 if ((p = PKTGET(di->osh, di->rxbufsize + extra_offset,
817 FALSE)) == NULL) {
818 DMA_ERROR(("%s: dma_rxfill: out of rxbufs\n", di->name));
819 di->hnddma.rxnobuf++;
820 break;
821 }
822 /* reserve an extra headroom, if applicable */
823 if (extra_offset)
824 PKTPULL(di->osh, p, extra_offset);
825
826 /* Do a cached write instead of uncached write since DMA_MAP
827 * will flush the cache.
828 */
829 *(uint32*)(PKTDATA(di->osh, p)) = 0;
830
831 pa = (uint32) DMA_MAP(di->osh, PKTDATA(di->osh, p),
832 di->rxbufsize, DMA_RX, p,
833 &di->rxp_dmah[rxout]);
834
835 ASSERT(ISALIGNED(pa, 4));
836
837 /* save the free packet pointer */
838 ASSERT(di->rxp[rxout] == NULL);
839 di->rxp[rxout] = p;
840
841 /* reset flags for each descriptor */
842 flags = 0;
843 if (DMA64_ENAB(di)) {
844 if (rxout == (di->nrxd - 1))
845 flags = D64_CTRL1_EOT;
846
847 dma64_dd_upd(di, di->rxd64, pa, rxout, &flags, di->rxbufsize);
848 } else {
849 if (rxout == (di->nrxd - 1))
850 flags = CTRL_EOT;
851
852 dma32_dd_upd(di, di->rxd32, pa, rxout, &flags, di->rxbufsize);
853 }
854 rxout = NEXTRXD(rxout);
855 }
856
857 di->rxout = rxout;
858
859 /* update the chip lastdscr pointer */
860 if (DMA64_ENAB(di)) {
861 W_REG(di->osh, &di->d64rxregs->ptr, I2B(rxout, dma64dd_t));
862 } else {
863 W_REG(di->osh, &di->d32rxregs->ptr, I2B(rxout, dma32dd_t));
864 }
865 }
866
867 /* like getnexttxp but no reclaim */
868 static void *
869 _dma_peeknexttxp(dma_info_t *di)
870 {
871 uint end, i;
872
873 if (di->ntxd == 0)
874 return (NULL);
875
876 if (DMA64_ENAB(di)) {
877 end = B2I(R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_CD_MASK, dma64dd_t);
878 } else {
879 end = B2I(R_REG(di->osh, &di->d32txregs->status) & XS_CD_MASK, dma32dd_t);
880 }
881
882 for (i = di->txin; i != end; i = NEXTTXD(i))
883 if (di->txp[i])
884 return (di->txp[i]);
885
886 return (NULL);
887 }
888
889 static void
890 _dma_rxreclaim(dma_info_t *di)
891 {
892 void *p;
893
894 /* "unused local" warning suppression for OSLs that
895 * define PKTFREE() without using the di->osh arg
896 */
897 di = di;
898
899 DMA_TRACE(("%s: dma_rxreclaim\n", di->name));
900
901 while ((p = _dma_getnextrxp(di, TRUE)))
902 PKTFREE(di->osh, p, FALSE);
903 }
904
905 static void *
906 _dma_getnextrxp(dma_info_t *di, bool forceall)
907 {
908 if (di->nrxd == 0)
909 return (NULL);
910
911 if (DMA64_ENAB(di)) {
912 return dma64_getnextrxp(di, forceall);
913 } else {
914 return dma32_getnextrxp(di, forceall);
915 }
916 }
917
918 static void
919 _dma_txblock(dma_info_t *di)
920 {
921 di->hnddma.txavail = 0;
922 }
923
924 static void
925 _dma_txunblock(dma_info_t *di)
926 {
927 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
928 }
929
930 static uint
931 _dma_txactive(dma_info_t *di)
932 {
933 return (NTXDACTIVE(di->txin, di->txout));
934 }
935
936 static void
937 _dma_counterreset(dma_info_t *di)
938 {
939 /* reset all software counter */
940 di->hnddma.rxgiants = 0;
941 di->hnddma.rxnobuf = 0;
942 di->hnddma.txnobuf = 0;
943 }
944
945 /* get the address of the var in order to change later */
946 static uintptr
947 _dma_getvar(dma_info_t *di, const char *name)
948 {
949 if (!strcmp(name, "&txavail"))
950 return ((uintptr) &(di->hnddma.txavail));
951 else {
952 ASSERT(0);
953 }
954 return (0);
955 }
956
957 void
958 dma_txpioloopback(osl_t *osh, dma32regs_t *regs)
959 {
960 OR_REG(osh, &regs->control, XC_LE);
961 }
962
963
964
965 /* 32 bits DMA functions */
966 static void
967 dma32_txinit(dma_info_t *di)
968 {
969 DMA_TRACE(("%s: dma_txinit\n", di->name));
970
971 if (di->ntxd == 0)
972 return;
973
974 di->txin = di->txout = 0;
975 di->hnddma.txavail = di->ntxd - 1;
976
977 /* clear tx descriptor ring */
978 BZERO_SM((void *)(uintptr)di->txd32, (di->ntxd * sizeof(dma32dd_t)));
979 W_REG(di->osh, &di->d32txregs->control, XC_XE);
980 _dma_ddtable_init(di, DMA_TX, di->txdpa);
981 }
982
983 static bool
984 dma32_txenabled(dma_info_t *di)
985 {
986 uint32 xc;
987
988 /* If the chip is dead, it is not enabled :-) */
989 xc = R_REG(di->osh, &di->d32txregs->control);
990 return ((xc != 0xffffffff) && (xc & XC_XE));
991 }
992
993 static void
994 dma32_txsuspend(dma_info_t *di)
995 {
996 DMA_TRACE(("%s: dma_txsuspend\n", di->name));
997
998 if (di->ntxd == 0)
999 return;
1000
1001 OR_REG(di->osh, &di->d32txregs->control, XC_SE);
1002 }
1003
1004 static void
1005 dma32_txresume(dma_info_t *di)
1006 {
1007 DMA_TRACE(("%s: dma_txresume\n", di->name));
1008
1009 if (di->ntxd == 0)
1010 return;
1011
1012 AND_REG(di->osh, &di->d32txregs->control, ~XC_SE);
1013 }
1014
1015 static bool
1016 dma32_txsuspended(dma_info_t *di)
1017 {
1018 return (di->ntxd == 0) || ((R_REG(di->osh, &di->d32txregs->control) & XC_SE) == XC_SE);
1019 }
1020
1021 static void
1022 dma32_txreclaim(dma_info_t *di, bool forceall)
1023 {
1024 void *p;
1025
1026 DMA_TRACE(("%s: dma_txreclaim %s\n", di->name, forceall ? "all" : ""));
1027
1028 while ((p = dma32_getnexttxp(di, forceall)))
1029 PKTFREE(di->osh, p, TRUE);
1030 }
1031
1032 static bool
1033 dma32_txstopped(dma_info_t *di)
1034 {
1035 return ((R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK) == XS_XS_STOPPED);
1036 }
1037
1038 static bool
1039 dma32_rxstopped(dma_info_t *di)
1040 {
1041 return ((R_REG(di->osh, &di->d32rxregs->status) & RS_RS_MASK) == RS_RS_STOPPED);
1042 }
1043
1044 static bool
1045 dma32_alloc(dma_info_t *di, uint direction)
1046 {
1047 uint size;
1048 uint ddlen;
1049 void *va;
1050
1051 ddlen = sizeof(dma32dd_t);
1052
1053 size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);
1054
1055 if (!ISALIGNED(DMA_CONSISTENT_ALIGN, D32RINGALIGN))
1056 size += D32RINGALIGN;
1057
1058
1059 if (direction == DMA_TX) {
1060 if ((va = DMA_ALLOC_CONSISTENT(di->osh, size, &di->txdpa, &di->tx_dmah)) == NULL) {
1061 DMA_ERROR(("%s: dma_attach: DMA_ALLOC_CONSISTENT(ntxd) failed\n",
1062 di->name));
1063 return FALSE;
1064 }
1065
1066 di->txd32 = (dma32dd_t *) ROUNDUP((uintptr)va, D32RINGALIGN);
1067 di->txdalign = (uint)((int8*)(uintptr)di->txd32 - (int8*)va);
1068 di->txdpa += di->txdalign;
1069 di->txdalloc = size;
1070 ASSERT(ISALIGNED((uintptr)di->txd32, D32RINGALIGN));
1071 } else {
1072 if ((va = DMA_ALLOC_CONSISTENT(di->osh, size, &di->rxdpa, &di->rx_dmah)) == NULL) {
1073 DMA_ERROR(("%s: dma_attach: DMA_ALLOC_CONSISTENT(nrxd) failed\n",
1074 di->name));
1075 return FALSE;
1076 }
1077 di->rxd32 = (dma32dd_t *) ROUNDUP((uintptr)va, D32RINGALIGN);
1078 di->rxdalign = (uint)((int8*)(uintptr)di->rxd32 - (int8*)va);
1079 di->rxdpa += di->rxdalign;
1080 di->rxdalloc = size;
1081 ASSERT(ISALIGNED((uintptr)di->rxd32, D32RINGALIGN));
1082 }
1083
1084 return TRUE;
1085 }
1086
1087 static bool
1088 dma32_txreset(dma_info_t *di)
1089 {
1090 uint32 status;
1091
1092 if (di->ntxd == 0)
1093 return TRUE;
1094
1095 /* suspend tx DMA first */
1096 W_REG(di->osh, &di->d32txregs->control, XC_SE);
1097 SPINWAIT(((status = (R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK))
1098 != XS_XS_DISABLED) &&
1099 (status != XS_XS_IDLE) &&
1100 (status != XS_XS_STOPPED),
1101 (10000));
1102
1103 W_REG(di->osh, &di->d32txregs->control, 0);
1104 SPINWAIT(((status = (R_REG(di->osh,
1105 &di->d32txregs->status) & XS_XS_MASK)) != XS_XS_DISABLED),
1106 10000);
1107
1108 /* wait for the last transaction to complete */
1109 OSL_DELAY(300);
1110
1111 return (status == XS_XS_DISABLED);
1112 }
1113
1114 static bool
1115 dma32_rxidle(dma_info_t *di)
1116 {
1117 DMA_TRACE(("%s: dma_rxidle\n", di->name));
1118
1119 if (di->nrxd == 0)
1120 return TRUE;
1121
1122 return ((R_REG(di->osh, &di->d32rxregs->status) & RS_CD_MASK) ==
1123 R_REG(di->osh, &di->d32rxregs->ptr));
1124 }
1125
1126 static bool
1127 dma32_rxreset(dma_info_t *di)
1128 {
1129 uint32 status;
1130
1131 if (di->nrxd == 0)
1132 return TRUE;
1133
1134 W_REG(di->osh, &di->d32rxregs->control, 0);
1135 SPINWAIT(((status = (R_REG(di->osh,
1136 &di->d32rxregs->status) & RS_RS_MASK)) != RS_RS_DISABLED),
1137 10000);
1138
1139 return (status == RS_RS_DISABLED);
1140 }
1141
1142 static bool
1143 dma32_rxenabled(dma_info_t *di)
1144 {
1145 uint32 rc;
1146
1147 rc = R_REG(di->osh, &di->d32rxregs->control);
1148 return ((rc != 0xffffffff) && (rc & RC_RE));
1149 }
1150
1151 static bool
1152 dma32_txsuspendedidle(dma_info_t *di)
1153 {
1154 if (di->ntxd == 0)
1155 return TRUE;
1156
1157 if (!(R_REG(di->osh, &di->d32txregs->control) & XC_SE))
1158 return 0;
1159
1160 if ((R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK) != XS_XS_IDLE)
1161 return 0;
1162
1163 OSL_DELAY(2);
1164 return ((R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK) == XS_XS_IDLE);
1165 }
1166
1167 /* !! tx entry routine
1168 * supports full 32bit dma engine buffer addressing so
1169 * dma buffers can cross 4 Kbyte page boundaries.
1170 */
1171 static int
1172 dma32_txfast(dma_info_t *di, void *p0, bool commit)
1173 {
1174 void *p, *next;
1175 uchar *data;
1176 uint len;
1177 uint txout;
1178 uint32 flags = 0;
1179 uint32 pa;
1180
1181 DMA_TRACE(("%s: dma_txfast\n", di->name));
1182
1183 txout = di->txout;
1184
1185 /*
1186 * Walk the chain of packet buffers
1187 * allocating and initializing transmit descriptor entries.
1188 */
1189 for (p = p0; p; p = next) {
1190 data = PKTDATA(di->osh, p);
1191 len = PKTLEN(di->osh, p);
1192 next = PKTNEXT(di->osh, p);
1193
1194 /* return nonzero if out of tx descriptors */
1195 if (NEXTTXD(txout) == di->txin)
1196 goto outoftxd;
1197
1198 if (len == 0)
1199 continue;
1200
1201 /* get physical address of buffer start */
1202 pa = (uint32) DMA_MAP(di->osh, data, len, DMA_TX, p, &di->txp_dmah[txout]);
1203
1204 flags = 0;
1205 if (p == p0)
1206 flags |= CTRL_SOF;
1207 if (next == NULL)
1208 flags |= (CTRL_IOC | CTRL_EOF);
1209 if (txout == (di->ntxd - 1))
1210 flags |= CTRL_EOT;
1211
1212 dma32_dd_upd(di, di->txd32, pa, txout, &flags, len);
1213 ASSERT(di->txp[txout] == NULL);
1214
1215 txout = NEXTTXD(txout);
1216 }
1217
1218 /* if last txd eof not set, fix it */
1219 if (!(flags & CTRL_EOF))
1220 W_SM(&di->txd32[PREVTXD(txout)].ctrl, BUS_SWAP32(flags | CTRL_IOC | CTRL_EOF));
1221
1222 /* save the packet */
1223 di->txp[PREVTXD(txout)] = p0;
1224
1225 /* bump the tx descriptor index */
1226 di->txout = txout;
1227
1228 /* kick the chip */
1229 if (commit)
1230 W_REG(di->osh, &di->d32txregs->ptr, I2B(txout, dma32dd_t));
1231
1232 /* tx flow control */
1233 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1234
1235 return (0);
1236
1237 outoftxd:
1238 DMA_ERROR(("%s: dma_txfast: out of txds\n", di->name));
1239 PKTFREE(di->osh, p0, TRUE);
1240 di->hnddma.txavail = 0;
1241 di->hnddma.txnobuf++;
1242 return (-1);
1243 }
1244
1245 /*
1246 * Reclaim next completed txd (txds if using chained buffers) and
1247 * return associated packet.
1248 * If 'force' is true, reclaim txd(s) and return associated packet
1249 * regardless of the value of the hardware "curr" pointer.
1250 */
1251 static void *
1252 dma32_getnexttxp(dma_info_t *di, bool forceall)
1253 {
1254 uint start, end, i;
1255 void *txp;
1256
1257 DMA_TRACE(("%s: dma_getnexttxp %s\n", di->name, forceall ? "all" : ""));
1258
1259 if (di->ntxd == 0)
1260 return (NULL);
1261
1262 txp = NULL;
1263
1264 start = di->txin;
1265 if (forceall)
1266 end = di->txout;
1267 else
1268 end = B2I(R_REG(di->osh, &di->d32txregs->status) & XS_CD_MASK, dma32dd_t);
1269
1270 if ((start == 0) && (end > di->txout))
1271 goto bogus;
1272
1273 for (i = start; i != end && !txp; i = NEXTTXD(i)) {
1274 DMA_UNMAP(di->osh, (BUS_SWAP32(R_SM(&di->txd32[i].addr)) - di->dataoffsetlow),
1275 (BUS_SWAP32(R_SM(&di->txd32[i].ctrl)) & CTRL_BC_MASK),
1276 DMA_TX, di->txp[i], &di->txp_dmah[i]);
1277
1278 W_SM(&di->txd32[i].addr, 0xdeadbeef);
1279 txp = di->txp[i];
1280 di->txp[i] = NULL;
1281 }
1282
1283 di->txin = i;
1284
1285 /* tx flow control */
1286 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1287
1288 return (txp);
1289
1290 bogus:
1291 /*
1292 DMA_ERROR(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n",
1293 start, end, di->txout, forceall));
1294 */
1295 return (NULL);
1296 }
1297
1298 static void *
1299 dma32_getnextrxp(dma_info_t *di, bool forceall)
1300 {
1301 uint i;
1302 void *rxp;
1303
1304 /* if forcing, dma engine must be disabled */
1305 ASSERT(!forceall || !dma32_rxenabled(di));
1306
1307 i = di->rxin;
1308
1309 /* return if no packets posted */
1310 if (i == di->rxout)
1311 return (NULL);
1312
1313 /* ignore curr if forceall */
1314 if (!forceall && (i == B2I(R_REG(di->osh, &di->d32rxregs->status) & RS_CD_MASK, dma32dd_t)))
1315 return (NULL);
1316
1317 /* get the packet pointer that corresponds to the rx descriptor */
1318 rxp = di->rxp[i];
1319 ASSERT(rxp);
1320 di->rxp[i] = NULL;
1321
1322 /* clear this packet from the descriptor ring */
1323 DMA_UNMAP(di->osh, (BUS_SWAP32(R_SM(&di->rxd32[i].addr)) - di->dataoffsetlow),
1324 di->rxbufsize, DMA_RX, rxp, &di->rxp_dmah[i]);
1325
1326 W_SM(&di->rxd32[i].addr, 0xdeadbeef);
1327
1328 di->rxin = NEXTRXD(i);
1329
1330 return (rxp);
1331 }
1332
1333 /*
1334 * Rotate all active tx dma ring entries "forward" by (ActiveDescriptor - txin).
1335 */
1336 static void
1337 dma32_txrotate(dma_info_t *di)
1338 {
1339 uint ad;
1340 uint nactive;
1341 uint rot;
1342 uint old, new;
1343 uint32 w;
1344 uint first, last;
1345
1346 ASSERT(dma32_txsuspendedidle(di));
1347
1348 nactive = _dma_txactive(di);
1349 ad = B2I(((R_REG(di->osh, &di->d32txregs->status) & XS_AD_MASK) >> XS_AD_SHIFT), dma32dd_t);
1350 rot = TXD(ad - di->txin);
1351
1352 ASSERT(rot < di->ntxd);
1353
1354 /* full-ring case is a lot harder - don't worry about this */
1355 if (rot >= (di->ntxd - nactive)) {
1356 DMA_ERROR(("%s: dma_txrotate: ring full - punt\n", di->name));
1357 return;
1358 }
1359
1360 first = di->txin;
1361 last = PREVTXD(di->txout);
1362
1363 /* move entries starting at last and moving backwards to first */
1364 for (old = last; old != PREVTXD(first); old = PREVTXD(old)) {
1365 new = TXD(old + rot);
1366
1367 /*
1368 * Move the tx dma descriptor.
1369 * EOT is set only in the last entry in the ring.
1370 */
1371 w = BUS_SWAP32(R_SM(&di->txd32[old].ctrl)) & ~CTRL_EOT;
1372 if (new == (di->ntxd - 1))
1373 w |= CTRL_EOT;
1374 W_SM(&di->txd32[new].ctrl, BUS_SWAP32(w));
1375 W_SM(&di->txd32[new].addr, R_SM(&di->txd32[old].addr));
1376
1377 /* zap the old tx dma descriptor address field */
1378 W_SM(&di->txd32[old].addr, BUS_SWAP32(0xdeadbeef));
1379
1380 /* move the corresponding txp[] entry */
1381 ASSERT(di->txp[new] == NULL);
1382 di->txp[new] = di->txp[old];
1383 di->txp[old] = NULL;
1384 }
1385
1386 /* update txin and txout */
1387 di->txin = ad;
1388 di->txout = TXD(di->txout + rot);
1389 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1390
1391 /* kick the chip */
1392 W_REG(di->osh, &di->d32txregs->ptr, I2B(di->txout, dma32dd_t));
1393 }
1394
1395 /* 64 bits DMA functions */
1396
1397 #ifdef BCMDMA64
1398 static void
1399 dma64_txinit(dma_info_t *di)
1400 {
1401 DMA_TRACE(("%s: dma_txinit\n", di->name));
1402
1403 if (di->ntxd == 0)
1404 return;
1405
1406 di->txin = di->txout = 0;
1407 di->hnddma.txavail = di->ntxd - 1;
1408
1409 /* clear tx descriptor ring */
1410 BZERO_SM((void *)(uintptr)di->txd64, (di->ntxd * sizeof(dma64dd_t)));
1411 W_REG(di->osh, &di->d64txregs->control, D64_XC_XE);
1412 _dma_ddtable_init(di, DMA_TX, di->txdpa);
1413 }
1414
1415 static bool
1416 dma64_txenabled(dma_info_t *di)
1417 {
1418 uint32 xc;
1419
1420 /* If the chip is dead, it is not enabled :-) */
1421 xc = R_REG(di->osh, &di->d64txregs->control);
1422 return ((xc != 0xffffffff) && (xc & D64_XC_XE));
1423 }
1424
1425 static void
1426 dma64_txsuspend(dma_info_t *di)
1427 {
1428 DMA_TRACE(("%s: dma_txsuspend\n", di->name));
1429
1430 if (di->ntxd == 0)
1431 return;
1432
1433 OR_REG(di->osh, &di->d64txregs->control, D64_XC_SE);
1434 }
1435
1436 static void
1437 dma64_txresume(dma_info_t *di)
1438 {
1439 DMA_TRACE(("%s: dma_txresume\n", di->name));
1440
1441 if (di->ntxd == 0)
1442 return;
1443
1444 AND_REG(di->osh, &di->d64txregs->control, ~D64_XC_SE);
1445 }
1446
1447 static bool
1448 dma64_txsuspended(dma_info_t *di)
1449 {
1450 return (di->ntxd == 0) || ((R_REG(di->osh, &di->d64txregs->control) & D64_XC_SE)
1451 == D64_XC_SE);
1452 }
1453
1454 static void
1455 dma64_txreclaim(dma_info_t *di, bool forceall)
1456 {
1457 void *p;
1458
1459 DMA_TRACE(("%s: dma_txreclaim %s\n", di->name, forceall ? "all" : ""));
1460
1461 while ((p = dma64_getnexttxp(di, forceall)))
1462 PKTFREE(di->osh, p, TRUE);
1463 }
1464
1465 static bool
1466 dma64_txstopped(dma_info_t *di)
1467 {
1468 return ((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK) == D64_XS0_XS_STOPPED);
1469 }
1470
1471 static bool
1472 dma64_rxstopped(dma_info_t *di)
1473 {
1474 return ((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_RS_MASK) == D64_RS0_RS_STOPPED);
1475 }
1476
1477 static bool
1478 dma64_alloc(dma_info_t *di, uint direction)
1479 {
1480 uint size;
1481 uint ddlen;
1482 uint32 alignbytes;
1483 void *va;
1484
1485 ddlen = sizeof(dma64dd_t);
1486
1487 size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);
1488
1489 alignbytes = di->dma64align;
1490
1491 if (!ISALIGNED(DMA_CONSISTENT_ALIGN, alignbytes))
1492 size += alignbytes;
1493
1494 if (direction == DMA_TX) {
1495 if ((va = DMA_ALLOC_CONSISTENT(di->osh, size, &di->txdpa, &di->tx_dmah)) == NULL) {
1496 DMA_ERROR(("%s: dma_attach: DMA_ALLOC_CONSISTENT(ntxd) failed\n",
1497 di->name));
1498 return FALSE;
1499 }
1500
1501 di->txd64 = (dma64dd_t *) ROUNDUP((uintptr)va, alignbytes);
1502 di->txdalign = (uint)((int8*)(uintptr)di->txd64 - (int8*)va);
1503 di->txdpa += di->txdalign;
1504 di->txdalloc = size;
1505 ASSERT(ISALIGNED((uintptr)di->txd64, alignbytes));
1506 } else {
1507 if ((va = DMA_ALLOC_CONSISTENT(di->osh, size, &di->rxdpa, &di->rx_dmah)) == NULL) {
1508 DMA_ERROR(("%s: dma_attach: DMA_ALLOC_CONSISTENT(nrxd) failed\n",
1509 di->name));
1510 return FALSE;
1511 }
1512 di->rxd64 = (dma64dd_t *) ROUNDUP((uintptr)va, alignbytes);
1513 di->rxdalign = (uint)((int8*)(uintptr)di->rxd64 - (int8*)va);
1514 di->rxdpa += di->rxdalign;
1515 di->rxdalloc = size;
1516 ASSERT(ISALIGNED((uintptr)di->rxd64, alignbytes));
1517 }
1518
1519 return TRUE;
1520 }
1521
1522 static bool
1523 dma64_txreset(dma_info_t *di)
1524 {
1525 uint32 status;
1526
1527 if (di->ntxd == 0)
1528 return TRUE;
1529
1530 /* suspend tx DMA first */
1531 W_REG(di->osh, &di->d64txregs->control, D64_XC_SE);
1532 SPINWAIT(((status = (R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK)) !=
1533 D64_XS0_XS_DISABLED) &&
1534 (status != D64_XS0_XS_IDLE) &&
1535 (status != D64_XS0_XS_STOPPED),
1536 10000);
1537
1538 W_REG(di->osh, &di->d64txregs->control, 0);
1539 SPINWAIT(((status = (R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK)) !=
1540 D64_XS0_XS_DISABLED),
1541 10000);
1542
1543 /* wait for the last transaction to complete */
1544 OSL_DELAY(300);
1545
1546 return (status == D64_XS0_XS_DISABLED);
1547 }
1548
1549 static bool
1550 dma64_rxidle(dma_info_t *di)
1551 {
1552 DMA_TRACE(("%s: dma_rxidle\n", di->name));
1553
1554 if (di->nrxd == 0)
1555 return TRUE;
1556
1557 return ((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) ==
1558 R_REG(di->osh, &di->d64rxregs->ptr));
1559 }
1560
1561 static bool
1562 dma64_rxreset(dma_info_t *di)
1563 {
1564 uint32 status;
1565
1566 if (di->nrxd == 0)
1567 return TRUE;
1568
1569 W_REG(di->osh, &di->d64rxregs->control, 0);
1570 SPINWAIT(((status = (R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_RS_MASK)) !=
1571 D64_RS0_RS_DISABLED),
1572 10000);
1573
1574 return (status == D64_RS0_RS_DISABLED);
1575 }
1576
1577 static bool
1578 dma64_rxenabled(dma_info_t *di)
1579 {
1580 uint32 rc;
1581
1582 rc = R_REG(di->osh, &di->d64rxregs->control);
1583 return ((rc != 0xffffffff) && (rc & D64_RC_RE));
1584 }
1585
1586 static bool
1587 dma64_txsuspendedidle(dma_info_t *di)
1588 {
1589
1590 if (di->ntxd == 0)
1591 return TRUE;
1592
1593 if (!(R_REG(di->osh, &di->d64txregs->control) & D64_XC_SE))
1594 return 0;
1595
1596 if ((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK) == D64_XS0_XS_IDLE)
1597 return 1;
1598
1599 return 0;
1600 }
1601
1602
1603 /* !! tx entry routine */
1604 static int
1605 dma64_txfast(dma_info_t *di, void *p0, bool commit)
1606 {
1607 void *p, *next;
1608 uchar *data;
1609 uint len;
1610 uint txout;
1611 uint32 flags = 0;
1612 uint32 pa;
1613
1614 DMA_TRACE(("%s: dma_txfast\n", di->name));
1615
1616 txout = di->txout;
1617
1618 /*
1619 * Walk the chain of packet buffers
1620 * allocating and initializing transmit descriptor entries.
1621 */
1622 for (p = p0; p; p = next) {
1623 data = PKTDATA(di->osh, p);
1624 len = PKTLEN(di->osh, p);
1625 next = PKTNEXT(di->osh, p);
1626
1627 /* return nonzero if out of tx descriptors */
1628 if (NEXTTXD(txout) == di->txin)
1629 goto outoftxd;
1630
1631 if (len == 0)
1632 continue;
1633
1634 /* get physical address of buffer start */
1635 pa = (uint32) DMA_MAP(di->osh, data, len, DMA_TX, p, &di->txp_dmah[txout]);
1636
1637 flags = 0;
1638 if (p == p0)
1639 flags |= D64_CTRL1_SOF;
1640 if (next == NULL)
1641 flags |= (D64_CTRL1_IOC | D64_CTRL1_EOF);
1642 if (txout == (di->ntxd - 1))
1643 flags |= D64_CTRL1_EOT;
1644
1645 dma64_dd_upd(di, di->txd64, pa, txout, &flags, len);
1646 ASSERT(di->txp[txout] == NULL);
1647
1648 txout = NEXTTXD(txout);
1649 }
1650
1651 /* if last txd eof not set, fix it */
1652 if (!(flags & D64_CTRL1_EOF))
1653 W_SM(&di->txd64[PREVTXD(txout)].ctrl1,
1654 BUS_SWAP32(flags | D64_CTRL1_IOC | D64_CTRL1_EOF));
1655
1656 /* save the packet */
1657 di->txp[PREVTXD(txout)] = p0;
1658
1659 /* bump the tx descriptor index */
1660 di->txout = txout;
1661
1662 /* kick the chip */
1663 if (commit)
1664 W_REG(di->osh, &di->d64txregs->ptr, I2B(txout, dma64dd_t));
1665
1666 /* tx flow control */
1667 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1668
1669 return (0);
1670
1671 outoftxd:
1672 DMA_ERROR(("%s: dma_txfast: out of txds\n", di->name));
1673 PKTFREE(di->osh, p0, TRUE);
1674 di->hnddma.txavail = 0;
1675 di->hnddma.txnobuf++;
1676 return (-1);
1677 }
1678
1679 /*
1680 * Reclaim next completed txd (txds if using chained buffers) and
1681 * return associated packet.
1682 * If 'force' is true, reclaim txd(s) and return associated packet
1683 * regardless of the value of the hardware "curr" pointer.
1684 */
1685 static void *
1686 dma64_getnexttxp(dma_info_t *di, bool forceall)
1687 {
1688 uint start, end, i;
1689 void *txp;
1690
1691 DMA_TRACE(("%s: dma_getnexttxp %s\n", di->name, forceall ? "all" : ""));
1692
1693 if (di->ntxd == 0)
1694 return (NULL);
1695
1696 txp = NULL;
1697
1698 start = di->txin;
1699 if (forceall)
1700 end = di->txout;
1701 else
1702 end = B2I(R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_CD_MASK, dma64dd_t);
1703
1704 if ((start == 0) && (end > di->txout))
1705 goto bogus;
1706
1707 for (i = start; i != end && !txp; i = NEXTTXD(i)) {
1708 DMA_UNMAP(di->osh, (BUS_SWAP32(R_SM(&di->txd64[i].addrlow)) - di->dataoffsetlow),
1709 (BUS_SWAP32(R_SM(&di->txd64[i].ctrl2)) & D64_CTRL2_BC_MASK),
1710 DMA_TX, di->txp[i], &di->txp_dmah[i]);
1711
1712 W_SM(&di->txd64[i].addrlow, 0xdeadbeef);
1713 W_SM(&di->txd64[i].addrhigh, 0xdeadbeef);
1714
1715 txp = di->txp[i];
1716 di->txp[i] = NULL;
1717 }
1718
1719 di->txin = i;
1720
1721 /* tx flow control */
1722 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1723
1724 return (txp);
1725
1726 bogus:
1727 /*
1728 DMA_ERROR(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n",
1729 start, end, di->txout, forceall));
1730 */
1731 return (NULL);
1732 }
1733
1734 static void *
1735 dma64_getnextrxp(dma_info_t *di, bool forceall)
1736 {
1737 uint i;
1738 void *rxp;
1739
1740 /* if forcing, dma engine must be disabled */
1741 ASSERT(!forceall || !dma64_rxenabled(di));
1742
1743 i = di->rxin;
1744
1745 /* return if no packets posted */
1746 if (i == di->rxout)
1747 return (NULL);
1748
1749 /* ignore curr if forceall */
1750 if (!forceall &&
1751 (i == B2I(R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK, dma64dd_t)))
1752 return (NULL);
1753
1754 /* get the packet pointer that corresponds to the rx descriptor */
1755 rxp = di->rxp[i];
1756 ASSERT(rxp);
1757 di->rxp[i] = NULL;
1758
1759 /* clear this packet from the descriptor ring */
1760 DMA_UNMAP(di->osh, (BUS_SWAP32(R_SM(&di->rxd64[i].addrlow)) - di->dataoffsetlow),
1761 di->rxbufsize, DMA_RX, rxp, &di->rxp_dmah[i]);
1762
1763 W_SM(&di->rxd64[i].addrlow, 0xdeadbeef);
1764 W_SM(&di->rxd64[i].addrhigh, 0xdeadbeef);
1765
1766 di->rxin = NEXTRXD(i);
1767
1768 return (rxp);
1769 }
1770
1771 static bool
1772 _dma64_addrext(osl_t *osh, dma64regs_t *dma64regs)
1773 {
1774 uint32 w;
1775 OR_REG(osh, &dma64regs->control, D64_XC_AE);
1776 w = R_REG(osh, &dma64regs->control);
1777 AND_REG(osh, &dma64regs->control, ~D64_XC_AE);
1778 return ((w & D64_XC_AE) == D64_XC_AE);
1779 }
1780
1781 /*
1782 * Rotate all active tx dma ring entries "forward" by (ActiveDescriptor - txin).
1783 */
1784 static void
1785 dma64_txrotate(dma_info_t *di)
1786 {
1787 uint ad;
1788 uint nactive;
1789 uint rot;
1790 uint old, new;
1791 uint32 w;
1792 uint first, last;
1793
1794 ASSERT(dma64_txsuspendedidle(di));
1795
1796 nactive = _dma_txactive(di);
1797 ad = B2I((R_REG(di->osh, &di->d64txregs->status1) & D64_XS1_AD_MASK), dma64dd_t);
1798 rot = TXD(ad - di->txin);
1799
1800 ASSERT(rot < di->ntxd);
1801
1802 /* full-ring case is a lot harder - don't worry about this */
1803 if (rot >= (di->ntxd - nactive)) {
1804 DMA_ERROR(("%s: dma_txrotate: ring full - punt\n", di->name));
1805 return;
1806 }
1807
1808 first = di->txin;
1809 last = PREVTXD(di->txout);
1810
1811 /* move entries starting at last and moving backwards to first */
1812 for (old = last; old != PREVTXD(first); old = PREVTXD(old)) {
1813 new = TXD(old + rot);
1814
1815 /*
1816 * Move the tx dma descriptor.
1817 * EOT is set only in the last entry in the ring.
1818 */
1819 w = BUS_SWAP32(R_SM(&di->txd64[old].ctrl1)) & ~D64_CTRL1_EOT;
1820 if (new == (di->ntxd - 1))
1821 w |= D64_CTRL1_EOT;
1822 W_SM(&di->txd64[new].ctrl1, BUS_SWAP32(w));
1823
1824 w = BUS_SWAP32(R_SM(&di->txd64[old].ctrl2));
1825 W_SM(&di->txd64[new].ctrl2, BUS_SWAP32(w));
1826
1827 W_SM(&di->txd64[new].addrlow, R_SM(&di->txd64[old].addrlow));
1828 W_SM(&di->txd64[new].addrhigh, R_SM(&di->txd64[old].addrhigh));
1829
1830 /* zap the old tx dma descriptor address field */
1831 W_SM(&di->txd64[old].addrlow, BUS_SWAP32(0xdeadbeef));
1832 W_SM(&di->txd64[old].addrhigh, BUS_SWAP32(0xdeadbeef));
1833
1834 /* move the corresponding txp[] entry */
1835 ASSERT(di->txp[new] == NULL);
1836 di->txp[new] = di->txp[old];
1837 di->txp[old] = NULL;
1838 }
1839
1840 /* update txin and txout */
1841 di->txin = ad;
1842 di->txout = TXD(di->txout + rot);
1843 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1844
1845 /* kick the chip */
1846 W_REG(di->osh, &di->d64txregs->ptr, I2B(di->txout, dma64dd_t));
1847 }
1848
1849 #endif /* BCMDMA64 */
1850
1851 uint
1852 dma_addrwidth(sb_t *sbh, void *dmaregs)
1853 {
1854 dma32regs_t *dma32regs;
1855 osl_t *osh;
1856
1857 osh = sb_osh(sbh);
1858
1859 if (DMA64_CAP) {
1860 /* DMA engine is 64-bit capable */
1861 if (((sb_coreflagshi(sbh, 0, 0) & SBTMH_DMA64) == SBTMH_DMA64)) {
1862 /* backplane are 64 bits capable */
1863 if (sb_backplane64(sbh))
1864 /* If bus is System Backplane or PCIE then we can access 64-bits */
1865 if ((BUSTYPE(sbh->bustype) == SB_BUS) ||
1866 ((BUSTYPE(sbh->bustype) == PCI_BUS) &&
1867 sbh->buscoretype == SB_PCIE))
1868 return (DMADDRWIDTH_64);
1869
1870 /* DMA64 is always 32 bits capable, AE is always TRUE */
1871 #ifdef BCMDMA64
1872 ASSERT(_dma64_addrext(osh, (dma64regs_t *)dmaregs));
1873 #endif
1874 return (DMADDRWIDTH_32);
1875 }
1876 }
1877
1878 /* Start checking for 32-bit / 30-bit addressing */
1879 dma32regs = (dma32regs_t *)dmaregs;
1880
1881 /* For System Backplane, PCIE bus or addrext feature, 32-bits ok */
1882 if ((BUSTYPE(sbh->bustype) == SB_BUS) ||
1883 ((BUSTYPE(sbh->bustype) == PCI_BUS) && sbh->buscoretype == SB_PCIE) ||
1884 (_dma32_addrext(osh, dma32regs)))
1885 return (DMADDRWIDTH_32);
1886
1887 /* Fallthru */
1888 return (DMADDRWIDTH_30);
1889 }
Tomato firmware and modifications.