?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
staging: brcm80211: Move #includes out of header
[wandboard.git] / drivers / staging / brcm80211 / util / hnddma.c
blobb35dcf03aed4dc15031a3d788f21058ae39f63f9
1 /*
2 * Copyright (c) 2010 Broadcom Corporation
3 *
4 * Permission to use, copy, modify, and/or distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
11 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
13 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
14 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
16
17 #include <typedefs.h>
18 #include <linux/kernel.h>
19 #include <linux/string.h>
20 #include <linuxver.h>
21 #include <bcmdefs.h>
22 #include <bcmdevs.h>
23 #include <osl.h>
24 #include <bcmendian.h>
25 #include <hndsoc.h>
26 #include <bcmutils.h>
27 #include <siutils.h>
28
29 #include <sbhnddma.h>
30 #include <hnddma.h>
31
32 /* debug/trace */
33 #ifdef BCMDBG
34 #define DMA_ERROR(args) \
35 do { \
36 if (!(*di->msg_level & 1)) \
37 ; \
38 else \
39 printf args; \
40 } while (0)
41 #define DMA_TRACE(args) \
42 do { \
43 if (!(*di->msg_level & 2)) \
44 ; \
45 else \
46 printf args; \
47 } while (0)
48 #else
49 #define DMA_ERROR(args)
50 #define DMA_TRACE(args)
51 #endif /* BCMDBG */
52
53 #define DMA_NONE(args)
54
55 #define d32txregs dregs.d32_u.txregs_32
56 #define d32rxregs dregs.d32_u.rxregs_32
57 #define txd32 dregs.d32_u.txd_32
58 #define rxd32 dregs.d32_u.rxd_32
59
60 #define d64txregs dregs.d64_u.txregs_64
61 #define d64rxregs dregs.d64_u.rxregs_64
62 #define txd64 dregs.d64_u.txd_64
63 #define rxd64 dregs.d64_u.rxd_64
64
65 /* default dma message level (if input msg_level pointer is null in dma_attach()) */
66 static uint dma_msg_level;
67
68 #define MAXNAMEL 8 /* 8 char names */
69
70 #define DI_INFO(dmah) ((dma_info_t *)dmah)
71
72 /* dma engine software state */
73 typedef struct dma_info {
74 struct hnddma_pub hnddma; /* exported structure, don't use hnddma_t,
75 * which could be const
76 */
77 uint *msg_level; /* message level pointer */
78 char name[MAXNAMEL]; /* callers name for diag msgs */
79
80 void *osh; /* os handle */
81 si_t *sih; /* sb handle */
82
83 bool dma64; /* this dma engine is operating in 64-bit mode */
84 bool addrext; /* this dma engine supports DmaExtendedAddrChanges */
85
86 union {
87 struct {
88 dma32regs_t *txregs_32; /* 32-bit dma tx engine registers */
89 dma32regs_t *rxregs_32; /* 32-bit dma rx engine registers */
90 dma32dd_t *txd_32; /* pointer to dma32 tx descriptor ring */
91 dma32dd_t *rxd_32; /* pointer to dma32 rx descriptor ring */
92 } d32_u;
93 struct {
94 dma64regs_t *txregs_64; /* 64-bit dma tx engine registers */
95 dma64regs_t *rxregs_64; /* 64-bit dma rx engine registers */
96 dma64dd_t *txd_64; /* pointer to dma64 tx descriptor ring */
97 dma64dd_t *rxd_64; /* pointer to dma64 rx descriptor ring */
98 } d64_u;
99 } dregs;
100
101 uint16 dmadesc_align; /* alignment requirement for dma descriptors */
102
103 uint16 ntxd; /* # tx descriptors tunable */
104 uint16 txin; /* index of next descriptor to reclaim */
105 uint16 txout; /* index of next descriptor to post */
106 void **txp; /* pointer to parallel array of pointers to packets */
107 osldma_t *tx_dmah; /* DMA TX descriptor ring handle */
108 hnddma_seg_map_t *txp_dmah; /* DMA MAP meta-data handle */
109 dmaaddr_t txdpa; /* Aligned physical address of descriptor ring */
110 dmaaddr_t txdpaorig; /* Original physical address of descriptor ring */
111 uint16 txdalign; /* #bytes added to alloc'd mem to align txd */
112 uint32 txdalloc; /* #bytes allocated for the ring */
113 uint32 xmtptrbase; /* When using unaligned descriptors, the ptr register
114 * is not just an index, it needs all 13 bits to be
115 * an offset from the addr register.
116 */
117
118 uint16 nrxd; /* # rx descriptors tunable */
119 uint16 rxin; /* index of next descriptor to reclaim */
120 uint16 rxout; /* index of next descriptor to post */
121 void **rxp; /* pointer to parallel array of pointers to packets */
122 osldma_t *rx_dmah; /* DMA RX descriptor ring handle */
123 hnddma_seg_map_t *rxp_dmah; /* DMA MAP meta-data handle */
124 dmaaddr_t rxdpa; /* Aligned physical address of descriptor ring */
125 dmaaddr_t rxdpaorig; /* Original physical address of descriptor ring */
126 uint16 rxdalign; /* #bytes added to alloc'd mem to align rxd */
127 uint32 rxdalloc; /* #bytes allocated for the ring */
128 uint32 rcvptrbase; /* Base for ptr reg when using unaligned descriptors */
129
130 /* tunables */
131 uint16 rxbufsize; /* rx buffer size in bytes,
132 * not including the extra headroom
133 */
134 uint rxextrahdrroom; /* extra rx headroom, reverseved to assist upper stack
135 * e.g. some rx pkt buffers will be bridged to tx side
136 * without byte copying. The extra headroom needs to be
137 * large enough to fit txheader needs.
138 * Some dongle driver may not need it.
139 */
140 uint nrxpost; /* # rx buffers to keep posted */
141 uint rxoffset; /* rxcontrol offset */
142 uint ddoffsetlow; /* add to get dma address of descriptor ring, low 32 bits */
143 uint ddoffsethigh; /* high 32 bits */
144 uint dataoffsetlow; /* add to get dma address of data buffer, low 32 bits */
145 uint dataoffsethigh; /* high 32 bits */
146 bool aligndesc_4k; /* descriptor base need to be aligned or not */
147 } dma_info_t;
148
149 /*
150 * If BCMDMA32 is defined, hnddma will support both 32-bit and 64-bit DMA engines.
151 * Otherwise it will support only 64-bit.
152 *
153 * DMA32_ENAB indicates whether hnddma is compiled with support for 32-bit DMA engines.
154 * DMA64_ENAB indicates whether hnddma is compiled with support for 64-bit DMA engines.
155 *
156 * DMA64_MODE indicates whether the current DMA engine is running as 64-bit.
157 */
158 #ifdef BCMDMA32
159 #define DMA32_ENAB(di) 1
160 #define DMA64_ENAB(di) 1
161 #define DMA64_MODE(di) ((di)->dma64)
162 #else /* !BCMDMA32 */
163 #define DMA32_ENAB(di) 0
164 #define DMA64_ENAB(di) 1
165 #define DMA64_MODE(di) 1
166 #endif /* !BCMDMA32 */
167
168 /* DMA Scatter-gather list is supported. Note this is limited to TX direction only */
169 #ifdef BCMDMASGLISTOSL
170 #define DMASGLIST_ENAB TRUE
171 #else
172 #define DMASGLIST_ENAB FALSE
173 #endif /* BCMDMASGLISTOSL */
174
175 /* descriptor bumping macros */
176 #define XXD(x, n) ((x) & ((n) - 1)) /* faster than %, but n must be power of 2 */
177 #define TXD(x) XXD((x), di->ntxd)
178 #define RXD(x) XXD((x), di->nrxd)
179 #define NEXTTXD(i) TXD((i) + 1)
180 #define PREVTXD(i) TXD((i) - 1)
181 #define NEXTRXD(i) RXD((i) + 1)
182 #define PREVRXD(i) RXD((i) - 1)
183
184 #define NTXDACTIVE(h, t) TXD((t) - (h))
185 #define NRXDACTIVE(h, t) RXD((t) - (h))
186
187 /* macros to convert between byte offsets and indexes */
188 #define B2I(bytes, type) ((bytes) / sizeof(type))
189 #define I2B(index, type) ((index) * sizeof(type))
190
191 #define PCI32ADDR_HIGH 0xc0000000 /* address[31:30] */
192 #define PCI32ADDR_HIGH_SHIFT 30 /* address[31:30] */
193
194 #define PCI64ADDR_HIGH 0x80000000 /* address[63] */
195 #define PCI64ADDR_HIGH_SHIFT 31 /* address[63] */
196
197 /* Common prototypes */
198 static bool _dma_isaddrext(dma_info_t *di);
199 static bool _dma_descriptor_align(dma_info_t *di);
200 static bool _dma_alloc(dma_info_t *di, uint direction);
201 static void _dma_detach(dma_info_t *di);
202 static void _dma_ddtable_init(dma_info_t *di, uint direction, dmaaddr_t pa);
203 static void _dma_rxinit(dma_info_t *di);
204 static void *_dma_rx(dma_info_t *di);
205 static bool _dma_rxfill(dma_info_t *di);
206 static void _dma_rxreclaim(dma_info_t *di);
207 static void _dma_rxenable(dma_info_t *di);
208 static void *_dma_getnextrxp(dma_info_t *di, bool forceall);
209 static void _dma_rx_param_get(dma_info_t *di, uint16 *rxoffset,
210 uint16 *rxbufsize);
211
212 static void _dma_txblock(dma_info_t *di);
213 static void _dma_txunblock(dma_info_t *di);
214 static uint _dma_txactive(dma_info_t *di);
215 static uint _dma_rxactive(dma_info_t *di);
216 static uint _dma_txpending(dma_info_t *di);
217 static uint _dma_txcommitted(dma_info_t *di);
218
219 static void *_dma_peeknexttxp(dma_info_t *di);
220 static void *_dma_peeknextrxp(dma_info_t *di);
221 static uintptr _dma_getvar(dma_info_t *di, const char *name);
222 static void _dma_counterreset(dma_info_t *di);
223 static void _dma_fifoloopbackenable(dma_info_t *di);
224 static uint _dma_ctrlflags(dma_info_t *di, uint mask, uint flags);
225 static uint8 dma_align_sizetobits(uint size);
226 static void *dma_ringalloc(osl_t *osh, uint32 boundary, uint size,
227 uint16 *alignbits, uint *alloced,
228 dmaaddr_t *descpa, osldma_t **dmah);
229
230 /* Prototypes for 32-bit routines */
231 static bool dma32_alloc(dma_info_t *di, uint direction);
232 static bool dma32_txreset(dma_info_t *di);
233 static bool dma32_rxreset(dma_info_t *di);
234 static bool dma32_txsuspendedidle(dma_info_t *di);
235 static int dma32_txfast(dma_info_t *di, void *p0, bool commit);
236 static void *dma32_getnexttxp(dma_info_t *di, txd_range_t range);
237 static void *dma32_getnextrxp(dma_info_t *di, bool forceall);
238 static void dma32_txrotate(dma_info_t *di);
239 static bool dma32_rxidle(dma_info_t *di);
240 static void dma32_txinit(dma_info_t *di);
241 static bool dma32_txenabled(dma_info_t *di);
242 static void dma32_txsuspend(dma_info_t *di);
243 static void dma32_txresume(dma_info_t *di);
244 static bool dma32_txsuspended(dma_info_t *di);
245 static void dma32_txreclaim(dma_info_t *di, txd_range_t range);
246 static bool dma32_txstopped(dma_info_t *di);
247 static bool dma32_rxstopped(dma_info_t *di);
248 static bool dma32_rxenabled(dma_info_t *di);
249
250 static bool _dma32_addrext(osl_t *osh, dma32regs_t *dma32regs);
251
252 /* Prototypes for 64-bit routines */
253 static bool dma64_alloc(dma_info_t *di, uint direction);
254 static bool dma64_txreset(dma_info_t *di);
255 static bool dma64_rxreset(dma_info_t *di);
256 static bool dma64_txsuspendedidle(dma_info_t *di);
257 static int dma64_txfast(dma_info_t *di, void *p0, bool commit);
258 static int dma64_txunframed(dma_info_t *di, void *p0, uint len, bool commit);
259 static void *dma64_getpos(dma_info_t *di, bool direction);
260 static void *dma64_getnexttxp(dma_info_t *di, txd_range_t range);
261 static void *dma64_getnextrxp(dma_info_t *di, bool forceall);
262 static void dma64_txrotate(dma_info_t *di);
263
264 static bool dma64_rxidle(dma_info_t *di);
265 static void dma64_txinit(dma_info_t *di);
266 static bool dma64_txenabled(dma_info_t *di);
267 static void dma64_txsuspend(dma_info_t *di);
268 static void dma64_txresume(dma_info_t *di);
269 static bool dma64_txsuspended(dma_info_t *di);
270 static void dma64_txreclaim(dma_info_t *di, txd_range_t range);
271 static bool dma64_txstopped(dma_info_t *di);
272 static bool dma64_rxstopped(dma_info_t *di);
273 static bool dma64_rxenabled(dma_info_t *di);
274 static bool _dma64_addrext(osl_t *osh, dma64regs_t *dma64regs);
275
276 STATIC INLINE uint32 parity32(uint32 data);
277
278 const di_fcn_t dma64proc = {
279 (di_detach_t) _dma_detach,
280 (di_txinit_t) dma64_txinit,
281 (di_txreset_t) dma64_txreset,
282 (di_txenabled_t) dma64_txenabled,
283 (di_txsuspend_t) dma64_txsuspend,
284 (di_txresume_t) dma64_txresume,
285 (di_txsuspended_t) dma64_txsuspended,
286 (di_txsuspendedidle_t) dma64_txsuspendedidle,
287 (di_txfast_t) dma64_txfast,
288 (di_txunframed_t) dma64_txunframed,
289 (di_getpos_t) dma64_getpos,
290 (di_txstopped_t) dma64_txstopped,
291 (di_txreclaim_t) dma64_txreclaim,
292 (di_getnexttxp_t) dma64_getnexttxp,
293 (di_peeknexttxp_t) _dma_peeknexttxp,
294 (di_txblock_t) _dma_txblock,
295 (di_txunblock_t) _dma_txunblock,
296 (di_txactive_t) _dma_txactive,
297 (di_txrotate_t) dma64_txrotate,
298
299 (di_rxinit_t) _dma_rxinit,
300 (di_rxreset_t) dma64_rxreset,
301 (di_rxidle_t) dma64_rxidle,
302 (di_rxstopped_t) dma64_rxstopped,
303 (di_rxenable_t) _dma_rxenable,
304 (di_rxenabled_t) dma64_rxenabled,
305 (di_rx_t) _dma_rx,
306 (di_rxfill_t) _dma_rxfill,
307 (di_rxreclaim_t) _dma_rxreclaim,
308 (di_getnextrxp_t) _dma_getnextrxp,
309 (di_peeknextrxp_t) _dma_peeknextrxp,
310 (di_rxparam_get_t) _dma_rx_param_get,
311
312 (di_fifoloopbackenable_t) _dma_fifoloopbackenable,
313 (di_getvar_t) _dma_getvar,
314 (di_counterreset_t) _dma_counterreset,
315 (di_ctrlflags_t) _dma_ctrlflags,
316 NULL,
317 NULL,
318 NULL,
319 (di_rxactive_t) _dma_rxactive,
320 (di_txpending_t) _dma_txpending,
321 (di_txcommitted_t) _dma_txcommitted,
322 39
323 };
324
325 static const di_fcn_t dma32proc = {
326 (di_detach_t) _dma_detach,
327 (di_txinit_t) dma32_txinit,
328 (di_txreset_t) dma32_txreset,
329 (di_txenabled_t) dma32_txenabled,
330 (di_txsuspend_t) dma32_txsuspend,
331 (di_txresume_t) dma32_txresume,
332 (di_txsuspended_t) dma32_txsuspended,
333 (di_txsuspendedidle_t) dma32_txsuspendedidle,
334 (di_txfast_t) dma32_txfast,
335 NULL,
336 NULL,
337 (di_txstopped_t) dma32_txstopped,
338 (di_txreclaim_t) dma32_txreclaim,
339 (di_getnexttxp_t) dma32_getnexttxp,
340 (di_peeknexttxp_t) _dma_peeknexttxp,
341 (di_txblock_t) _dma_txblock,
342 (di_txunblock_t) _dma_txunblock,
343 (di_txactive_t) _dma_txactive,
344 (di_txrotate_t) dma32_txrotate,
345
346 (di_rxinit_t) _dma_rxinit,
347 (di_rxreset_t) dma32_rxreset,
348 (di_rxidle_t) dma32_rxidle,
349 (di_rxstopped_t) dma32_rxstopped,
350 (di_rxenable_t) _dma_rxenable,
351 (di_rxenabled_t) dma32_rxenabled,
352 (di_rx_t) _dma_rx,
353 (di_rxfill_t) _dma_rxfill,
354 (di_rxreclaim_t) _dma_rxreclaim,
355 (di_getnextrxp_t) _dma_getnextrxp,
356 (di_peeknextrxp_t) _dma_peeknextrxp,
357 (di_rxparam_get_t) _dma_rx_param_get,
358
359 (di_fifoloopbackenable_t) _dma_fifoloopbackenable,
360 (di_getvar_t) _dma_getvar,
361 (di_counterreset_t) _dma_counterreset,
362 (di_ctrlflags_t) _dma_ctrlflags,
363 NULL,
364 NULL,
365 NULL,
366 (di_rxactive_t) _dma_rxactive,
367 (di_txpending_t) _dma_txpending,
368 (di_txcommitted_t) _dma_txcommitted,
369 39
370 };
371
372 hnddma_t *dma_attach(osl_t *osh, char *name, si_t *sih, void *dmaregstx,
373 void *dmaregsrx, uint ntxd, uint nrxd, uint rxbufsize,
374 int rxextheadroom, uint nrxpost, uint rxoffset,
375 uint *msg_level)
376 {
377 dma_info_t *di;
378 uint size;
379
380 /* allocate private info structure */
381 di = MALLOC(osh, sizeof(dma_info_t));
382 if (di == NULL) {
383 #ifdef BCMDBG
384 printf("dma_attach: out of memory, malloced %d bytes\n",
385 MALLOCED(osh));
386 #endif
387 return NULL;
388 }
389
390 bzero((char *)di, sizeof(dma_info_t));
391
392 di->msg_level = msg_level ? msg_level : &dma_msg_level;
393
394 /* old chips w/o sb is no longer supported */
395 ASSERT(sih != NULL);
396
397 if (DMA64_ENAB(di))
398 di->dma64 =
399 ((si_core_sflags(sih, 0, 0) & SISF_DMA64) == SISF_DMA64);
400 else
401 di->dma64 = 0;
402
403 /* check arguments */
404 ASSERT(ISPOWEROF2(ntxd));
405 ASSERT(ISPOWEROF2(nrxd));
406
407 if (nrxd == 0)
408 ASSERT(dmaregsrx == NULL);
409 if (ntxd == 0)
410 ASSERT(dmaregstx == NULL);
411
412 /* init dma reg pointer */
413 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
414 ASSERT(ntxd <= D64MAXDD);
415 ASSERT(nrxd <= D64MAXDD);
416 di->d64txregs = (dma64regs_t *) dmaregstx;
417 di->d64rxregs = (dma64regs_t *) dmaregsrx;
418 di->hnddma.di_fn = (const di_fcn_t *)&dma64proc;
419 } else if (DMA32_ENAB(di)) {
420 ASSERT(ntxd <= D32MAXDD);
421 ASSERT(nrxd <= D32MAXDD);
422 di->d32txregs = (dma32regs_t *) dmaregstx;
423 di->d32rxregs = (dma32regs_t *) dmaregsrx;
424 di->hnddma.di_fn = (const di_fcn_t *)&dma32proc;
425 } else {
426 DMA_ERROR(("dma_attach: driver doesn't support 32-bit DMA\n"));
427 ASSERT(0);
428 goto fail;
429 }
430
431 /* Default flags (which can be changed by the driver calling dma_ctrlflags
432 * before enable): For backwards compatibility both Rx Overflow Continue
433 * and Parity are DISABLED.
434 * supports it.
435 */
436 di->hnddma.di_fn->ctrlflags(&di->hnddma, DMA_CTRL_ROC | DMA_CTRL_PEN,
437 0);
438
439 DMA_TRACE(("%s: dma_attach: %s osh %p flags 0x%x ntxd %d nrxd %d rxbufsize %d " "rxextheadroom %d nrxpost %d rxoffset %d dmaregstx %p dmaregsrx %p\n", name, (DMA64_MODE(di) ? "DMA64" : "DMA32"), osh, di->hnddma.dmactrlflags, ntxd, nrxd, rxbufsize, rxextheadroom, nrxpost, rxoffset, dmaregstx, dmaregsrx));
440
441 /* make a private copy of our callers name */
442 strncpy(di->name, name, MAXNAMEL);
443 di->name[MAXNAMEL - 1] = '\0';
444
445 di->osh = osh;
446 di->sih = sih;
447
448 /* save tunables */
449 di->ntxd = (uint16) ntxd;
450 di->nrxd = (uint16) nrxd;
451
452 /* the actual dma size doesn't include the extra headroom */
453 di->rxextrahdrroom =
454 (rxextheadroom == -1) ? BCMEXTRAHDROOM : rxextheadroom;
455 if (rxbufsize > BCMEXTRAHDROOM)
456 di->rxbufsize = (uint16) (rxbufsize - di->rxextrahdrroom);
457 else
458 di->rxbufsize = (uint16) rxbufsize;
459
460 di->nrxpost = (uint16) nrxpost;
461 di->rxoffset = (uint8) rxoffset;
462
463 /*
464 * figure out the DMA physical address offset for dd and data
465 * PCI/PCIE: they map silicon backplace address to zero based memory, need offset
466 * Other bus: use zero
467 * SI_BUS BIGENDIAN kludge: use sdram swapped region for data buffer, not descriptor
468 */
469 di->ddoffsetlow = 0;
470 di->dataoffsetlow = 0;
471 /* for pci bus, add offset */
472 if (sih->bustype == PCI_BUS) {
473 if ((sih->buscoretype == PCIE_CORE_ID) && DMA64_MODE(di)) {
474 /* pcie with DMA64 */
475 di->ddoffsetlow = 0;
476 di->ddoffsethigh = SI_PCIE_DMA_H32;
477 } else {
478 /* pci(DMA32/DMA64) or pcie with DMA32 */
479 di->ddoffsetlow = SI_PCI_DMA;
480 di->ddoffsethigh = 0;
481 }
482 di->dataoffsetlow = di->ddoffsetlow;
483 di->dataoffsethigh = di->ddoffsethigh;
484 }
485 #if defined(__mips__) && defined(IL_BIGENDIAN)
486 di->dataoffsetlow = di->dataoffsetlow + SI_SDRAM_SWAPPED;
487 #endif /* defined(__mips__) && defined(IL_BIGENDIAN) */
488 /* WAR64450 : DMACtl.Addr ext fields are not supported in SDIOD core. */
489 if ((si_coreid(sih) == SDIOD_CORE_ID)
490 && ((si_corerev(sih) > 0) && (si_corerev(sih) <= 2)))
491 di->addrext = 0;
492 else if ((si_coreid(sih) == I2S_CORE_ID) &&
493 ((si_corerev(sih) == 0) || (si_corerev(sih) == 1)))
494 di->addrext = 0;
495 else
496 di->addrext = _dma_isaddrext(di);
497
498 /* does the descriptors need to be aligned and if yes, on 4K/8K or not */
499 di->aligndesc_4k = _dma_descriptor_align(di);
500 if (di->aligndesc_4k) {
501 if (DMA64_MODE(di)) {
502 di->dmadesc_align = D64RINGALIGN_BITS;
503 if ((ntxd < D64MAXDD / 2) && (nrxd < D64MAXDD / 2)) {
504 /* for smaller dd table, HW relax the alignment requirement */
505 di->dmadesc_align = D64RINGALIGN_BITS - 1;
506 }
507 } else
508 di->dmadesc_align = D32RINGALIGN_BITS;
509 } else
510 di->dmadesc_align = 4; /* 16 byte alignment */
511
512 DMA_NONE(("DMA descriptor align_needed %d, align %d\n",
513 di->aligndesc_4k, di->dmadesc_align));
514
515 /* allocate tx packet pointer vector */
516 if (ntxd) {
517 size = ntxd * sizeof(void *);
518 di->txp = MALLOC(osh, size);
519 if (di->txp == NULL) {
520 DMA_ERROR(("%s: dma_attach: out of tx memory, malloced %d bytes\n", di->name, MALLOCED(osh)));
521 goto fail;
522 }
523 bzero((char *)di->txp, size);
524 }
525
526 /* allocate rx packet pointer vector */
527 if (nrxd) {
528 size = nrxd * sizeof(void *);
529 di->rxp = MALLOC(osh, size);
530 if (di->rxp == NULL) {
531 DMA_ERROR(("%s: dma_attach: out of rx memory, malloced %d bytes\n", di->name, MALLOCED(osh)));
532 goto fail;
533 }
534 bzero((char *)di->rxp, size);
535 }
536
537 /* allocate transmit descriptor ring, only need ntxd descriptors but it must be aligned */
538 if (ntxd) {
539 if (!_dma_alloc(di, DMA_TX))
540 goto fail;
541 }
542
543 /* allocate receive descriptor ring, only need nrxd descriptors but it must be aligned */
544 if (nrxd) {
545 if (!_dma_alloc(di, DMA_RX))
546 goto fail;
547 }
548
549 if ((di->ddoffsetlow != 0) && !di->addrext) {
550 if (PHYSADDRLO(di->txdpa) > SI_PCI_DMA_SZ) {
551 DMA_ERROR(("%s: dma_attach: txdpa 0x%x: addrext not supported\n", di->name, (uint32) PHYSADDRLO(di->txdpa)));
552 goto fail;
553 }
554 if (PHYSADDRLO(di->rxdpa) > SI_PCI_DMA_SZ) {
555 DMA_ERROR(("%s: dma_attach: rxdpa 0x%x: addrext not supported\n", di->name, (uint32) PHYSADDRLO(di->rxdpa)));
556 goto fail;
557 }
558 }
559
560 DMA_TRACE(("ddoffsetlow 0x%x ddoffsethigh 0x%x dataoffsetlow 0x%x dataoffsethigh " "0x%x addrext %d\n", di->ddoffsetlow, di->ddoffsethigh, di->dataoffsetlow, di->dataoffsethigh, di->addrext));
561
562 /* allocate DMA mapping vectors */
563 if (DMASGLIST_ENAB) {
564 if (ntxd) {
565 size = ntxd * sizeof(hnddma_seg_map_t);
566 di->txp_dmah = (hnddma_seg_map_t *) MALLOC(osh, size);
567 if (di->txp_dmah == NULL)
568 goto fail;
569 bzero((char *)di->txp_dmah, size);
570 }
571
572 if (nrxd) {
573 size = nrxd * sizeof(hnddma_seg_map_t);
574 di->rxp_dmah = (hnddma_seg_map_t *) MALLOC(osh, size);
575 if (di->rxp_dmah == NULL)
576 goto fail;
577 bzero((char *)di->rxp_dmah, size);
578 }
579 }
580
581 return (hnddma_t *) di;
582
583 fail:
584 _dma_detach(di);
585 return NULL;
586 }
587
588 /* init the tx or rx descriptor */
589 static INLINE void
590 dma32_dd_upd(dma_info_t *di, dma32dd_t *ddring, dmaaddr_t pa, uint outidx,
591 uint32 *flags, uint32 bufcount)
592 {
593 /* dma32 uses 32-bit control to fit both flags and bufcounter */
594 *flags = *flags | (bufcount & CTRL_BC_MASK);
595
596 if ((di->dataoffsetlow == 0) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) {
597 W_SM(&ddring[outidx].addr,
598 BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow));
599 W_SM(&ddring[outidx].ctrl, BUS_SWAP32(*flags));
600 } else {
601 /* address extension */
602 uint32 ae;
603 ASSERT(di->addrext);
604 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
605 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH;
606
607 *flags |= (ae << CTRL_AE_SHIFT);
608 W_SM(&ddring[outidx].addr,
609 BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow));
610 W_SM(&ddring[outidx].ctrl, BUS_SWAP32(*flags));
611 }
612 }
613
614 /* Check for odd number of 1's */
615 STATIC INLINE uint32 parity32(uint32 data)
616 {
617 data ^= data >> 16;
618 data ^= data >> 8;
619 data ^= data >> 4;
620 data ^= data >> 2;
621 data ^= data >> 1;
622
623 return data & 1;
624 }
625
626 #define DMA64_DD_PARITY(dd) parity32((dd)->addrlow ^ (dd)->addrhigh ^ (dd)->ctrl1 ^ (dd)->ctrl2)
627
628 static INLINE void
629 dma64_dd_upd(dma_info_t *di, dma64dd_t *ddring, dmaaddr_t pa, uint outidx,
630 uint32 *flags, uint32 bufcount)
631 {
632 uint32 ctrl2 = bufcount & D64_CTRL2_BC_MASK;
633
634 /* PCI bus with big(>1G) physical address, use address extension */
635 #if defined(__mips__) && defined(IL_BIGENDIAN)
636 if ((di->dataoffsetlow == SI_SDRAM_SWAPPED)
637 || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) {
638 #else
639 if ((di->dataoffsetlow == 0) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) {
640 #endif /* defined(__mips__) && defined(IL_BIGENDIAN) */
641 ASSERT((PHYSADDRHI(pa) & PCI64ADDR_HIGH) == 0);
642
643 W_SM(&ddring[outidx].addrlow,
644 BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow));
645 W_SM(&ddring[outidx].addrhigh,
646 BUS_SWAP32(PHYSADDRHI(pa) + di->dataoffsethigh));
647 W_SM(&ddring[outidx].ctrl1, BUS_SWAP32(*flags));
648 W_SM(&ddring[outidx].ctrl2, BUS_SWAP32(ctrl2));
649 } else {
650 /* address extension for 32-bit PCI */
651 uint32 ae;
652 ASSERT(di->addrext);
653
654 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
655 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH;
656 ASSERT(PHYSADDRHI(pa) == 0);
657
658 ctrl2 |= (ae << D64_CTRL2_AE_SHIFT) & D64_CTRL2_AE;
659 W_SM(&ddring[outidx].addrlow,
660 BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow));
661 W_SM(&ddring[outidx].addrhigh,
662 BUS_SWAP32(0 + di->dataoffsethigh));
663 W_SM(&ddring[outidx].ctrl1, BUS_SWAP32(*flags));
664 W_SM(&ddring[outidx].ctrl2, BUS_SWAP32(ctrl2));
665 }
666 if (di->hnddma.dmactrlflags & DMA_CTRL_PEN) {
667 if (DMA64_DD_PARITY(&ddring[outidx])) {
668 W_SM(&ddring[outidx].ctrl2,
669 BUS_SWAP32(ctrl2 | D64_CTRL2_PARITY));
670 }
671 }
672 }
673
674 static bool _dma32_addrext(osl_t *osh, dma32regs_t *dma32regs)
675 {
676 uint32 w;
677
678 OR_REG(osh, &dma32regs->control, XC_AE);
679 w = R_REG(osh, &dma32regs->control);
680 AND_REG(osh, &dma32regs->control, ~XC_AE);
681 return (w & XC_AE) == XC_AE;
682 }
683
684 static bool _dma_alloc(dma_info_t *di, uint direction)
685 {
686 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
687 return dma64_alloc(di, direction);
688 } else if (DMA32_ENAB(di)) {
689 return dma32_alloc(di, direction);
690 } else
691 ASSERT(0);
692 }
693
694 /* !! may be called with core in reset */
695 static void _dma_detach(dma_info_t *di)
696 {
697
698 DMA_TRACE(("%s: dma_detach\n", di->name));
699
700 /* shouldn't be here if descriptors are unreclaimed */
701 ASSERT(di->txin == di->txout);
702 ASSERT(di->rxin == di->rxout);
703
704 /* free dma descriptor rings */
705 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
706 if (di->txd64)
707 DMA_FREE_CONSISTENT(di->osh,
708 ((int8 *) (uintptr) di->txd64 -
709 di->txdalign), di->txdalloc,
710 (di->txdpaorig), &di->tx_dmah);
711 if (di->rxd64)
712 DMA_FREE_CONSISTENT(di->osh,
713 ((int8 *) (uintptr) di->rxd64 -
714 di->rxdalign), di->rxdalloc,
715 (di->rxdpaorig), &di->rx_dmah);
716 } else if (DMA32_ENAB(di)) {
717 if (di->txd32)
718 DMA_FREE_CONSISTENT(di->osh,
719 ((int8 *) (uintptr) di->txd32 -
720 di->txdalign), di->txdalloc,
721 (di->txdpaorig), &di->tx_dmah);
722 if (di->rxd32)
723 DMA_FREE_CONSISTENT(di->osh,
724 ((int8 *) (uintptr) di->rxd32 -
725 di->rxdalign), di->rxdalloc,
726 (di->rxdpaorig), &di->rx_dmah);
727 } else
728 ASSERT(0);
729
730 /* free packet pointer vectors */
731 if (di->txp)
732 MFREE(di->osh, (void *)di->txp, (di->ntxd * sizeof(void *)));
733 if (di->rxp)
734 MFREE(di->osh, (void *)di->rxp, (di->nrxd * sizeof(void *)));
735
736 /* free tx packet DMA handles */
737 if (di->txp_dmah)
738 MFREE(di->osh, (void *)di->txp_dmah,
739 di->ntxd * sizeof(hnddma_seg_map_t));
740
741 /* free rx packet DMA handles */
742 if (di->rxp_dmah)
743 MFREE(di->osh, (void *)di->rxp_dmah,
744 di->nrxd * sizeof(hnddma_seg_map_t));
745
746 /* free our private info structure */
747 MFREE(di->osh, (void *)di, sizeof(dma_info_t));
748
749 }
750
751 static bool _dma_descriptor_align(dma_info_t *di)
752 {
753 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
754 uint32 addrl;
755
756 /* Check to see if the descriptors need to be aligned on 4K/8K or not */
757 if (di->d64txregs != NULL) {
758 W_REG(di->osh, &di->d64txregs->addrlow, 0xff0);
759 addrl = R_REG(di->osh, &di->d64txregs->addrlow);
760 if (addrl != 0)
761 return FALSE;
762 } else if (di->d64rxregs != NULL) {
763 W_REG(di->osh, &di->d64rxregs->addrlow, 0xff0);
764 addrl = R_REG(di->osh, &di->d64rxregs->addrlow);
765 if (addrl != 0)
766 return FALSE;
767 }
768 }
769 return TRUE;
770 }
771
772 /* return TRUE if this dma engine supports DmaExtendedAddrChanges, otherwise FALSE */
773 static bool _dma_isaddrext(dma_info_t *di)
774 {
775 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
776 /* DMA64 supports full 32- or 64-bit operation. AE is always valid */
777
778 /* not all tx or rx channel are available */
779 if (di->d64txregs != NULL) {
780 if (!_dma64_addrext(di->osh, di->d64txregs)) {
781 DMA_ERROR(("%s: _dma_isaddrext: DMA64 tx doesn't have AE set\n", di->name));
782 ASSERT(0);
783 }
784 return TRUE;
785 } else if (di->d64rxregs != NULL) {
786 if (!_dma64_addrext(di->osh, di->d64rxregs)) {
787 DMA_ERROR(("%s: _dma_isaddrext: DMA64 rx doesn't have AE set\n", di->name));
788 ASSERT(0);
789 }
790 return TRUE;
791 }
792 return FALSE;
793 } else if (DMA32_ENAB(di)) {
794 if (di->d32txregs)
795 return _dma32_addrext(di->osh, di->d32txregs);
796 else if (di->d32rxregs)
797 return _dma32_addrext(di->osh, di->d32rxregs);
798 } else
799 ASSERT(0);
800
801 return FALSE;
802 }
803
804 /* initialize descriptor table base address */
805 static void _dma_ddtable_init(dma_info_t *di, uint direction, dmaaddr_t pa)
806 {
807 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
808 if (!di->aligndesc_4k) {
809 if (direction == DMA_TX)
810 di->xmtptrbase = PHYSADDRLO(pa);
811 else
812 di->rcvptrbase = PHYSADDRLO(pa);
813 }
814
815 if ((di->ddoffsetlow == 0)
816 || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) {
817 if (direction == DMA_TX) {
818 W_REG(di->osh, &di->d64txregs->addrlow,
819 (PHYSADDRLO(pa) + di->ddoffsetlow));
820 W_REG(di->osh, &di->d64txregs->addrhigh,
821 (PHYSADDRHI(pa) + di->ddoffsethigh));
822 } else {
823 W_REG(di->osh, &di->d64rxregs->addrlow,
824 (PHYSADDRLO(pa) + di->ddoffsetlow));
825 W_REG(di->osh, &di->d64rxregs->addrhigh,
826 (PHYSADDRHI(pa) + di->ddoffsethigh));
827 }
828 } else {
829 /* DMA64 32bits address extension */
830 uint32 ae;
831 ASSERT(di->addrext);
832 ASSERT(PHYSADDRHI(pa) == 0);
833
834 /* shift the high bit(s) from pa to ae */
835 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >>
836 PCI32ADDR_HIGH_SHIFT;
837 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH;
838
839 if (direction == DMA_TX) {
840 W_REG(di->osh, &di->d64txregs->addrlow,
841 (PHYSADDRLO(pa) + di->ddoffsetlow));
842 W_REG(di->osh, &di->d64txregs->addrhigh,
843 di->ddoffsethigh);
844 SET_REG(di->osh, &di->d64txregs->control,
845 D64_XC_AE, (ae << D64_XC_AE_SHIFT));
846 } else {
847 W_REG(di->osh, &di->d64rxregs->addrlow,
848 (PHYSADDRLO(pa) + di->ddoffsetlow));
849 W_REG(di->osh, &di->d64rxregs->addrhigh,
850 di->ddoffsethigh);
851 SET_REG(di->osh, &di->d64rxregs->control,
852 D64_RC_AE, (ae << D64_RC_AE_SHIFT));
853 }
854 }
855
856 } else if (DMA32_ENAB(di)) {
857 ASSERT(PHYSADDRHI(pa) == 0);
858 if ((di->ddoffsetlow == 0)
859 || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) {
860 if (direction == DMA_TX)
861 W_REG(di->osh, &di->d32txregs->addr,
862 (PHYSADDRLO(pa) + di->ddoffsetlow));
863 else
864 W_REG(di->osh, &di->d32rxregs->addr,
865 (PHYSADDRLO(pa) + di->ddoffsetlow));
866 } else {
867 /* dma32 address extension */
868 uint32 ae;
869 ASSERT(di->addrext);
870
871 /* shift the high bit(s) from pa to ae */
872 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >>
873 PCI32ADDR_HIGH_SHIFT;
874 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH;
875
876 if (direction == DMA_TX) {
877 W_REG(di->osh, &di->d32txregs->addr,
878 (PHYSADDRLO(pa) + di->ddoffsetlow));
879 SET_REG(di->osh, &di->d32txregs->control, XC_AE,
880 ae << XC_AE_SHIFT);
881 } else {
882 W_REG(di->osh, &di->d32rxregs->addr,
883 (PHYSADDRLO(pa) + di->ddoffsetlow));
884 SET_REG(di->osh, &di->d32rxregs->control, RC_AE,
885 ae << RC_AE_SHIFT);
886 }
887 }
888 } else
889 ASSERT(0);
890 }
891
892 static void _dma_fifoloopbackenable(dma_info_t *di)
893 {
894 DMA_TRACE(("%s: dma_fifoloopbackenable\n", di->name));
895
896 if (DMA64_ENAB(di) && DMA64_MODE(di))
897 OR_REG(di->osh, &di->d64txregs->control, D64_XC_LE);
898 else if (DMA32_ENAB(di))
899 OR_REG(di->osh, &di->d32txregs->control, XC_LE);
900 else
901 ASSERT(0);
902 }
903
904 static void _dma_rxinit(dma_info_t *di)
905 {
906 DMA_TRACE(("%s: dma_rxinit\n", di->name));
907
908 if (di->nrxd == 0)
909 return;
910
911 di->rxin = di->rxout = 0;
912
913 /* clear rx descriptor ring */
914 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
915 BZERO_SM((void *)(uintptr) di->rxd64,
916 (di->nrxd * sizeof(dma64dd_t)));
917
918 /* DMA engine with out alignment requirement requires table to be inited
919 * before enabling the engine
920 */
921 if (!di->aligndesc_4k)
922 _dma_ddtable_init(di, DMA_RX, di->rxdpa);
923
924 _dma_rxenable(di);
925
926 if (di->aligndesc_4k)
927 _dma_ddtable_init(di, DMA_RX, di->rxdpa);
928 } else if (DMA32_ENAB(di)) {
929 BZERO_SM((void *)(uintptr) di->rxd32,
930 (di->nrxd * sizeof(dma32dd_t)));
931 _dma_rxenable(di);
932 _dma_ddtable_init(di, DMA_RX, di->rxdpa);
933 } else
934 ASSERT(0);
935 }
936
937 static void _dma_rxenable(dma_info_t *di)
938 {
939 uint dmactrlflags = di->hnddma.dmactrlflags;
940
941 DMA_TRACE(("%s: dma_rxenable\n", di->name));
942
943 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
944 uint32 control =
945 (R_REG(di->osh, &di->d64rxregs->control) & D64_RC_AE) |
946 D64_RC_RE;
947
948 if ((dmactrlflags & DMA_CTRL_PEN) == 0)
949 control |= D64_RC_PD;
950
951 if (dmactrlflags & DMA_CTRL_ROC)
952 control |= D64_RC_OC;
953
954 W_REG(di->osh, &di->d64rxregs->control,
955 ((di->rxoffset << D64_RC_RO_SHIFT) | control));
956 } else if (DMA32_ENAB(di)) {
957 uint32 control =
958 (R_REG(di->osh, &di->d32rxregs->control) & RC_AE) | RC_RE;
959
960 if ((dmactrlflags & DMA_CTRL_PEN) == 0)
961 control |= RC_PD;
962
963 if (dmactrlflags & DMA_CTRL_ROC)
964 control |= RC_OC;
965
966 W_REG(di->osh, &di->d32rxregs->control,
967 ((di->rxoffset << RC_RO_SHIFT) | control));
968 } else
969 ASSERT(0);
970 }
971
972 static void
973 _dma_rx_param_get(dma_info_t *di, uint16 *rxoffset, uint16 *rxbufsize)
974 {
975 /* the normal values fit into 16 bits */
976 *rxoffset = (uint16) di->rxoffset;
977 *rxbufsize = (uint16) di->rxbufsize;
978 }
979
980 /* !! rx entry routine
981 * returns a pointer to the next frame received, or NULL if there are no more
982 * if DMA_CTRL_RXMULTI is defined, DMA scattering(multiple buffers) is supported
983 * with pkts chain
984 * otherwise, it's treated as giant pkt and will be tossed.
985 * The DMA scattering starts with normal DMA header, followed by first buffer data.
986 * After it reaches the max size of buffer, the data continues in next DMA descriptor
987 * buffer WITHOUT DMA header
988 */
989 static void *BCMFASTPATH _dma_rx(dma_info_t *di)
990 {
991 void *p, *head, *tail;
992 uint len;
993 uint pkt_len;
994 int resid = 0;
995
996 next_frame:
997 head = _dma_getnextrxp(di, FALSE);
998 if (head == NULL)
999 return NULL;
1000
1001 len = ltoh16(*(uint16 *) (PKTDATA(head)));
1002 DMA_TRACE(("%s: dma_rx len %d\n", di->name, len));
1003
1004 #if defined(__mips__)
1005 if (!len) {
1006 while (!(len = *(uint16 *) OSL_UNCACHED(PKTDATA(head))))
1007 OSL_DELAY(1);
1008
1009 *(uint16 *) PKTDATA(head) = htol16((uint16) len);
1010 }
1011 #endif /* defined(__mips__) */
1012
1013 /* set actual length */
1014 pkt_len = MIN((di->rxoffset + len), di->rxbufsize);
1015 PKTSETLEN(head, pkt_len);
1016 resid = len - (di->rxbufsize - di->rxoffset);
1017
1018 /* check for single or multi-buffer rx */
1019 if (resid > 0) {
1020 tail = head;
1021 while ((resid > 0) && (p = _dma_getnextrxp(di, FALSE))) {
1022 PKTSETNEXT(tail, p);
1023 pkt_len = MIN(resid, (int)di->rxbufsize);
1024 PKTSETLEN(p, pkt_len);
1025
1026 tail = p;
1027 resid -= di->rxbufsize;
1028 }
1029
1030 #ifdef BCMDBG
1031 if (resid > 0) {
1032 uint cur;
1033 ASSERT(p == NULL);
1034 cur = (DMA64_ENAB(di) && DMA64_MODE(di)) ?
1035 B2I(((R_REG(di->osh, &di->d64rxregs->status0) &
1036 D64_RS0_CD_MASK) -
1037 di->rcvptrbase) & D64_RS0_CD_MASK,
1038 dma64dd_t) : B2I(R_REG(di->osh,
1039 &di->d32rxregs->
1040 status) & RS_CD_MASK,
1041 dma32dd_t);
1042 DMA_ERROR(("_dma_rx, rxin %d rxout %d, hw_curr %d\n",
1043 di->rxin, di->rxout, cur));
1044 }
1045 #endif /* BCMDBG */
1046
1047 if ((di->hnddma.dmactrlflags & DMA_CTRL_RXMULTI) == 0) {
1048 DMA_ERROR(("%s: dma_rx: bad frame length (%d)\n",
1049 di->name, len));
1050 PKTFREE(di->osh, head, FALSE);
1051 di->hnddma.rxgiants++;
1052 goto next_frame;
1053 }
1054 }
1055
1056 return head;
1057 }
1058
1059 /* post receive buffers
1060 * return FALSE is refill failed completely and ring is empty
1061 * this will stall the rx dma and user might want to call rxfill again asap
1062 * This unlikely happens on memory-rich NIC, but often on memory-constrained dongle
1063 */
1064 static bool BCMFASTPATH _dma_rxfill(dma_info_t *di)
1065 {
1066 void *p;
1067 uint16 rxin, rxout;
1068 uint32 flags = 0;
1069 uint n;
1070 uint i;
1071 dmaaddr_t pa;
1072 uint extra_offset = 0;
1073 bool ring_empty;
1074
1075 ring_empty = FALSE;
1076
1077 /*
1078 * Determine how many receive buffers we're lacking
1079 * from the full complement, allocate, initialize,
1080 * and post them, then update the chip rx lastdscr.
1081 */
1082
1083 rxin = di->rxin;
1084 rxout = di->rxout;
1085
1086 n = di->nrxpost - NRXDACTIVE(rxin, rxout);
1087
1088 DMA_TRACE(("%s: dma_rxfill: post %d\n", di->name, n));
1089
1090 if (di->rxbufsize > BCMEXTRAHDROOM)
1091 extra_offset = di->rxextrahdrroom;
1092
1093 for (i = 0; i < n; i++) {
1094 /* the di->rxbufsize doesn't include the extra headroom, we need to add it to the
1095 size to be allocated
1096 */
1097
1098 p = osl_pktget(di->osh, di->rxbufsize + extra_offset);
1099
1100 if (p == NULL) {
1101 DMA_ERROR(("%s: dma_rxfill: out of rxbufs\n",
1102 di->name));
1103 if (i == 0) {
1104 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1105 if (dma64_rxidle(di)) {
1106 DMA_ERROR(("%s: rxfill64: ring is empty !\n", di->name));
1107 ring_empty = TRUE;
1108 }
1109 } else if (DMA32_ENAB(di)) {
1110 if (dma32_rxidle(di)) {
1111 DMA_ERROR(("%s: rxfill32: ring is empty !\n", di->name));
1112 ring_empty = TRUE;
1113 }
1114 } else
1115 ASSERT(0);
1116 }
1117 di->hnddma.rxnobuf++;
1118 break;
1119 }
1120 /* reserve an extra headroom, if applicable */
1121 if (extra_offset)
1122 PKTPULL(p, extra_offset);
1123
1124 /* Do a cached write instead of uncached write since DMA_MAP
1125 * will flush the cache.
1126 */
1127 *(uint32 *) (PKTDATA(p)) = 0;
1128
1129 if (DMASGLIST_ENAB)
1130 bzero(&di->rxp_dmah[rxout], sizeof(hnddma_seg_map_t));
1131
1132 pa = DMA_MAP(di->osh, PKTDATA(p),
1133 di->rxbufsize, DMA_RX, p, &di->rxp_dmah[rxout]);
1134
1135 ASSERT(ISALIGNED(PHYSADDRLO(pa), 4));
1136
1137 /* save the free packet pointer */
1138 ASSERT(di->rxp[rxout] == NULL);
1139 di->rxp[rxout] = p;
1140
1141 /* reset flags for each descriptor */
1142 flags = 0;
1143 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1144 if (rxout == (di->nrxd - 1))
1145 flags = D64_CTRL1_EOT;
1146
1147 dma64_dd_upd(di, di->rxd64, pa, rxout, &flags,
1148 di->rxbufsize);
1149 } else if (DMA32_ENAB(di)) {
1150 if (rxout == (di->nrxd - 1))
1151 flags = CTRL_EOT;
1152
1153 ASSERT(PHYSADDRHI(pa) == 0);
1154 dma32_dd_upd(di, di->rxd32, pa, rxout, &flags,
1155 di->rxbufsize);
1156 } else
1157 ASSERT(0);
1158 rxout = NEXTRXD(rxout);
1159 }
1160
1161 di->rxout = rxout;
1162
1163 /* update the chip lastdscr pointer */
1164 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1165 W_REG(di->osh, &di->d64rxregs->ptr,
1166 di->rcvptrbase + I2B(rxout, dma64dd_t));
1167 } else if (DMA32_ENAB(di)) {
1168 W_REG(di->osh, &di->d32rxregs->ptr, I2B(rxout, dma32dd_t));
1169 } else
1170 ASSERT(0);
1171
1172 return ring_empty;
1173 }
1174
1175 /* like getnexttxp but no reclaim */
1176 static void *_dma_peeknexttxp(dma_info_t *di)
1177 {
1178 uint end, i;
1179
1180 if (di->ntxd == 0)
1181 return NULL;
1182
1183 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1184 end =
1185 B2I(((R_REG(di->osh, &di->d64txregs->status0) &
1186 D64_XS0_CD_MASK) - di->xmtptrbase) & D64_XS0_CD_MASK,
1187 dma64dd_t);
1188 } else if (DMA32_ENAB(di)) {
1189 end =
1190 B2I(R_REG(di->osh, &di->d32txregs->status) & XS_CD_MASK,
1191 dma32dd_t);
1192 } else
1193 ASSERT(0);
1194
1195 for (i = di->txin; i != end; i = NEXTTXD(i))
1196 if (di->txp[i])
1197 return di->txp[i];
1198
1199 return NULL;
1200 }
1201
1202 /* like getnextrxp but not take off the ring */
1203 static void *_dma_peeknextrxp(dma_info_t *di)
1204 {
1205 uint end, i;
1206
1207 if (di->nrxd == 0)
1208 return NULL;
1209
1210 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1211 end =
1212 B2I(((R_REG(di->osh, &di->d64rxregs->status0) &
1213 D64_RS0_CD_MASK) - di->rcvptrbase) & D64_RS0_CD_MASK,
1214 dma64dd_t);
1215 } else if (DMA32_ENAB(di)) {
1216 end =
1217 B2I(R_REG(di->osh, &di->d32rxregs->status) & RS_CD_MASK,
1218 dma32dd_t);
1219 } else
1220 ASSERT(0);
1221
1222 for (i = di->rxin; i != end; i = NEXTRXD(i))
1223 if (di->rxp[i])
1224 return di->rxp[i];
1225
1226 return NULL;
1227 }
1228
1229 static void _dma_rxreclaim(dma_info_t *di)
1230 {
1231 void *p;
1232
1233 /* "unused local" warning suppression for OSLs that
1234 * define PKTFREE() without using the di->osh arg
1235 */
1236 di = di;
1237
1238 DMA_TRACE(("%s: dma_rxreclaim\n", di->name));
1239
1240 while ((p = _dma_getnextrxp(di, TRUE)))
1241 PKTFREE(di->osh, p, FALSE);
1242 }
1243
1244 static void *BCMFASTPATH _dma_getnextrxp(dma_info_t *di, bool forceall)
1245 {
1246 if (di->nrxd == 0)
1247 return NULL;
1248
1249 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1250 return dma64_getnextrxp(di, forceall);
1251 } else if (DMA32_ENAB(di)) {
1252 return dma32_getnextrxp(di, forceall);
1253 } else
1254 ASSERT(0);
1255 }
1256
1257 static void _dma_txblock(dma_info_t *di)
1258 {
1259 di->hnddma.txavail = 0;
1260 }
1261
1262 static void _dma_txunblock(dma_info_t *di)
1263 {
1264 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1265 }
1266
1267 static uint _dma_txactive(dma_info_t *di)
1268 {
1269 return NTXDACTIVE(di->txin, di->txout);
1270 }
1271
1272 static uint _dma_txpending(dma_info_t *di)
1273 {
1274 uint curr;
1275
1276 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1277 curr =
1278 B2I(((R_REG(di->osh, &di->d64txregs->status0) &
1279 D64_XS0_CD_MASK) - di->xmtptrbase) & D64_XS0_CD_MASK,
1280 dma64dd_t);
1281 } else if (DMA32_ENAB(di)) {
1282 curr =
1283 B2I(R_REG(di->osh, &di->d32txregs->status) & XS_CD_MASK,
1284 dma32dd_t);
1285 } else
1286 ASSERT(0);
1287
1288 return NTXDACTIVE(curr, di->txout);
1289 }
1290
1291 static uint _dma_txcommitted(dma_info_t *di)
1292 {
1293 uint ptr;
1294 uint txin = di->txin;
1295
1296 if (txin == di->txout)
1297 return 0;
1298
1299 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1300 ptr = B2I(R_REG(di->osh, &di->d64txregs->ptr), dma64dd_t);
1301 } else if (DMA32_ENAB(di)) {
1302 ptr = B2I(R_REG(di->osh, &di->d32txregs->ptr), dma32dd_t);
1303 } else
1304 ASSERT(0);
1305
1306 return NTXDACTIVE(di->txin, ptr);
1307 }
1308
1309 static uint _dma_rxactive(dma_info_t *di)
1310 {
1311 return NRXDACTIVE(di->rxin, di->rxout);
1312 }
1313
1314 static void _dma_counterreset(dma_info_t *di)
1315 {
1316 /* reset all software counter */
1317 di->hnddma.rxgiants = 0;
1318 di->hnddma.rxnobuf = 0;
1319 di->hnddma.txnobuf = 0;
1320 }
1321
1322 static uint _dma_ctrlflags(dma_info_t *di, uint mask, uint flags)
1323 {
1324 uint dmactrlflags = di->hnddma.dmactrlflags;
1325
1326 if (di == NULL) {
1327 DMA_ERROR(("%s: _dma_ctrlflags: NULL dma handle\n", di->name));
1328 return 0;
1329 }
1330
1331 ASSERT((flags & ~mask) == 0);
1332
1333 dmactrlflags &= ~mask;
1334 dmactrlflags |= flags;
1335
1336 /* If trying to enable parity, check if parity is actually supported */
1337 if (dmactrlflags & DMA_CTRL_PEN) {
1338 uint32 control;
1339
1340 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1341 control = R_REG(di->osh, &di->d64txregs->control);
1342 W_REG(di->osh, &di->d64txregs->control,
1343 control | D64_XC_PD);
1344 if (R_REG(di->osh, &di->d64txregs->control) & D64_XC_PD) {
1345 /* We *can* disable it so it is supported,
1346 * restore control register
1347 */
1348 W_REG(di->osh, &di->d64txregs->control,
1349 control);
1350 } else {
1351 /* Not supported, don't allow it to be enabled */
1352 dmactrlflags &= ~DMA_CTRL_PEN;
1353 }
1354 } else if (DMA32_ENAB(di)) {
1355 control = R_REG(di->osh, &di->d32txregs->control);
1356 W_REG(di->osh, &di->d32txregs->control,
1357 control | XC_PD);
1358 if (R_REG(di->osh, &di->d32txregs->control) & XC_PD) {
1359 W_REG(di->osh, &di->d32txregs->control,
1360 control);
1361 } else {
1362 /* Not supported, don't allow it to be enabled */
1363 dmactrlflags &= ~DMA_CTRL_PEN;
1364 }
1365 } else
1366 ASSERT(0);
1367 }
1368
1369 di->hnddma.dmactrlflags = dmactrlflags;
1370
1371 return dmactrlflags;
1372 }
1373
1374 /* get the address of the var in order to change later */
1375 static uintptr _dma_getvar(dma_info_t *di, const char *name)
1376 {
1377 if (!strcmp(name, "&txavail"))
1378 return (uintptr) &(di->hnddma.txavail);
1379 else {
1380 ASSERT(0);
1381 }
1382 return 0;
1383 }
1384
1385 void dma_txpioloopback(osl_t *osh, dma32regs_t *regs)
1386 {
1387 OR_REG(osh, &regs->control, XC_LE);
1388 }
1389
1390 static
1391 uint8 dma_align_sizetobits(uint size)
1392 {
1393 uint8 bitpos = 0;
1394 ASSERT(size);
1395 ASSERT(!(size & (size - 1)));
1396 while (size >>= 1) {
1397 bitpos++;
1398 }
1399 return bitpos;
1400 }
1401
1402 /* This function ensures that the DMA descriptor ring will not get allocated
1403 * across Page boundary. If the allocation is done across the page boundary
1404 * at the first time, then it is freed and the allocation is done at
1405 * descriptor ring size aligned location. This will ensure that the ring will
1406 * not cross page boundary
1407 */
1408 static void *dma_ringalloc(osl_t *osh, uint32 boundary, uint size,
1409 uint16 *alignbits, uint *alloced,
1410 dmaaddr_t *descpa, osldma_t **dmah)
1411 {
1412 void *va;
1413 uint32 desc_strtaddr;
1414 uint32 alignbytes = 1 << *alignbits;
1415
1416 va = DMA_ALLOC_CONSISTENT(osh, size, *alignbits, alloced, descpa,
1417 dmah);
1418 if (NULL == va)
1419 return NULL;
1420
1421 desc_strtaddr = (uint32) ROUNDUP((uintptr) va, alignbytes);
1422 if (((desc_strtaddr + size - 1) & boundary) != (desc_strtaddr
1423 & boundary)) {
1424 *alignbits = dma_align_sizetobits(size);
1425 DMA_FREE_CONSISTENT(osh, va, size, *descpa, dmah);
1426 va = DMA_ALLOC_CONSISTENT(osh, size, *alignbits, alloced,
1427 descpa, dmah);
1428 }
1429 return va;
1430 }
1431
1432 /* 32-bit DMA functions */
1433
1434 static void dma32_txinit(dma_info_t *di)
1435 {
1436 uint32 control = XC_XE;
1437
1438 DMA_TRACE(("%s: dma_txinit\n", di->name));
1439
1440 if (di->ntxd == 0)
1441 return;
1442
1443 di->txin = di->txout = 0;
1444 di->hnddma.txavail = di->ntxd - 1;
1445
1446 /* clear tx descriptor ring */
1447 BZERO_SM((void *)(uintptr) di->txd32, (di->ntxd * sizeof(dma32dd_t)));
1448
1449 if ((di->hnddma.dmactrlflags & DMA_CTRL_PEN) == 0)
1450 control |= XC_PD;
1451 W_REG(di->osh, &di->d32txregs->control, control);
1452 _dma_ddtable_init(di, DMA_TX, di->txdpa);
1453 }
1454
1455 static bool dma32_txenabled(dma_info_t *di)
1456 {
1457 uint32 xc;
1458
1459 /* If the chip is dead, it is not enabled :-) */
1460 xc = R_REG(di->osh, &di->d32txregs->control);
1461 return (xc != 0xffffffff) && (xc & XC_XE);
1462 }
1463
1464 static void dma32_txsuspend(dma_info_t *di)
1465 {
1466 DMA_TRACE(("%s: dma_txsuspend\n", di->name));
1467
1468 if (di->ntxd == 0)
1469 return;
1470
1471 OR_REG(di->osh, &di->d32txregs->control, XC_SE);
1472 }
1473
1474 static void dma32_txresume(dma_info_t *di)
1475 {
1476 DMA_TRACE(("%s: dma_txresume\n", di->name));
1477
1478 if (di->ntxd == 0)
1479 return;
1480
1481 AND_REG(di->osh, &di->d32txregs->control, ~XC_SE);
1482 }
1483
1484 static bool dma32_txsuspended(dma_info_t *di)
1485 {
1486 return (di->ntxd == 0)
1487 || ((R_REG(di->osh, &di->d32txregs->control) & XC_SE) == XC_SE);
1488 }
1489
1490 static void dma32_txreclaim(dma_info_t *di, txd_range_t range)
1491 {
1492 void *p;
1493
1494 DMA_TRACE(("%s: dma_txreclaim %s\n", di->name,
1495 (range == HNDDMA_RANGE_ALL) ? "all" :
1496 ((range ==
1497 HNDDMA_RANGE_TRANSMITTED) ? "transmitted" :
1498 "transfered")));
1499
1500 if (di->txin == di->txout)
1501 return;
1502
1503 while ((p = dma32_getnexttxp(di, range)))
1504 PKTFREE(di->osh, p, TRUE);
1505 }
1506
1507 static bool dma32_txstopped(dma_info_t *di)
1508 {
1509 return ((R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK) ==
1510 XS_XS_STOPPED);
1511 }
1512
1513 static bool dma32_rxstopped(dma_info_t *di)
1514 {
1515 return ((R_REG(di->osh, &di->d32rxregs->status) & RS_RS_MASK) ==
1516 RS_RS_STOPPED);
1517 }
1518
1519 static bool dma32_alloc(dma_info_t *di, uint direction)
1520 {
1521 uint size;
1522 uint ddlen;
1523 void *va;
1524 uint alloced;
1525 uint16 align;
1526 uint16 align_bits;
1527
1528 ddlen = sizeof(dma32dd_t);
1529
1530 size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);
1531
1532 alloced = 0;
1533 align_bits = di->dmadesc_align;
1534 align = (1 << align_bits);
1535
1536 if (direction == DMA_TX) {
1537 va = dma_ringalloc(di->osh, D32RINGALIGN, size, &align_bits,
1538 &alloced, &di->txdpaorig, &di->tx_dmah);
1539 if (va == NULL) {
1540 DMA_ERROR(("%s: dma_alloc: DMA_ALLOC_CONSISTENT(ntxd) failed\n", di->name));
1541 return FALSE;
1542 }
1543
1544 PHYSADDRHISET(di->txdpa, 0);
1545 ASSERT(PHYSADDRHI(di->txdpaorig) == 0);
1546 di->txd32 = (dma32dd_t *) ROUNDUP((uintptr) va, align);
1547 di->txdalign =
1548 (uint) ((int8 *) (uintptr) di->txd32 - (int8 *) va);
1549
1550 PHYSADDRLOSET(di->txdpa,
1551 PHYSADDRLO(di->txdpaorig) + di->txdalign);
1552 /* Make sure that alignment didn't overflow */
1553 ASSERT(PHYSADDRLO(di->txdpa) >= PHYSADDRLO(di->txdpaorig));
1554
1555 di->txdalloc = alloced;
1556 ASSERT(ISALIGNED((uintptr) di->txd32, align));
1557 } else {
1558 va = dma_ringalloc(di->osh, D32RINGALIGN, size, &align_bits,
1559 &alloced, &di->rxdpaorig, &di->rx_dmah);
1560 if (va == NULL) {
1561 DMA_ERROR(("%s: dma_alloc: DMA_ALLOC_CONSISTENT(nrxd) failed\n", di->name));
1562 return FALSE;
1563 }
1564
1565 PHYSADDRHISET(di->rxdpa, 0);
1566 ASSERT(PHYSADDRHI(di->rxdpaorig) == 0);
1567 di->rxd32 = (dma32dd_t *) ROUNDUP((uintptr) va, align);
1568 di->rxdalign =
1569 (uint) ((int8 *) (uintptr) di->rxd32 - (int8 *) va);
1570
1571 PHYSADDRLOSET(di->rxdpa,
1572 PHYSADDRLO(di->rxdpaorig) + di->rxdalign);
1573 /* Make sure that alignment didn't overflow */
1574 ASSERT(PHYSADDRLO(di->rxdpa) >= PHYSADDRLO(di->rxdpaorig));
1575 di->rxdalloc = alloced;
1576 ASSERT(ISALIGNED((uintptr) di->rxd32, align));
1577 }
1578
1579 return TRUE;
1580 }
1581
1582 static bool dma32_txreset(dma_info_t *di)
1583 {
1584 uint32 status;
1585
1586 if (di->ntxd == 0)
1587 return TRUE;
1588
1589 /* suspend tx DMA first */
1590 W_REG(di->osh, &di->d32txregs->control, XC_SE);
1591 SPINWAIT(((status =
1592 (R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK))
1593 != XS_XS_DISABLED) && (status != XS_XS_IDLE)
1594 && (status != XS_XS_STOPPED), (10000));
1595
1596 W_REG(di->osh, &di->d32txregs->control, 0);
1597 SPINWAIT(((status = (R_REG(di->osh,
1598 &di->d32txregs->status) & XS_XS_MASK)) !=
1599 XS_XS_DISABLED), 10000);
1600
1601 /* wait for the last transaction to complete */
1602 OSL_DELAY(300);
1603
1604 return status == XS_XS_DISABLED;
1605 }
1606
1607 static bool dma32_rxidle(dma_info_t *di)
1608 {
1609 DMA_TRACE(("%s: dma_rxidle\n", di->name));
1610
1611 if (di->nrxd == 0)
1612 return TRUE;
1613
1614 return ((R_REG(di->osh, &di->d32rxregs->status) & RS_CD_MASK) ==
1615 R_REG(di->osh, &di->d32rxregs->ptr));
1616 }
1617
1618 static bool dma32_rxreset(dma_info_t *di)
1619 {
1620 uint32 status;
1621
1622 if (di->nrxd == 0)
1623 return TRUE;
1624
1625 W_REG(di->osh, &di->d32rxregs->control, 0);
1626 SPINWAIT(((status = (R_REG(di->osh,
1627 &di->d32rxregs->status) & RS_RS_MASK)) !=
1628 RS_RS_DISABLED), 10000);
1629
1630 return status == RS_RS_DISABLED;
1631 }
1632
1633 static bool dma32_rxenabled(dma_info_t *di)
1634 {
1635 uint32 rc;
1636
1637 rc = R_REG(di->osh, &di->d32rxregs->control);
1638 return (rc != 0xffffffff) && (rc & RC_RE);
1639 }
1640
1641 static bool dma32_txsuspendedidle(dma_info_t *di)
1642 {
1643 if (di->ntxd == 0)
1644 return TRUE;
1645
1646 if (!(R_REG(di->osh, &di->d32txregs->control) & XC_SE))
1647 return 0;
1648
1649 if ((R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK) != XS_XS_IDLE)
1650 return 0;
1651
1652 OSL_DELAY(2);
1653 return ((R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK) ==
1654 XS_XS_IDLE);
1655 }
1656
1657 /* !! tx entry routine
1658 * supports full 32bit dma engine buffer addressing so
1659 * dma buffers can cross 4 Kbyte page boundaries.
1660 *
1661 * WARNING: call must check the return value for error.
1662 * the error(toss frames) could be fatal and cause many subsequent hard to debug problems
1663 */
1664 static int dma32_txfast(dma_info_t *di, void *p0, bool commit)
1665 {
1666 void *p, *next;
1667 uchar *data;
1668 uint len;
1669 uint16 txout;
1670 uint32 flags = 0;
1671 dmaaddr_t pa;
1672
1673 DMA_TRACE(("%s: dma_txfast\n", di->name));
1674
1675 txout = di->txout;
1676
1677 /*
1678 * Walk the chain of packet buffers
1679 * allocating and initializing transmit descriptor entries.
1680 */
1681 for (p = p0; p; p = next) {
1682 uint nsegs, j;
1683 hnddma_seg_map_t *map;
1684
1685 data = PKTDATA(p);
1686 len = PKTLEN(p);
1687 #ifdef BCM_DMAPAD
1688 len += PKTDMAPAD(di->osh, p);
1689 #endif
1690 next = PKTNEXT(p);
1691
1692 /* return nonzero if out of tx descriptors */
1693 if (NEXTTXD(txout) == di->txin)
1694 goto outoftxd;
1695
1696 if (len == 0)
1697 continue;
1698
1699 if (DMASGLIST_ENAB)
1700 bzero(&di->txp_dmah[txout], sizeof(hnddma_seg_map_t));
1701
1702 /* get physical address of buffer start */
1703 pa = DMA_MAP(di->osh, data, len, DMA_TX, p,
1704 &di->txp_dmah[txout]);
1705
1706 if (DMASGLIST_ENAB) {
1707 map = &di->txp_dmah[txout];
1708
1709 /* See if all the segments can be accounted for */
1710 if (map->nsegs >
1711 (uint) (di->ntxd - NTXDACTIVE(di->txin, di->txout) -
1712 1))
1713 goto outoftxd;
1714
1715 nsegs = map->nsegs;
1716 } else
1717 nsegs = 1;
1718
1719 for (j = 1; j <= nsegs; j++) {
1720 flags = 0;
1721 if (p == p0 && j == 1)
1722 flags |= CTRL_SOF;
1723
1724 /* With a DMA segment list, Descriptor table is filled
1725 * using the segment list instead of looping over
1726 * buffers in multi-chain DMA. Therefore, EOF for SGLIST is when
1727 * end of segment list is reached.
1728 */
1729 if ((!DMASGLIST_ENAB && next == NULL) ||
1730 (DMASGLIST_ENAB && j == nsegs))
1731 flags |= (CTRL_IOC | CTRL_EOF);
1732 if (txout == (di->ntxd - 1))
1733 flags |= CTRL_EOT;
1734
1735 if (DMASGLIST_ENAB) {
1736 len = map->segs[j - 1].length;
1737 pa = map->segs[j - 1].addr;
1738 }
1739 ASSERT(PHYSADDRHI(pa) == 0);
1740
1741 dma32_dd_upd(di, di->txd32, pa, txout, &flags, len);
1742 ASSERT(di->txp[txout] == NULL);
1743
1744 txout = NEXTTXD(txout);
1745 }
1746
1747 /* See above. No need to loop over individual buffers */
1748 if (DMASGLIST_ENAB)
1749 break;
1750 }
1751
1752 /* if last txd eof not set, fix it */
1753 if (!(flags & CTRL_EOF))
1754 W_SM(&di->txd32[PREVTXD(txout)].ctrl,
1755 BUS_SWAP32(flags | CTRL_IOC | CTRL_EOF));
1756
1757 /* save the packet */
1758 di->txp[PREVTXD(txout)] = p0;
1759
1760 /* bump the tx descriptor index */
1761 di->txout = txout;
1762
1763 /* kick the chip */
1764 if (commit)
1765 W_REG(di->osh, &di->d32txregs->ptr, I2B(txout, dma32dd_t));
1766
1767 /* tx flow control */
1768 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1769
1770 return 0;
1771
1772 outoftxd:
1773 DMA_ERROR(("%s: dma_txfast: out of txds\n", di->name));
1774 PKTFREE(di->osh, p0, TRUE);
1775 di->hnddma.txavail = 0;
1776 di->hnddma.txnobuf++;
1777 return -1;
1778 }
1779
1780 /*
1781 * Reclaim next completed txd (txds if using chained buffers) in the range
1782 * specified and return associated packet.
1783 * If range is HNDDMA_RANGE_TRANSMITTED, reclaim descriptors that have be
1784 * transmitted as noted by the hardware "CurrDescr" pointer.
1785 * If range is HNDDMA_RANGE_TRANSFERED, reclaim descriptors that have be
1786 * transfered by the DMA as noted by the hardware "ActiveDescr" pointer.
1787 * If range is HNDDMA_RANGE_ALL, reclaim all txd(s) posted to the ring and
1788 * return associated packet regardless of the value of hardware pointers.
1789 */
1790 static void *dma32_getnexttxp(dma_info_t *di, txd_range_t range)
1791 {
1792 uint16 start, end, i;
1793 uint16 active_desc;
1794 void *txp;
1795
1796 DMA_TRACE(("%s: dma_getnexttxp %s\n", di->name,
1797 (range == HNDDMA_RANGE_ALL) ? "all" :
1798 ((range ==
1799 HNDDMA_RANGE_TRANSMITTED) ? "transmitted" :
1800 "transfered")));
1801
1802 if (di->ntxd == 0)
1803 return NULL;
1804
1805 txp = NULL;
1806
1807 start = di->txin;
1808 if (range == HNDDMA_RANGE_ALL)
1809 end = di->txout;
1810 else {
1811 dma32regs_t *dregs = di->d32txregs;
1812
1813 end =
1814 (uint16) B2I(R_REG(di->osh, &dregs->status) & XS_CD_MASK,
1815 dma32dd_t);
1816
1817 if (range == HNDDMA_RANGE_TRANSFERED) {
1818 active_desc =
1819 (uint16) ((R_REG(di->osh, &dregs->status) &
1820 XS_AD_MASK) >> XS_AD_SHIFT);
1821 active_desc = (uint16) B2I(active_desc, dma32dd_t);
1822 if (end != active_desc)
1823 end = PREVTXD(active_desc);
1824 }
1825 }
1826
1827 if ((start == 0) && (end > di->txout))
1828 goto bogus;
1829
1830 for (i = start; i != end && !txp; i = NEXTTXD(i)) {
1831 dmaaddr_t pa;
1832 hnddma_seg_map_t *map = NULL;
1833 uint size, j, nsegs;
1834
1835 PHYSADDRLOSET(pa,
1836 (BUS_SWAP32(R_SM(&di->txd32[i].addr)) -
1837 di->dataoffsetlow));
1838 PHYSADDRHISET(pa, 0);
1839
1840 if (DMASGLIST_ENAB) {
1841 map = &di->txp_dmah[i];
1842 size = map->origsize;
1843 nsegs = map->nsegs;
1844 } else {
1845 size =
1846 (BUS_SWAP32(R_SM(&di->txd32[i].ctrl)) &
1847 CTRL_BC_MASK);
1848 nsegs = 1;
1849 }
1850
1851 for (j = nsegs; j > 0; j--) {
1852 W_SM(&di->txd32[i].addr, 0xdeadbeef);
1853
1854 txp = di->txp[i];
1855 di->txp[i] = NULL;
1856 if (j > 1)
1857 i = NEXTTXD(i);
1858 }
1859
1860 DMA_UNMAP(di->osh, pa, size, DMA_TX, txp, map);
1861 }
1862
1863 di->txin = i;
1864
1865 /* tx flow control */
1866 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1867
1868 return txp;
1869
1870 bogus:
1871 DMA_NONE(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n", start, end, di->txout, forceall));
1872 return NULL;
1873 }
1874
1875 static void *dma32_getnextrxp(dma_info_t *di, bool forceall)
1876 {
1877 uint i, curr;
1878 void *rxp;
1879 dmaaddr_t pa;
1880 /* if forcing, dma engine must be disabled */
1881 ASSERT(!forceall || !dma32_rxenabled(di));
1882
1883 i = di->rxin;
1884
1885 /* return if no packets posted */
1886 if (i == di->rxout)
1887 return NULL;
1888
1889 curr =
1890 B2I(R_REG(di->osh, &di->d32rxregs->status) & RS_CD_MASK, dma32dd_t);
1891
1892 /* ignore curr if forceall */
1893 if (!forceall && (i == curr))
1894 return NULL;
1895
1896 /* get the packet pointer that corresponds to the rx descriptor */
1897 rxp = di->rxp[i];
1898 ASSERT(rxp);
1899 di->rxp[i] = NULL;
1900
1901 PHYSADDRLOSET(pa,
1902 (BUS_SWAP32(R_SM(&di->rxd32[i].addr)) -
1903 di->dataoffsetlow));
1904 PHYSADDRHISET(pa, 0);
1905
1906 /* clear this packet from the descriptor ring */
1907 DMA_UNMAP(di->osh, pa, di->rxbufsize, DMA_RX, rxp, &di->rxp_dmah[i]);
1908
1909 W_SM(&di->rxd32[i].addr, 0xdeadbeef);
1910
1911 di->rxin = NEXTRXD(i);
1912
1913 return rxp;
1914 }
1915
1916 /*
1917 * Rotate all active tx dma ring entries "forward" by (ActiveDescriptor - txin).
1918 */
1919 static void dma32_txrotate(dma_info_t *di)
1920 {
1921 uint16 ad;
1922 uint nactive;
1923 uint rot;
1924 uint16 old, new;
1925 uint32 w;
1926 uint16 first, last;
1927
1928 ASSERT(dma32_txsuspendedidle(di));
1929
1930 nactive = _dma_txactive(di);
1931 ad = (uint16) (B2I
1932 (((R_REG(di->osh, &di->d32txregs->status) & XS_AD_MASK)
1933 >> XS_AD_SHIFT), dma32dd_t));
1934 rot = TXD(ad - di->txin);
1935
1936 ASSERT(rot < di->ntxd);
1937
1938 /* full-ring case is a lot harder - don't worry about this */
1939 if (rot >= (di->ntxd - nactive)) {
1940 DMA_ERROR(("%s: dma_txrotate: ring full - punt\n", di->name));
1941 return;
1942 }
1943
1944 first = di->txin;
1945 last = PREVTXD(di->txout);
1946
1947 /* move entries starting at last and moving backwards to first */
1948 for (old = last; old != PREVTXD(first); old = PREVTXD(old)) {
1949 new = TXD(old + rot);
1950
1951 /*
1952 * Move the tx dma descriptor.
1953 * EOT is set only in the last entry in the ring.
1954 */
1955 w = BUS_SWAP32(R_SM(&di->txd32[old].ctrl)) & ~CTRL_EOT;
1956 if (new == (di->ntxd - 1))
1957 w |= CTRL_EOT;
1958 W_SM(&di->txd32[new].ctrl, BUS_SWAP32(w));
1959 W_SM(&di->txd32[new].addr, R_SM(&di->txd32[old].addr));
1960
1961 /* zap the old tx dma descriptor address field */
1962 W_SM(&di->txd32[old].addr, BUS_SWAP32(0xdeadbeef));
1963
1964 /* move the corresponding txp[] entry */
1965 ASSERT(di->txp[new] == NULL);
1966 di->txp[new] = di->txp[old];
1967
1968 /* Move the segment map as well */
1969 if (DMASGLIST_ENAB) {
1970 bcopy(&di->txp_dmah[old], &di->txp_dmah[new],
1971 sizeof(hnddma_seg_map_t));
1972 bzero(&di->txp_dmah[old], sizeof(hnddma_seg_map_t));
1973 }
1974
1975 di->txp[old] = NULL;
1976 }
1977
1978 /* update txin and txout */
1979 di->txin = ad;
1980 di->txout = TXD(di->txout + rot);
1981 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1982
1983 /* kick the chip */
1984 W_REG(di->osh, &di->d32txregs->ptr, I2B(di->txout, dma32dd_t));
1985 }
1986
1987 /* 64-bit DMA functions */
1988
1989 static void dma64_txinit(dma_info_t *di)
1990 {
1991 uint32 control = D64_XC_XE;
1992
1993 DMA_TRACE(("%s: dma_txinit\n", di->name));
1994
1995 if (di->ntxd == 0)
1996 return;
1997
1998 di->txin = di->txout = 0;
1999 di->hnddma.txavail = di->ntxd - 1;
2000
2001 /* clear tx descriptor ring */
2002 BZERO_SM((void *)(uintptr) di->txd64, (di->ntxd * sizeof(dma64dd_t)));
2003
2004 /* DMA engine with out alignment requirement requires table to be inited
2005 * before enabling the engine
2006 */
2007 if (!di->aligndesc_4k)
2008 _dma_ddtable_init(di, DMA_TX, di->txdpa);
2009
2010 if ((di->hnddma.dmactrlflags & DMA_CTRL_PEN) == 0)
2011 control |= D64_XC_PD;
2012 OR_REG(di->osh, &di->d64txregs->control, control);
2013
2014 /* DMA engine with alignment requirement requires table to be inited
2015 * before enabling the engine
2016 */
2017 if (di->aligndesc_4k)
2018 _dma_ddtable_init(di, DMA_TX, di->txdpa);
2019 }
2020
2021 static bool dma64_txenabled(dma_info_t *di)
2022 {
2023 uint32 xc;
2024
2025 /* If the chip is dead, it is not enabled :-) */
2026 xc = R_REG(di->osh, &di->d64txregs->control);
2027 return (xc != 0xffffffff) && (xc & D64_XC_XE);
2028 }
2029
2030 static void dma64_txsuspend(dma_info_t *di)
2031 {
2032 DMA_TRACE(("%s: dma_txsuspend\n", di->name));
2033
2034 if (di->ntxd == 0)
2035 return;
2036
2037 OR_REG(di->osh, &di->d64txregs->control, D64_XC_SE);
2038 }
2039
2040 static void dma64_txresume(dma_info_t *di)
2041 {
2042 DMA_TRACE(("%s: dma_txresume\n", di->name));
2043
2044 if (di->ntxd == 0)
2045 return;
2046
2047 AND_REG(di->osh, &di->d64txregs->control, ~D64_XC_SE);
2048 }
2049
2050 static bool dma64_txsuspended(dma_info_t *di)
2051 {
2052 return (di->ntxd == 0) ||
2053 ((R_REG(di->osh, &di->d64txregs->control) & D64_XC_SE) ==
2054 D64_XC_SE);
2055 }
2056
2057 static void BCMFASTPATH dma64_txreclaim(dma_info_t *di, txd_range_t range)
2058 {
2059 void *p;
2060
2061 DMA_TRACE(("%s: dma_txreclaim %s\n", di->name,
2062 (range == HNDDMA_RANGE_ALL) ? "all" :
2063 ((range ==
2064 HNDDMA_RANGE_TRANSMITTED) ? "transmitted" :
2065 "transfered")));
2066
2067 if (di->txin == di->txout)
2068 return;
2069
2070 while ((p = dma64_getnexttxp(di, range))) {
2071 /* For unframed data, we don't have any packets to free */
2072 if (!(di->hnddma.dmactrlflags & DMA_CTRL_UNFRAMED))
2073 PKTFREE(di->osh, p, TRUE);
2074 }
2075 }
2076
2077 static bool dma64_txstopped(dma_info_t *di)
2078 {
2079 return ((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK) ==
2080 D64_XS0_XS_STOPPED);
2081 }
2082
2083 static bool dma64_rxstopped(dma_info_t *di)
2084 {
2085 return ((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_RS_MASK) ==
2086 D64_RS0_RS_STOPPED);
2087 }
2088
2089 static bool dma64_alloc(dma_info_t *di, uint direction)
2090 {
2091 uint16 size;
2092 uint ddlen;
2093 void *va;
2094 uint alloced = 0;
2095 uint16 align;
2096 uint16 align_bits;
2097
2098 ddlen = sizeof(dma64dd_t);
2099
2100 size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);
2101 align_bits = di->dmadesc_align;
2102 align = (1 << align_bits);
2103
2104 if (direction == DMA_TX) {
2105 va = dma_ringalloc(di->osh, D64RINGALIGN, size, &align_bits,
2106 &alloced, &di->txdpaorig, &di->tx_dmah);
2107 if (va == NULL) {
2108 DMA_ERROR(("%s: dma64_alloc: DMA_ALLOC_CONSISTENT(ntxd) failed\n", di->name));
2109 return FALSE;
2110 }
2111 align = (1 << align_bits);
2112 di->txd64 = (dma64dd_t *) ROUNDUP((uintptr) va, align);
2113 di->txdalign =
2114 (uint) ((int8 *) (uintptr) di->txd64 - (int8 *) va);
2115 PHYSADDRLOSET(di->txdpa,
2116 PHYSADDRLO(di->txdpaorig) + di->txdalign);
2117 /* Make sure that alignment didn't overflow */
2118 ASSERT(PHYSADDRLO(di->txdpa) >= PHYSADDRLO(di->txdpaorig));
2119
2120 PHYSADDRHISET(di->txdpa, PHYSADDRHI(di->txdpaorig));
2121 di->txdalloc = alloced;
2122 ASSERT(ISALIGNED((uintptr) di->txd64, align));
2123 } else {
2124 va = dma_ringalloc(di->osh, D64RINGALIGN, size, &align_bits,
2125 &alloced, &di->rxdpaorig, &di->rx_dmah);
2126 if (va == NULL) {
2127 DMA_ERROR(("%s: dma64_alloc: DMA_ALLOC_CONSISTENT(nrxd) failed\n", di->name));
2128 return FALSE;
2129 }
2130 align = (1 << align_bits);
2131 di->rxd64 = (dma64dd_t *) ROUNDUP((uintptr) va, align);
2132 di->rxdalign =
2133 (uint) ((int8 *) (uintptr) di->rxd64 - (int8 *) va);
2134 PHYSADDRLOSET(di->rxdpa,
2135 PHYSADDRLO(di->rxdpaorig) + di->rxdalign);
2136 /* Make sure that alignment didn't overflow */
2137 ASSERT(PHYSADDRLO(di->rxdpa) >= PHYSADDRLO(di->rxdpaorig));
2138
2139 PHYSADDRHISET(di->rxdpa, PHYSADDRHI(di->rxdpaorig));
2140 di->rxdalloc = alloced;
2141 ASSERT(ISALIGNED((uintptr) di->rxd64, align));
2142 }
2143
2144 return TRUE;
2145 }
2146
2147 static bool dma64_txreset(dma_info_t *di)
2148 {
2149 uint32 status;
2150
2151 if (di->ntxd == 0)
2152 return TRUE;
2153
2154 /* suspend tx DMA first */
2155 W_REG(di->osh, &di->d64txregs->control, D64_XC_SE);
2156 SPINWAIT(((status =
2157 (R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK))
2158 != D64_XS0_XS_DISABLED) && (status != D64_XS0_XS_IDLE)
2159 && (status != D64_XS0_XS_STOPPED), 10000);
2160
2161 W_REG(di->osh, &di->d64txregs->control, 0);
2162 SPINWAIT(((status =
2163 (R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK))
2164 != D64_XS0_XS_DISABLED), 10000);
2165
2166 /* wait for the last transaction to complete */
2167 OSL_DELAY(300);
2168
2169 return status == D64_XS0_XS_DISABLED;
2170 }
2171
2172 static bool dma64_rxidle(dma_info_t *di)
2173 {
2174 DMA_TRACE(("%s: dma_rxidle\n", di->name));
2175
2176 if (di->nrxd == 0)
2177 return TRUE;
2178
2179 return ((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) ==
2180 (R_REG(di->osh, &di->d64rxregs->ptr) & D64_RS0_CD_MASK));
2181 }
2182
2183 static bool dma64_rxreset(dma_info_t *di)
2184 {
2185 uint32 status;
2186
2187 if (di->nrxd == 0)
2188 return TRUE;
2189
2190 W_REG(di->osh, &di->d64rxregs->control, 0);
2191 SPINWAIT(((status =
2192 (R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_RS_MASK))
2193 != D64_RS0_RS_DISABLED), 10000);
2194
2195 return status == D64_RS0_RS_DISABLED;
2196 }
2197
2198 static bool dma64_rxenabled(dma_info_t *di)
2199 {
2200 uint32 rc;
2201
2202 rc = R_REG(di->osh, &di->d64rxregs->control);
2203 return (rc != 0xffffffff) && (rc & D64_RC_RE);
2204 }
2205
2206 static bool dma64_txsuspendedidle(dma_info_t *di)
2207 {
2208
2209 if (di->ntxd == 0)
2210 return TRUE;
2211
2212 if (!(R_REG(di->osh, &di->d64txregs->control) & D64_XC_SE))
2213 return 0;
2214
2215 if ((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK) ==
2216 D64_XS0_XS_IDLE)
2217 return 1;
2218
2219 return 0;
2220 }
2221
2222 /* Useful when sending unframed data. This allows us to get a progress report from the DMA.
2223 * We return a pointer to the beginning of the DATA buffer of the current descriptor.
2224 * If DMA is idle, we return NULL.
2225 */
2226 static void *dma64_getpos(dma_info_t *di, bool direction)
2227 {
2228 void *va;
2229 bool idle;
2230 uint32 cd_offset;
2231
2232 if (direction == DMA_TX) {
2233 cd_offset =
2234 R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_CD_MASK;
2235 idle = !NTXDACTIVE(di->txin, di->txout);
2236 va = di->txp[B2I(cd_offset, dma64dd_t)];
2237 } else {
2238 cd_offset =
2239 R_REG(di->osh, &di->d64rxregs->status0) & D64_XS0_CD_MASK;
2240 idle = !NRXDACTIVE(di->rxin, di->rxout);
2241 va = di->rxp[B2I(cd_offset, dma64dd_t)];
2242 }
2243
2244 /* If DMA is IDLE, return NULL */
2245 if (idle) {
2246 DMA_TRACE(("%s: DMA idle, return NULL\n", __func__));
2247 va = NULL;
2248 }
2249
2250 return va;
2251 }
2252
2253 /* TX of unframed data
2254 *
2255 * Adds a DMA ring descriptor for the data pointed to by "buf".
2256 * This is for DMA of a buffer of data and is unlike other hnddma TX functions
2257 * that take a pointer to a "packet"
2258 * Each call to this is results in a single descriptor being added for "len" bytes of
2259 * data starting at "buf", it doesn't handle chained buffers.
2260 */
2261 static int dma64_txunframed(dma_info_t *di, void *buf, uint len, bool commit)
2262 {
2263 uint16 txout;
2264 uint32 flags = 0;
2265 dmaaddr_t pa; /* phys addr */
2266
2267 txout = di->txout;
2268
2269 /* return nonzero if out of tx descriptors */
2270 if (NEXTTXD(txout) == di->txin)
2271 goto outoftxd;
2272
2273 if (len == 0)
2274 return 0;
2275
2276 pa = DMA_MAP(di->osh, buf, len, DMA_TX, NULL, &di->txp_dmah[txout]);
2277
2278 flags = (D64_CTRL1_SOF | D64_CTRL1_IOC | D64_CTRL1_EOF);
2279
2280 if (txout == (di->ntxd - 1))
2281 flags |= D64_CTRL1_EOT;
2282
2283 dma64_dd_upd(di, di->txd64, pa, txout, &flags, len);
2284 ASSERT(di->txp[txout] == NULL);
2285
2286 /* save the buffer pointer - used by dma_getpos */
2287 di->txp[txout] = buf;
2288
2289 txout = NEXTTXD(txout);
2290 /* bump the tx descriptor index */
2291 di->txout = txout;
2292
2293 /* kick the chip */
2294 if (commit) {
2295 W_REG(di->osh, &di->d64txregs->ptr,
2296 di->xmtptrbase + I2B(txout, dma64dd_t));
2297 }
2298
2299 /* tx flow control */
2300 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
2301
2302 return 0;
2303
2304 outoftxd:
2305 DMA_ERROR(("%s: %s: out of txds !!!\n", di->name, __func__));
2306 di->hnddma.txavail = 0;
2307 di->hnddma.txnobuf++;
2308 return -1;
2309 }
2310
2311 /* !! tx entry routine
2312 * WARNING: call must check the return value for error.
2313 * the error(toss frames) could be fatal and cause many subsequent hard to debug problems
2314 */
2315 static int BCMFASTPATH dma64_txfast(dma_info_t *di, void *p0, bool commit)
2316 {
2317 void *p, *next;
2318 uchar *data;
2319 uint len;
2320 uint16 txout;
2321 uint32 flags = 0;
2322 dmaaddr_t pa;
2323
2324 DMA_TRACE(("%s: dma_txfast\n", di->name));
2325
2326 txout = di->txout;
2327
2328 /*
2329 * Walk the chain of packet buffers
2330 * allocating and initializing transmit descriptor entries.
2331 */
2332 for (p = p0; p; p = next) {
2333 uint nsegs, j;
2334 hnddma_seg_map_t *map;
2335
2336 data = PKTDATA(p);
2337 len = PKTLEN(p);
2338 #ifdef BCM_DMAPAD
2339 len += PKTDMAPAD(di->osh, p);
2340 #endif /* BCM_DMAPAD */
2341 next = PKTNEXT(p);
2342
2343 /* return nonzero if out of tx descriptors */
2344 if (NEXTTXD(txout) == di->txin)
2345 goto outoftxd;
2346
2347 if (len == 0)
2348 continue;
2349
2350 /* get physical address of buffer start */
2351 if (DMASGLIST_ENAB)
2352 bzero(&di->txp_dmah[txout], sizeof(hnddma_seg_map_t));
2353
2354 pa = DMA_MAP(di->osh, data, len, DMA_TX, p,
2355 &di->txp_dmah[txout]);
2356
2357 if (DMASGLIST_ENAB) {
2358 map = &di->txp_dmah[txout];
2359
2360 /* See if all the segments can be accounted for */
2361 if (map->nsegs >
2362 (uint) (di->ntxd - NTXDACTIVE(di->txin, di->txout) -
2363 1))
2364 goto outoftxd;
2365
2366 nsegs = map->nsegs;
2367 } else
2368 nsegs = 1;
2369
2370 for (j = 1; j <= nsegs; j++) {
2371 flags = 0;
2372 if (p == p0 && j == 1)
2373 flags |= D64_CTRL1_SOF;
2374
2375 /* With a DMA segment list, Descriptor table is filled
2376 * using the segment list instead of looping over
2377 * buffers in multi-chain DMA. Therefore, EOF for SGLIST is when
2378 * end of segment list is reached.
2379 */
2380 if ((!DMASGLIST_ENAB && next == NULL) ||
2381 (DMASGLIST_ENAB && j == nsegs))
2382 flags |= (D64_CTRL1_IOC | D64_CTRL1_EOF);
2383 if (txout == (di->ntxd - 1))
2384 flags |= D64_CTRL1_EOT;
2385
2386 if (DMASGLIST_ENAB) {
2387 len = map->segs[j - 1].length;
2388 pa = map->segs[j - 1].addr;
2389 }
2390 dma64_dd_upd(di, di->txd64, pa, txout, &flags, len);
2391 ASSERT(di->txp[txout] == NULL);
2392
2393 txout = NEXTTXD(txout);
2394 }
2395
2396 /* See above. No need to loop over individual buffers */
2397 if (DMASGLIST_ENAB)
2398 break;
2399 }
2400
2401 /* if last txd eof not set, fix it */
2402 if (!(flags & D64_CTRL1_EOF))
2403 W_SM(&di->txd64[PREVTXD(txout)].ctrl1,
2404 BUS_SWAP32(flags | D64_CTRL1_IOC | D64_CTRL1_EOF));
2405
2406 /* save the packet */
2407 di->txp[PREVTXD(txout)] = p0;
2408
2409 /* bump the tx descriptor index */
2410 di->txout = txout;
2411
2412 /* kick the chip */
2413 if (commit)
2414 W_REG(di->osh, &di->d64txregs->ptr,
2415 di->xmtptrbase + I2B(txout, dma64dd_t));
2416
2417 /* tx flow control */
2418 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
2419
2420 return 0;
2421
2422 outoftxd:
2423 DMA_ERROR(("%s: dma_txfast: out of txds !!!\n", di->name));
2424 PKTFREE(di->osh, p0, TRUE);
2425 di->hnddma.txavail = 0;
2426 di->hnddma.txnobuf++;
2427 return -1;
2428 }
2429
2430 /*
2431 * Reclaim next completed txd (txds if using chained buffers) in the range
2432 * specified and return associated packet.
2433 * If range is HNDDMA_RANGE_TRANSMITTED, reclaim descriptors that have be
2434 * transmitted as noted by the hardware "CurrDescr" pointer.
2435 * If range is HNDDMA_RANGE_TRANSFERED, reclaim descriptors that have be
2436 * transfered by the DMA as noted by the hardware "ActiveDescr" pointer.
2437 * If range is HNDDMA_RANGE_ALL, reclaim all txd(s) posted to the ring and
2438 * return associated packet regardless of the value of hardware pointers.
2439 */
2440 static void *BCMFASTPATH dma64_getnexttxp(dma_info_t *di, txd_range_t range)
2441 {
2442 uint16 start, end, i;
2443 uint16 active_desc;
2444 void *txp;
2445
2446 DMA_TRACE(("%s: dma_getnexttxp %s\n", di->name,
2447 (range == HNDDMA_RANGE_ALL) ? "all" :
2448 ((range ==
2449 HNDDMA_RANGE_TRANSMITTED) ? "transmitted" :
2450 "transfered")));
2451
2452 if (di->ntxd == 0)
2453 return NULL;
2454
2455 txp = NULL;
2456
2457 start = di->txin;
2458 if (range == HNDDMA_RANGE_ALL)
2459 end = di->txout;
2460 else {
2461 dma64regs_t *dregs = di->d64txregs;
2462
2463 end =
2464 (uint16) (B2I
2465 (((R_REG(di->osh, &dregs->status0) &
2466 D64_XS0_CD_MASK) -
2467 di->xmtptrbase) & D64_XS0_CD_MASK, dma64dd_t));
2468
2469 if (range == HNDDMA_RANGE_TRANSFERED) {
2470 active_desc =
2471 (uint16) (R_REG(di->osh, &dregs->status1) &
2472 D64_XS1_AD_MASK);
2473 active_desc =
2474 (active_desc - di->xmtptrbase) & D64_XS0_CD_MASK;
2475 active_desc = B2I(active_desc, dma64dd_t);
2476 if (end != active_desc)
2477 end = PREVTXD(active_desc);
2478 }
2479 }
2480
2481 if ((start == 0) && (end > di->txout))
2482 goto bogus;
2483
2484 for (i = start; i != end && !txp; i = NEXTTXD(i)) {
2485 dmaaddr_t pa;
2486 hnddma_seg_map_t *map = NULL;
2487 uint size, j, nsegs;
2488
2489 PHYSADDRLOSET(pa,
2490 (BUS_SWAP32(R_SM(&di->txd64[i].addrlow)) -
2491 di->dataoffsetlow));
2492 PHYSADDRHISET(pa,
2493 (BUS_SWAP32(R_SM(&di->txd64[i].addrhigh)) -
2494 di->dataoffsethigh));
2495
2496 if (DMASGLIST_ENAB) {
2497 map = &di->txp_dmah[i];
2498 size = map->origsize;
2499 nsegs = map->nsegs;
2500 } else {
2501 size =
2502 (BUS_SWAP32(R_SM(&di->txd64[i].ctrl2)) &
2503 D64_CTRL2_BC_MASK);
2504 nsegs = 1;
2505 }
2506
2507 for (j = nsegs; j > 0; j--) {
2508 W_SM(&di->txd64[i].addrlow, 0xdeadbeef);
2509 W_SM(&di->txd64[i].addrhigh, 0xdeadbeef);
2510
2511 txp = di->txp[i];
2512 di->txp[i] = NULL;
2513 if (j > 1)
2514 i = NEXTTXD(i);
2515 }
2516
2517 DMA_UNMAP(di->osh, pa, size, DMA_TX, txp, map);
2518 }
2519
2520 di->txin = i;
2521
2522 /* tx flow control */
2523 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
2524
2525 return txp;
2526
2527 bogus:
2528 DMA_NONE(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n", start, end, di->txout, forceall));
2529 return NULL;
2530 }
2531
2532 static void *BCMFASTPATH dma64_getnextrxp(dma_info_t *di, bool forceall)
2533 {
2534 uint i, curr;
2535 void *rxp;
2536 dmaaddr_t pa;
2537
2538 /* if forcing, dma engine must be disabled */
2539 ASSERT(!forceall || !dma64_rxenabled(di));
2540
2541 i = di->rxin;
2542
2543 /* return if no packets posted */
2544 if (i == di->rxout)
2545 return NULL;
2546
2547 curr =
2548 B2I(((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) -
2549 di->rcvptrbase) & D64_RS0_CD_MASK, dma64dd_t);
2550
2551 /* ignore curr if forceall */
2552 if (!forceall && (i == curr))
2553 return NULL;
2554
2555 /* get the packet pointer that corresponds to the rx descriptor */
2556 rxp = di->rxp[i];
2557 ASSERT(rxp);
2558 di->rxp[i] = NULL;
2559
2560 PHYSADDRLOSET(pa,
2561 (BUS_SWAP32(R_SM(&di->rxd64[i].addrlow)) -
2562 di->dataoffsetlow));
2563 PHYSADDRHISET(pa,
2564 (BUS_SWAP32(R_SM(&di->rxd64[i].addrhigh)) -
2565 di->dataoffsethigh));
2566
2567 /* clear this packet from the descriptor ring */
2568 DMA_UNMAP(di->osh, pa, di->rxbufsize, DMA_RX, rxp, &di->rxp_dmah[i]);
2569
2570 W_SM(&di->rxd64[i].addrlow, 0xdeadbeef);
2571 W_SM(&di->rxd64[i].addrhigh, 0xdeadbeef);
2572
2573 di->rxin = NEXTRXD(i);
2574
2575 return rxp;
2576 }
2577
2578 static bool _dma64_addrext(osl_t *osh, dma64regs_t * dma64regs)
2579 {
2580 uint32 w;
2581 OR_REG(osh, &dma64regs->control, D64_XC_AE);
2582 w = R_REG(osh, &dma64regs->control);
2583 AND_REG(osh, &dma64regs->control, ~D64_XC_AE);
2584 return (w & D64_XC_AE) == D64_XC_AE;
2585 }
2586
2587 /*
2588 * Rotate all active tx dma ring entries "forward" by (ActiveDescriptor - txin).
2589 */
2590 static void dma64_txrotate(dma_info_t *di)
2591 {
2592 uint16 ad;
2593 uint nactive;
2594 uint rot;
2595 uint16 old, new;
2596 uint32 w;
2597 uint16 first, last;
2598
2599 ASSERT(dma64_txsuspendedidle(di));
2600
2601 nactive = _dma_txactive(di);
2602 ad = (uint16) (B2I
2603 ((((R_REG(di->osh, &di->d64txregs->status1) &
2604 D64_XS1_AD_MASK)
2605 - di->xmtptrbase) & D64_XS1_AD_MASK), dma64dd_t));
2606 rot = TXD(ad - di->txin);
2607
2608 ASSERT(rot < di->ntxd);
2609
2610 /* full-ring case is a lot harder - don't worry about this */
2611 if (rot >= (di->ntxd - nactive)) {
2612 DMA_ERROR(("%s: dma_txrotate: ring full - punt\n", di->name));
2613 return;
2614 }
2615
2616 first = di->txin;
2617 last = PREVTXD(di->txout);
2618
2619 /* move entries starting at last and moving backwards to first */
2620 for (old = last; old != PREVTXD(first); old = PREVTXD(old)) {
2621 new = TXD(old + rot);
2622
2623 /*
2624 * Move the tx dma descriptor.
2625 * EOT is set only in the last entry in the ring.
2626 */
2627 w = BUS_SWAP32(R_SM(&di->txd64[old].ctrl1)) & ~D64_CTRL1_EOT;
2628 if (new == (di->ntxd - 1))
2629 w |= D64_CTRL1_EOT;
2630 W_SM(&di->txd64[new].ctrl1, BUS_SWAP32(w));
2631
2632 w = BUS_SWAP32(R_SM(&di->txd64[old].ctrl2));
2633 W_SM(&di->txd64[new].ctrl2, BUS_SWAP32(w));
2634
2635 W_SM(&di->txd64[new].addrlow, R_SM(&di->txd64[old].addrlow));
2636 W_SM(&di->txd64[new].addrhigh, R_SM(&di->txd64[old].addrhigh));
2637
2638 /* zap the old tx dma descriptor address field */
2639 W_SM(&di->txd64[old].addrlow, BUS_SWAP32(0xdeadbeef));
2640 W_SM(&di->txd64[old].addrhigh, BUS_SWAP32(0xdeadbeef));
2641
2642 /* move the corresponding txp[] entry */
2643 ASSERT(di->txp[new] == NULL);
2644 di->txp[new] = di->txp[old];
2645
2646 /* Move the map */
2647 if (DMASGLIST_ENAB) {
2648 bcopy(&di->txp_dmah[old], &di->txp_dmah[new],
2649 sizeof(hnddma_seg_map_t));
2650 bzero(&di->txp_dmah[old], sizeof(hnddma_seg_map_t));
2651 }
2652
2653 di->txp[old] = NULL;
2654 }
2655
2656 /* update txin and txout */
2657 di->txin = ad;
2658 di->txout = TXD(di->txout + rot);
2659 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
2660
2661 /* kick the chip */
2662 W_REG(di->osh, &di->d64txregs->ptr,
2663 di->xmtptrbase + I2B(di->txout, dma64dd_t));
2664 }
2665
2666 uint dma_addrwidth(si_t *sih, void *dmaregs)
2667 {
2668 dma32regs_t *dma32regs;
2669 osl_t *osh;
2670
2671 osh = si_osh(sih);
2672
2673 /* Perform 64-bit checks only if we want to advertise 64-bit (> 32bit) capability) */
2674 /* DMA engine is 64-bit capable */
2675 if ((si_core_sflags(sih, 0, 0) & SISF_DMA64) == SISF_DMA64) {
2676 /* backplane are 64-bit capable */
2677 if (si_backplane64(sih))
2678 /* If bus is System Backplane or PCIE then we can access 64-bits */
2679 if ((BUSTYPE(sih->bustype) == SI_BUS) ||
2680 ((BUSTYPE(sih->bustype) == PCI_BUS) &&
2681 (sih->buscoretype == PCIE_CORE_ID)))
2682 return DMADDRWIDTH_64;
2683
2684 /* DMA64 is always 32-bit capable, AE is always TRUE */
2685 ASSERT(_dma64_addrext(osh, (dma64regs_t *) dmaregs));
2686
2687 return DMADDRWIDTH_32;
2688 }
2689
2690 /* Start checking for 32-bit / 30-bit addressing */
2691 dma32regs = (dma32regs_t *) dmaregs;
2692
2693 /* For System Backplane, PCIE bus or addrext feature, 32-bits ok */
2694 if ((BUSTYPE(sih->bustype) == SI_BUS) ||
2695 ((BUSTYPE(sih->bustype) == PCI_BUS)
2696 && sih->buscoretype == PCIE_CORE_ID)
2697 || (_dma32_addrext(osh, dma32regs)))
2698 return DMADDRWIDTH_32;
2699
2700 /* Fallthru */
2701 return DMADDRWIDTH_30;
2702 }
WandBoard kernel