?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Resync with broadcom drivers 5.100.138.20 and utilities.
[tomato.git] / release / src-rt / shared / hnddma.c
blobb953366c6af7ec34ec816abf7db968ac8837fe1c
1 /*
2 * Generic Broadcom Home Networking Division (HND) DMA module.
3 * This supports the following chips: BCM42xx, 44xx, 47xx .
4 *
5 * Copyright (C) 2010, Broadcom Corporation. All Rights Reserved.
6 *
7 * Permission to use, copy, modify, and/or distribute this software for any
8 * purpose with or without fee is hereby granted, provided that the above
9 * copyright notice and this permission notice appear in all copies.
10 *
11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
14 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
16 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
17 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18 *
19 * $Id: hnddma.c,v 1.231.10.17 2011-01-27 19:03:18 Exp $
20 */
21
22 #include <typedefs.h>
23 #include <bcmdefs.h>
24 #include <bcmdevs.h>
25 #include <osl.h>
26 #include <bcmendian.h>
27 #include <hndsoc.h>
28 #include <bcmutils.h>
29 #include <siutils.h>
30
31 #include <sbhnddma.h>
32 #include <hnddma.h>
33
34 /* debug/trace */
35 #ifdef BCMDBG
36 #define DMA_ERROR(args) if (!(*di->msg_level & 1)); else printf args
37 #define DMA_TRACE(args) if (!(*di->msg_level & 2)); else printf args
38 #elif defined(BCMDBG_ERR)
39 #define DMA_ERROR(args) if (!(*di->msg_level & 1)); else printf args
40 #define DMA_TRACE(args)
41 #else
42 #define DMA_ERROR(args)
43 #define DMA_TRACE(args)
44 #endif /* BCMDBG */
45
46 #define DMA_NONE(args)
47
48
49 #define d32txregs dregs.d32_u.txregs_32
50 #define d32rxregs dregs.d32_u.rxregs_32
51 #define txd32 dregs.d32_u.txd_32
52 #define rxd32 dregs.d32_u.rxd_32
53
54 #define d64txregs dregs.d64_u.txregs_64
55 #define d64rxregs dregs.d64_u.rxregs_64
56 #define txd64 dregs.d64_u.txd_64
57 #define rxd64 dregs.d64_u.rxd_64
58
59 /* default dma message level (if input msg_level pointer is null in dma_attach()) */
60 static uint dma_msg_level =
61 #ifdef BCMDBG_ERR
62 1;
63 #else
64 0;
65 #endif /* BCMDBG_ERR */
66
67 #define MAXNAMEL 8 /* 8 char names */
68
69 #define DI_INFO(dmah) ((dma_info_t *)dmah)
70
71 #ifdef BCMPKTPOOL
72 #define POOL_ENAB(di) ((di)->pktpool->inited)
73 #else /* BCMPKTPOOL */
74 #define POOL_ENAB(di) 0
75 #endif /* BCMPKTPOOL */
76
77 /* dma engine software state */
78 typedef struct dma_info {
79 struct hnddma_pub hnddma; /* exported structure, don't use hnddma_t,
80 * which could be const
81 */
82 uint *msg_level; /* message level pointer */
83 char name[MAXNAMEL]; /* callers name for diag msgs */
84
85 void *osh; /* os handle */
86 si_t *sih; /* sb handle */
87
88 bool dma64; /* this dma engine is operating in 64-bit mode */
89 bool addrext; /* this dma engine supports DmaExtendedAddrChanges */
90
91 union {
92 struct {
93 dma32regs_t *txregs_32; /* 32-bit dma tx engine registers */
94 dma32regs_t *rxregs_32; /* 32-bit dma rx engine registers */
95 dma32dd_t *txd_32; /* pointer to dma32 tx descriptor ring */
96 dma32dd_t *rxd_32; /* pointer to dma32 rx descriptor ring */
97 } d32_u;
98 struct {
99 dma64regs_t *txregs_64; /* 64-bit dma tx engine registers */
100 dma64regs_t *rxregs_64; /* 64-bit dma rx engine registers */
101 dma64dd_t *txd_64; /* pointer to dma64 tx descriptor ring */
102 dma64dd_t *rxd_64; /* pointer to dma64 rx descriptor ring */
103 } d64_u;
104 } dregs;
105
106 uint16 dmadesc_align; /* alignment requirement for dma descriptors */
107
108 uint16 ntxd; /* # tx descriptors tunable */
109 uint16 txin; /* index of next descriptor to reclaim */
110 uint16 txout; /* index of next descriptor to post */
111 void **txp; /* pointer to parallel array of pointers to packets */
112 osldma_t *tx_dmah; /* DMA TX descriptor ring handle */
113 hnddma_seg_map_t *txp_dmah; /* DMA MAP meta-data handle */
114 dmaaddr_t txdpa; /* Aligned physical address of descriptor ring */
115 dmaaddr_t txdpaorig; /* Original physical address of descriptor ring */
116 uint16 txdalign; /* #bytes added to alloc'd mem to align txd */
117 uint32 txdalloc; /* #bytes allocated for the ring */
118 uint32 xmtptrbase; /* When using unaligned descriptors, the ptr register
119 * is not just an index, it needs all 13 bits to be
120 * an offset from the addr register.
121 */
122
123 uint16 nrxd; /* # rx descriptors tunable */
124 uint16 rxin; /* index of next descriptor to reclaim */
125 uint16 rxout; /* index of next descriptor to post */
126 void **rxp; /* pointer to parallel array of pointers to packets */
127 osldma_t *rx_dmah; /* DMA RX descriptor ring handle */
128 hnddma_seg_map_t *rxp_dmah; /* DMA MAP meta-data handle */
129 dmaaddr_t rxdpa; /* Aligned physical address of descriptor ring */
130 dmaaddr_t rxdpaorig; /* Original physical address of descriptor ring */
131 uint16 rxdalign; /* #bytes added to alloc'd mem to align rxd */
132 uint32 rxdalloc; /* #bytes allocated for the ring */
133 uint32 rcvptrbase; /* Base for ptr reg when using unaligned descriptors */
134
135 /* tunables */
136 uint16 rxbufsize; /* rx buffer size in bytes,
137 * not including the extra headroom
138 */
139 uint rxextrahdrroom; /* extra rx headroom, reverseved to assist upper stack
140 * e.g. some rx pkt buffers will be bridged to tx side
141 * without byte copying. The extra headroom needs to be
142 * large enough to fit txheader needs.
143 * Some dongle driver may not need it.
144 */
145 uint nrxpost; /* # rx buffers to keep posted */
146 uint rxoffset; /* rxcontrol offset */
147 uint ddoffsetlow; /* add to get dma address of descriptor ring, low 32 bits */
148 uint ddoffsethigh; /* high 32 bits */
149 uint dataoffsetlow; /* add to get dma address of data buffer, low 32 bits */
150 uint dataoffsethigh; /* high 32 bits */
151 bool aligndesc_4k; /* descriptor base need to be aligned or not */
152 pktpool_t *pktpool; /* pktpool */
153 uint dma_avoidance_cnt;
154 } dma_info_t;
155
156 /*
157 * If BCMDMA32 is defined, hnddma will support both 32-bit and 64-bit DMA engines.
158 * Otherwise it will support only 64-bit.
159 *
160 * DMA32_ENAB indicates whether hnddma is compiled with support for 32-bit DMA engines.
161 * DMA64_ENAB indicates whether hnddma is compiled with support for 64-bit DMA engines.
162 *
163 * DMA64_MODE indicates whether the current DMA engine is running as 64-bit.
164 */
165 #ifdef BCMDMA32
166 #define DMA32_ENAB(di) 1
167 #define DMA64_ENAB(di) 1
168 #define DMA64_MODE(di) ((di)->dma64)
169 #else /* !BCMDMA32 */
170 #define DMA32_ENAB(di) 0
171 #define DMA64_ENAB(di) 1
172 #define DMA64_MODE(di) 1
173 #endif /* !BCMDMA32 */
174
175 /* DMA Scatter-gather list is supported. Note this is limited to TX direction only */
176 #ifdef BCMDMASGLISTOSL
177 #define DMASGLIST_ENAB TRUE
178 #else
179 #define DMASGLIST_ENAB FALSE
180 #endif /* BCMDMASGLISTOSL */
181
182 /* descriptor bumping macros */
183 #define XXD(x, n) ((x) & ((n) - 1)) /* faster than %, but n must be power of 2 */
184 #define TXD(x) XXD((x), di->ntxd)
185 #define RXD(x) XXD((x), di->nrxd)
186 #define NEXTTXD(i) TXD((i) + 1)
187 #define PREVTXD(i) TXD((i) - 1)
188 #define NEXTRXD(i) RXD((i) + 1)
189 #define PREVRXD(i) RXD((i) - 1)
190
191 #define NTXDACTIVE(h, t) TXD((t) - (h))
192 #define NRXDACTIVE(h, t) RXD((t) - (h))
193
194 /* macros to convert between byte offsets and indexes */
195 #define B2I(bytes, type) ((bytes) / sizeof(type))
196 #define I2B(index, type) ((index) * sizeof(type))
197
198 #define PCI32ADDR_HIGH 0xc0000000 /* address[31:30] */
199 #define PCI32ADDR_HIGH_SHIFT 30 /* address[31:30] */
200
201 #define PCI64ADDR_HIGH 0x80000000 /* address[63] */
202 #define PCI64ADDR_HIGH_SHIFT 31 /* address[63] */
203
204 /* Common prototypes */
205 static bool _dma_isaddrext(dma_info_t *di);
206 static bool _dma_descriptor_align(dma_info_t *di);
207 static bool _dma_alloc(dma_info_t *di, uint direction);
208 static void _dma_detach(dma_info_t *di);
209 static void _dma_ddtable_init(dma_info_t *di, uint direction, dmaaddr_t pa);
210 static void _dma_rxinit(dma_info_t *di);
211 static void *_dma_rx(dma_info_t *di);
212 static bool _dma_rxfill(dma_info_t *di);
213 static void _dma_rxreclaim(dma_info_t *di);
214 static void _dma_rxenable(dma_info_t *di);
215 static void *_dma_getnextrxp(dma_info_t *di, bool forceall);
216 static void _dma_rx_param_get(dma_info_t *di, uint16 *rxoffset, uint16 *rxbufsize);
217
218 static void _dma_txblock(dma_info_t *di);
219 static void _dma_txunblock(dma_info_t *di);
220 static uint _dma_txactive(dma_info_t *di);
221 static uint _dma_rxactive(dma_info_t *di);
222 static uint _dma_txpending(dma_info_t *di);
223 static uint _dma_txcommitted(dma_info_t *di);
224
225 static void *_dma_peeknexttxp(dma_info_t *di);
226 static void *_dma_peeknextrxp(dma_info_t *di);
227 static uintptr _dma_getvar(dma_info_t *di, const char *name);
228 static void _dma_counterreset(dma_info_t *di);
229 static void _dma_fifoloopbackenable(dma_info_t *di);
230 static uint _dma_ctrlflags(dma_info_t *di, uint mask, uint flags);
231 static uint8 dma_align_sizetobits(uint size);
232 static void *dma_ringalloc(osl_t *osh, uint32 boundary, uint size, uint16 *alignbits, uint* alloced,
233 dmaaddr_t *descpa, osldma_t **dmah);
234 static uint _dma_avoidancecnt(dma_info_t *di);
235
236 /* Prototypes for 32-bit routines */
237 static bool dma32_alloc(dma_info_t *di, uint direction);
238 static bool dma32_txreset(dma_info_t *di);
239 static bool dma32_rxreset(dma_info_t *di);
240 static bool dma32_txsuspendedidle(dma_info_t *di);
241 static int dma32_txfast(dma_info_t *di, void *p0, bool commit);
242 static void *dma32_getnexttxp(dma_info_t *di, txd_range_t range);
243 static void *dma32_getnextrxp(dma_info_t *di, bool forceall);
244 static void dma32_txrotate(dma_info_t *di);
245 static bool dma32_rxidle(dma_info_t *di);
246 static void dma32_txinit(dma_info_t *di);
247 static bool dma32_txenabled(dma_info_t *di);
248 static void dma32_txsuspend(dma_info_t *di);
249 static void dma32_txresume(dma_info_t *di);
250 static bool dma32_txsuspended(dma_info_t *di);
251 #ifdef WL_MULTIQUEUE
252 static void dma32_txflush(dma_info_t *di);
253 static void dma32_txflush_clear(dma_info_t *di);
254 #endif /* WL_MULTIQUEUE */
255 static void dma32_txreclaim(dma_info_t *di, txd_range_t range);
256 static bool dma32_txstopped(dma_info_t *di);
257 static bool dma32_rxstopped(dma_info_t *di);
258 static bool dma32_rxenabled(dma_info_t *di);
259 #if defined(BCMDBG) || defined(BCMDBG_DUMP)
260 static void dma32_dumpring(dma_info_t *di, struct bcmstrbuf *b, dma32dd_t *ring, uint start,
261 uint end, uint max_num);
262 static void dma32_dump(dma_info_t *di, struct bcmstrbuf *b, bool dumpring);
263 static void dma32_dumptx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring);
264 static void dma32_dumprx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring);
265 #endif /* defined(BCMDBG) || defined(BCMDBG_DUMP) */
266
267 static bool _dma32_addrext(osl_t *osh, dma32regs_t *dma32regs);
268
269 /* Prototypes for 64-bit routines */
270 static bool dma64_alloc(dma_info_t *di, uint direction);
271 static bool dma64_txreset(dma_info_t *di);
272 static bool dma64_rxreset(dma_info_t *di);
273 static bool dma64_txsuspendedidle(dma_info_t *di);
274 static int dma64_txfast(dma_info_t *di, void *p0, bool commit);
275 #if !defined(_CFE_)
276 static int dma64_txunframed(dma_info_t *di, void *p0, uint len, bool commit);
277 static void *dma64_getpos(dma_info_t *di, bool direction);
278 #endif /* !_CFE_ */
279 static void *dma64_getnexttxp(dma_info_t *di, txd_range_t range);
280 static void *dma64_getnextrxp(dma_info_t *di, bool forceall);
281 static void dma64_txrotate(dma_info_t *di);
282
283 static bool dma64_rxidle(dma_info_t *di);
284 static void dma64_txinit(dma_info_t *di);
285 static bool dma64_txenabled(dma_info_t *di);
286 static void dma64_txsuspend(dma_info_t *di);
287 static void dma64_txresume(dma_info_t *di);
288 static bool dma64_txsuspended(dma_info_t *di);
289 #ifdef WL_MULTIQUEUE
290 static void dma64_txflush(dma_info_t *di);
291 static void dma64_txflush_clear(dma_info_t *di);
292 #endif /* WL_MULTIQUEUE */
293 static void dma64_txreclaim(dma_info_t *di, txd_range_t range);
294 static bool dma64_txstopped(dma_info_t *di);
295 static bool dma64_rxstopped(dma_info_t *di);
296 static bool dma64_rxenabled(dma_info_t *di);
297 static bool _dma64_addrext(osl_t *osh, dma64regs_t *dma64regs);
298
299
300 STATIC INLINE uint32 parity32(uint32 data);
301
302 #if defined(BCMDBG) || defined(BCMDBG_DUMP)
303 static void dma64_dumpring(dma_info_t *di, struct bcmstrbuf *b, dma64dd_t *ring, uint start,
304 uint end, uint max_num);
305 static void dma64_dump(dma_info_t *di, struct bcmstrbuf *b, bool dumpring);
306 static void dma64_dumptx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring);
307 static void dma64_dumprx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring);
308 #endif /* defined(BCMDBG) || defined(BCMDBG_DUMP) */
309
310
311 const di_fcn_t dma64proc = {
312 (di_detach_t)_dma_detach,
313 (di_txinit_t)dma64_txinit,
314 (di_txreset_t)dma64_txreset,
315 (di_txenabled_t)dma64_txenabled,
316 (di_txsuspend_t)dma64_txsuspend,
317 (di_txresume_t)dma64_txresume,
318 (di_txsuspended_t)dma64_txsuspended,
319 (di_txsuspendedidle_t)dma64_txsuspendedidle,
320 #ifdef WL_MULTIQUEUE
321 (di_txflush_t)dma64_txflush,
322 (di_txflush_clear_t)dma64_txflush_clear,
323 #endif /* WL_MULTIQUEUE */
324 (di_txfast_t)dma64_txfast,
325 #if !defined(_CFE_)
326 (di_txunframed_t)dma64_txunframed,
327 (di_getpos_t)dma64_getpos,
328 #endif /* !_CFE_ */
329 (di_txstopped_t)dma64_txstopped,
330 (di_txreclaim_t)dma64_txreclaim,
331 (di_getnexttxp_t)dma64_getnexttxp,
332 (di_peeknexttxp_t)_dma_peeknexttxp,
333 (di_txblock_t)_dma_txblock,
334 (di_txunblock_t)_dma_txunblock,
335 (di_txactive_t)_dma_txactive,
336 (di_txrotate_t)dma64_txrotate,
337
338 (di_rxinit_t)_dma_rxinit,
339 (di_rxreset_t)dma64_rxreset,
340 (di_rxidle_t)dma64_rxidle,
341 (di_rxstopped_t)dma64_rxstopped,
342 (di_rxenable_t)_dma_rxenable,
343 (di_rxenabled_t)dma64_rxenabled,
344 (di_rx_t)_dma_rx,
345 (di_rxfill_t)_dma_rxfill,
346 (di_rxreclaim_t)_dma_rxreclaim,
347 (di_getnextrxp_t)_dma_getnextrxp,
348 (di_peeknextrxp_t)_dma_peeknextrxp,
349 (di_rxparam_get_t)_dma_rx_param_get,
350
351 (di_fifoloopbackenable_t)_dma_fifoloopbackenable,
352 (di_getvar_t)_dma_getvar,
353 (di_counterreset_t)_dma_counterreset,
354 (di_ctrlflags_t)_dma_ctrlflags,
355
356 #if defined(BCMDBG) || defined(BCMDBG_DUMP)
357 (di_dump_t)dma64_dump,
358 (di_dumptx_t)dma64_dumptx,
359 (di_dumprx_t)dma64_dumprx,
360 #else
361 NULL,
362 NULL,
363 NULL,
364 #endif /* defined(BCMDBG) || defined(BCMDBG_DUMP) */
365 (di_rxactive_t)_dma_rxactive,
366 (di_txpending_t)_dma_txpending,
367 (di_txcommitted_t)_dma_txcommitted,
368 (di_avoidancecnt_t)_dma_avoidancecnt,
369 40
370 };
371
372 static const di_fcn_t dma32proc = {
373 (di_detach_t)_dma_detach,
374 (di_txinit_t)dma32_txinit,
375 (di_txreset_t)dma32_txreset,
376 (di_txenabled_t)dma32_txenabled,
377 (di_txsuspend_t)dma32_txsuspend,
378 (di_txresume_t)dma32_txresume,
379 (di_txsuspended_t)dma32_txsuspended,
380 (di_txsuspendedidle_t)dma32_txsuspendedidle,
381 #ifdef WL_MULTIQUEUE
382 (di_txflush_t)dma32_txflush,
383 (di_txflush_clear_t)dma32_txflush_clear,
384 #endif /* WL_MULTIQUEUE */
385 (di_txfast_t)dma32_txfast,
386 #if !defined(_CFE_)
387 NULL,
388 NULL,
389 #endif /* !_CFE_ */
390 (di_txstopped_t)dma32_txstopped,
391 (di_txreclaim_t)dma32_txreclaim,
392 (di_getnexttxp_t)dma32_getnexttxp,
393 (di_peeknexttxp_t)_dma_peeknexttxp,
394 (di_txblock_t)_dma_txblock,
395 (di_txunblock_t)_dma_txunblock,
396 (di_txactive_t)_dma_txactive,
397 (di_txrotate_t)dma32_txrotate,
398
399 (di_rxinit_t)_dma_rxinit,
400 (di_rxreset_t)dma32_rxreset,
401 (di_rxidle_t)dma32_rxidle,
402 (di_rxstopped_t)dma32_rxstopped,
403 (di_rxenable_t)_dma_rxenable,
404 (di_rxenabled_t)dma32_rxenabled,
405 (di_rx_t)_dma_rx,
406 (di_rxfill_t)_dma_rxfill,
407 (di_rxreclaim_t)_dma_rxreclaim,
408 (di_getnextrxp_t)_dma_getnextrxp,
409 (di_peeknextrxp_t)_dma_peeknextrxp,
410 (di_rxparam_get_t)_dma_rx_param_get,
411
412 (di_fifoloopbackenable_t)_dma_fifoloopbackenable,
413 (di_getvar_t)_dma_getvar,
414 (di_counterreset_t)_dma_counterreset,
415 (di_ctrlflags_t)_dma_ctrlflags,
416
417 #if defined(BCMDBG) || defined(BCMDBG_DUMP)
418 (di_dump_t)dma32_dump,
419 (di_dumptx_t)dma32_dumptx,
420 (di_dumprx_t)dma32_dumprx,
421 #else
422 NULL,
423 NULL,
424 NULL,
425 #endif /* defined(BCMDBG) || defined(BCMDBG_DUMP) */
426 (di_rxactive_t)_dma_rxactive,
427 (di_txpending_t)_dma_txpending,
428 (di_txcommitted_t)_dma_txcommitted,
429 (di_avoidancecnt_t)_dma_avoidancecnt,
430 40
431 };
432
433 hnddma_t *
434 dma_attach(osl_t *osh, char *name, si_t *sih, void *dmaregstx, void *dmaregsrx,
435 uint ntxd, uint nrxd, uint rxbufsize, int rxextheadroom, uint nrxpost, uint rxoffset,
436 uint *msg_level)
437 {
438 dma_info_t *di;
439 uint size;
440
441 /* allocate private info structure */
442 if ((di = MALLOC(osh, sizeof (dma_info_t))) == NULL) {
443 #ifdef BCMDBG
444 printf("dma_attach: out of memory, malloced %d bytes\n", MALLOCED(osh));
445 #endif
446 return (NULL);
447 }
448
449 bzero((char *)di, sizeof(dma_info_t));
450
451 di->msg_level = msg_level ? msg_level : &dma_msg_level;
452
453 /* old chips w/o sb is no longer supported */
454 ASSERT(sih != NULL);
455
456 if (DMA64_ENAB(di))
457 di->dma64 = ((si_core_sflags(sih, 0, 0) & SISF_DMA64) == SISF_DMA64);
458 else
459 di->dma64 = 0;
460
461 /* check arguments */
462 ASSERT(ISPOWEROF2(ntxd));
463 ASSERT(ISPOWEROF2(nrxd));
464
465 if (nrxd == 0)
466 ASSERT(dmaregsrx == NULL);
467 if (ntxd == 0)
468 ASSERT(dmaregstx == NULL);
469
470 /* init dma reg pointer */
471 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
472 ASSERT(ntxd <= D64MAXDD);
473 ASSERT(nrxd <= D64MAXDD);
474 di->d64txregs = (dma64regs_t *)dmaregstx;
475 di->d64rxregs = (dma64regs_t *)dmaregsrx;
476 di->hnddma.di_fn = (const di_fcn_t *)&dma64proc;
477 } else if (DMA32_ENAB(di)) {
478 ASSERT(ntxd <= D32MAXDD);
479 ASSERT(nrxd <= D32MAXDD);
480 di->d32txregs = (dma32regs_t *)dmaregstx;
481 di->d32rxregs = (dma32regs_t *)dmaregsrx;
482 di->hnddma.di_fn = (const di_fcn_t *)&dma32proc;
483 } else {
484 DMA_ERROR(("dma_attach: driver doesn't support 32-bit DMA\n"));
485 ASSERT(0);
486 goto fail;
487 }
488
489 /* init the pktpool code if need be */
490 #ifdef BCMPKTPOOL
491 di->pktpool = &pktpool_shared;
492 #endif /* BCMPKTPOOL */
493
494
495 /* Default flags (which can be changed by the driver calling dma_ctrlflags
496 * before enable): For backwards compatibility both Rx Overflow Continue
497 * and Parity are DISABLED.
498 * supports it.
499 */
500 di->hnddma.di_fn->ctrlflags(&di->hnddma, DMA_CTRL_ROC | DMA_CTRL_PEN, 0);
501
502 DMA_TRACE(("%s: dma_attach: %s osh %p flags 0x%x ntxd %d nrxd %d rxbufsize %d "
503 "rxextheadroom %d nrxpost %d rxoffset %d dmaregstx %p dmaregsrx %p\n",
504 name, (DMA64_MODE(di) ? "DMA64" : "DMA32"),
505 osh, di->hnddma.dmactrlflags, ntxd, nrxd,
506 rxbufsize, rxextheadroom, nrxpost, rxoffset, dmaregstx, dmaregsrx));
507
508 /* make a private copy of our callers name */
509 strncpy(di->name, name, MAXNAMEL);
510 di->name[MAXNAMEL-1] = '\0';
511
512 di->osh = osh;
513 di->sih = sih;
514
515 /* save tunables */
516 di->ntxd = (uint16)ntxd;
517 di->nrxd = (uint16)nrxd;
518
519 /* the actual dma size doesn't include the extra headroom */
520 di->rxextrahdrroom = (rxextheadroom == -1) ? BCMEXTRAHDROOM : rxextheadroom;
521 if (rxbufsize > BCMEXTRAHDROOM)
522 di->rxbufsize = (uint16)(rxbufsize - di->rxextrahdrroom);
523 else
524 di->rxbufsize = (uint16)rxbufsize;
525
526 di->nrxpost = (uint16)nrxpost;
527 di->rxoffset = (uint8)rxoffset;
528
529 /*
530 * figure out the DMA physical address offset for dd and data
531 * PCI/PCIE: they map silicon backplace address to zero based memory, need offset
532 * Other bus: use zero
533 * SI_BUS BIGENDIAN kludge: use sdram swapped region for data buffer, not descriptor
534 */
535 di->ddoffsetlow = 0;
536 di->dataoffsetlow = 0;
537 /* for pci bus, add offset */
538 if (sih->bustype == PCI_BUS) {
539 if ((sih->buscoretype == PCIE_CORE_ID) && DMA64_MODE(di)) {
540 /* pcie with DMA64 */
541 di->ddoffsetlow = 0;
542 di->ddoffsethigh = SI_PCIE_DMA_H32;
543 } else {
544 /* pci(DMA32/DMA64) or pcie with DMA32 */
545 if ((CHIPID(sih->chip) == BCM4322_CHIP_ID) ||
546 (CHIPID(sih->chip) == BCM4342_CHIP_ID) ||
547 (CHIPID(sih->chip) == BCM43221_CHIP_ID) ||
548 (CHIPID(sih->chip) == BCM43231_CHIP_ID) ||
549 (CHIPID(sih->chip) == BCM43111_CHIP_ID) ||
550 (CHIPID(sih->chip) == BCM43112_CHIP_ID) ||
551 (CHIPID(sih->chip) == BCM43222_CHIP_ID) ||
552 (CHIPID(sih->chip) == BCM43420_CHIP_ID))
553 di->ddoffsetlow = SI_PCI_DMA2;
554 else
555 di->ddoffsetlow = SI_PCI_DMA;
556
557 di->ddoffsethigh = 0;
558 }
559 di->dataoffsetlow = di->ddoffsetlow;
560 di->dataoffsethigh = di->ddoffsethigh;
561 }
562
563 #if defined(__mips__) && defined(IL_BIGENDIAN)
564 di->dataoffsetlow = di->dataoffsetlow + SI_SDRAM_SWAPPED;
565 #endif /* defined(__mips__) && defined(IL_BIGENDIAN) */
566 /* WAR64450 : DMACtl.Addr ext fields are not supported in SDIOD core. */
567 if ((si_coreid(sih) == SDIOD_CORE_ID) && ((si_corerev(sih) > 0) && (si_corerev(sih) <= 2)))
568 di->addrext = 0;
569 else if ((si_coreid(sih) == I2S_CORE_ID) &&
570 ((si_corerev(sih) == 0) || (si_corerev(sih) == 1)))
571 di->addrext = 0;
572 else
573 di->addrext = _dma_isaddrext(di);
574
575 /* does the descriptors need to be aligned and if yes, on 4K/8K or not */
576 di->aligndesc_4k = _dma_descriptor_align(di);
577 if (di->aligndesc_4k) {
578 if (DMA64_MODE(di)) {
579 di->dmadesc_align = D64RINGALIGN_BITS;
580 if ((ntxd < D64MAXDD / 2) && (nrxd < D64MAXDD / 2)) {
581 /* for smaller dd table, HW relax the alignment requirement */
582 di->dmadesc_align = D64RINGALIGN_BITS - 1;
583 }
584 } else
585 di->dmadesc_align = D32RINGALIGN_BITS;
586 } else {
587 /* The start address of descriptor table should be algined to cache line size,
588 * or other structure may share a cache line with it, which can lead to memory
589 * overlapping due to cache write-back operation. In the case of MIPS 74k, the
590 * cache line size is 32 bytes.
591 */
592 #ifdef __mips__
593 di->dmadesc_align = 5; /* 32 byte alignment */
594 #else
595 di->dmadesc_align = 4; /* 16 byte alignment */
596 #endif
597 }
598
599 DMA_NONE(("DMA descriptor align_needed %d, align %d\n",
600 di->aligndesc_4k, di->dmadesc_align));
601
602 /* allocate tx packet pointer vector */
603 if (ntxd) {
604 size = ntxd * sizeof(void *);
605 if ((di->txp = MALLOC(osh, size)) == NULL) {
606 DMA_ERROR(("%s: dma_attach: out of tx memory, malloced %d bytes\n",
607 di->name, MALLOCED(osh)));
608 goto fail;
609 }
610 bzero((char *)di->txp, size);
611 }
612
613 /* allocate rx packet pointer vector */
614 if (nrxd) {
615 size = nrxd * sizeof(void *);
616 if ((di->rxp = MALLOC(osh, size)) == NULL) {
617 DMA_ERROR(("%s: dma_attach: out of rx memory, malloced %d bytes\n",
618 di->name, MALLOCED(osh)));
619 goto fail;
620 }
621 bzero((char *)di->rxp, size);
622 }
623
624 /* allocate transmit descriptor ring, only need ntxd descriptors but it must be aligned */
625 if (ntxd) {
626 if (!_dma_alloc(di, DMA_TX))
627 goto fail;
628 }
629
630 /* allocate receive descriptor ring, only need nrxd descriptors but it must be aligned */
631 if (nrxd) {
632 if (!_dma_alloc(di, DMA_RX))
633 goto fail;
634 }
635
636 if ((di->ddoffsetlow != 0) && !di->addrext) {
637 if (PHYSADDRLO(di->txdpa) > SI_PCI_DMA_SZ) {
638 DMA_ERROR(("%s: dma_attach: txdpa 0x%x: addrext not supported\n",
639 di->name, (uint32)PHYSADDRLO(di->txdpa)));
640 goto fail;
641 }
642 if (PHYSADDRLO(di->rxdpa) > SI_PCI_DMA_SZ) {
643 DMA_ERROR(("%s: dma_attach: rxdpa 0x%x: addrext not supported\n",
644 di->name, (uint32)PHYSADDRLO(di->rxdpa)));
645 goto fail;
646 }
647 }
648
649 DMA_TRACE(("ddoffsetlow 0x%x ddoffsethigh 0x%x dataoffsetlow 0x%x dataoffsethigh "
650 "0x%x addrext %d\n", di->ddoffsetlow, di->ddoffsethigh, di->dataoffsetlow,
651 di->dataoffsethigh, di->addrext));
652
653 /* allocate DMA mapping vectors */
654 if (DMASGLIST_ENAB) {
655 if (ntxd) {
656 size = ntxd * sizeof(hnddma_seg_map_t);
657 if ((di->txp_dmah = (hnddma_seg_map_t *)MALLOC(osh, size)) == NULL)
658 goto fail;
659 bzero((char*)di->txp_dmah, size);
660 }
661
662 if (nrxd) {
663 size = nrxd * sizeof(hnddma_seg_map_t);
664 if ((di->rxp_dmah = (hnddma_seg_map_t *)MALLOC(osh, size)) == NULL)
665 goto fail;
666 bzero((char*)di->rxp_dmah, size);
667 }
668 }
669
670 return ((hnddma_t *)di);
671
672 fail:
673 _dma_detach(di);
674 return (NULL);
675 }
676
677 /* init the tx or rx descriptor */
678 static INLINE void
679 dma32_dd_upd(dma_info_t *di, dma32dd_t *ddring, dmaaddr_t pa, uint outidx, uint32 *flags,
680 uint32 bufcount)
681 {
682 /* dma32 uses 32-bit control to fit both flags and bufcounter */
683 *flags = *flags | (bufcount & CTRL_BC_MASK);
684
685 if ((di->dataoffsetlow == 0) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) {
686 W_SM(&ddring[outidx].addr, BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow));
687 W_SM(&ddring[outidx].ctrl, BUS_SWAP32(*flags));
688 } else {
689 /* address extension */
690 uint32 ae;
691 ASSERT(di->addrext);
692 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
693 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH;
694
695 *flags |= (ae << CTRL_AE_SHIFT);
696 W_SM(&ddring[outidx].addr, BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow));
697 W_SM(&ddring[outidx].ctrl, BUS_SWAP32(*flags));
698 }
699 }
700
701 /* Check for odd number of 1's */
702 STATIC INLINE uint32 parity32(uint32 data)
703 {
704 data ^= data >> 16;
705 data ^= data >> 8;
706 data ^= data >> 4;
707 data ^= data >> 2;
708 data ^= data >> 1;
709
710 return (data & 1);
711 }
712
713 #define DMA64_DD_PARITY(dd) parity32((dd)->addrlow ^ (dd)->addrhigh ^ (dd)->ctrl1 ^ (dd)->ctrl2)
714
715 static INLINE void
716 dma64_dd_upd(dma_info_t *di, dma64dd_t *ddring, dmaaddr_t pa, uint outidx, uint32 *flags,
717 uint32 bufcount)
718 {
719 uint32 ctrl2 = bufcount & D64_CTRL2_BC_MASK;
720
721 /* PCI bus with big(>1G) physical address, use address extension */
722 #if defined(__mips__) && defined(IL_BIGENDIAN)
723 if ((di->dataoffsetlow == SI_SDRAM_SWAPPED) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) {
724 #else
725 if ((di->dataoffsetlow == 0) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) {
726 #endif /* defined(__mips__) && defined(IL_BIGENDIAN) */
727 ASSERT((PHYSADDRHI(pa) & PCI64ADDR_HIGH) == 0);
728
729 W_SM(&ddring[outidx].addrlow, BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow));
730 W_SM(&ddring[outidx].addrhigh, BUS_SWAP32(PHYSADDRHI(pa) + di->dataoffsethigh));
731 W_SM(&ddring[outidx].ctrl1, BUS_SWAP32(*flags));
732 W_SM(&ddring[outidx].ctrl2, BUS_SWAP32(ctrl2));
733 } else {
734 /* address extension for 32-bit PCI */
735 uint32 ae;
736 ASSERT(di->addrext);
737
738 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
739 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH;
740 ASSERT(PHYSADDRHI(pa) == 0);
741
742 ctrl2 |= (ae << D64_CTRL2_AE_SHIFT) & D64_CTRL2_AE;
743 W_SM(&ddring[outidx].addrlow, BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow));
744 W_SM(&ddring[outidx].addrhigh, BUS_SWAP32(0 + di->dataoffsethigh));
745 W_SM(&ddring[outidx].ctrl1, BUS_SWAP32(*flags));
746 W_SM(&ddring[outidx].ctrl2, BUS_SWAP32(ctrl2));
747 }
748 if (di->hnddma.dmactrlflags & DMA_CTRL_PEN) {
749 if (DMA64_DD_PARITY(&ddring[outidx])) {
750 W_SM(&ddring[outidx].ctrl2, BUS_SWAP32(ctrl2 | D64_CTRL2_PARITY));
751 }
752 }
753 }
754
755 static bool
756 _dma32_addrext(osl_t *osh, dma32regs_t *dma32regs)
757 {
758 uint32 w;
759
760 OR_REG(osh, &dma32regs->control, XC_AE);
761 w = R_REG(osh, &dma32regs->control);
762 AND_REG(osh, &dma32regs->control, ~XC_AE);
763 return ((w & XC_AE) == XC_AE);
764 }
765
766 static bool
767 _dma_alloc(dma_info_t *di, uint direction)
768 {
769 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
770 return dma64_alloc(di, direction);
771 } else if (DMA32_ENAB(di)) {
772 return dma32_alloc(di, direction);
773 } else
774 ASSERT(0);
775 }
776
777 /* !! may be called with core in reset */
778 static void
779 _dma_detach(dma_info_t *di)
780 {
781
782 DMA_TRACE(("%s: dma_detach\n", di->name));
783
784 /* shouldn't be here if descriptors are unreclaimed */
785 ASSERT(di->txin == di->txout);
786 ASSERT(di->rxin == di->rxout);
787
788 /* free dma descriptor rings */
789 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
790 if (di->txd64)
791 DMA_FREE_CONSISTENT(di->osh, ((int8 *)(uintptr)di->txd64 - di->txdalign),
792 di->txdalloc, (di->txdpaorig), &di->tx_dmah);
793 if (di->rxd64)
794 DMA_FREE_CONSISTENT(di->osh, ((int8 *)(uintptr)di->rxd64 - di->rxdalign),
795 di->rxdalloc, (di->rxdpaorig), &di->rx_dmah);
796 } else if (DMA32_ENAB(di)) {
797 if (di->txd32)
798 DMA_FREE_CONSISTENT(di->osh, ((int8 *)(uintptr)di->txd32 - di->txdalign),
799 di->txdalloc, (di->txdpaorig), &di->tx_dmah);
800 if (di->rxd32)
801 DMA_FREE_CONSISTENT(di->osh, ((int8 *)(uintptr)di->rxd32 - di->rxdalign),
802 di->rxdalloc, (di->rxdpaorig), &di->rx_dmah);
803 } else
804 ASSERT(0);
805
806 /* free packet pointer vectors */
807 if (di->txp)
808 MFREE(di->osh, (void *)di->txp, (di->ntxd * sizeof(void *)));
809 if (di->rxp)
810 MFREE(di->osh, (void *)di->rxp, (di->nrxd * sizeof(void *)));
811
812 /* free tx packet DMA handles */
813 if (di->txp_dmah)
814 MFREE(di->osh, (void *)di->txp_dmah, di->ntxd * sizeof(hnddma_seg_map_t));
815
816 /* free rx packet DMA handles */
817 if (di->rxp_dmah)
818 MFREE(di->osh, (void *)di->rxp_dmah, di->nrxd * sizeof(hnddma_seg_map_t));
819
820 /* free our private info structure */
821 MFREE(di->osh, (void *)di, sizeof(dma_info_t));
822
823 }
824
825 static bool
826 _dma_descriptor_align(dma_info_t *di)
827 {
828 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
829 uint32 addrl;
830
831 /* Check to see if the descriptors need to be aligned on 4K/8K or not */
832 if (di->d64txregs != NULL) {
833 W_REG(di->osh, &di->d64txregs->addrlow, 0xff0);
834 addrl = R_REG(di->osh, &di->d64txregs->addrlow);
835 if (addrl != 0)
836 return FALSE;
837 } else if (di->d64rxregs != NULL) {
838 W_REG(di->osh, &di->d64rxregs->addrlow, 0xff0);
839 addrl = R_REG(di->osh, &di->d64rxregs->addrlow);
840 if (addrl != 0)
841 return FALSE;
842 }
843 }
844 return TRUE;
845 }
846
847 /* return TRUE if this dma engine supports DmaExtendedAddrChanges, otherwise FALSE */
848 static bool
849 _dma_isaddrext(dma_info_t *di)
850 {
851 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
852 /* DMA64 supports full 32- or 64-bit operation. AE is always valid */
853
854 /* not all tx or rx channel are available */
855 if (di->d64txregs != NULL) {
856 if (!_dma64_addrext(di->osh, di->d64txregs)) {
857 DMA_ERROR(("%s: _dma_isaddrext: DMA64 tx doesn't have AE set\n",
858 di->name));
859 ASSERT(0);
860 }
861 return TRUE;
862 } else if (di->d64rxregs != NULL) {
863 if (!_dma64_addrext(di->osh, di->d64rxregs)) {
864 DMA_ERROR(("%s: _dma_isaddrext: DMA64 rx doesn't have AE set\n",
865 di->name));
866 ASSERT(0);
867 }
868 return TRUE;
869 }
870 return FALSE;
871 } else if (DMA32_ENAB(di)) {
872 if (di->d32txregs)
873 return (_dma32_addrext(di->osh, di->d32txregs));
874 else if (di->d32rxregs)
875 return (_dma32_addrext(di->osh, di->d32rxregs));
876 } else
877 ASSERT(0);
878
879 return FALSE;
880 }
881
882 /* initialize descriptor table base address */
883 static void
884 _dma_ddtable_init(dma_info_t *di, uint direction, dmaaddr_t pa)
885 {
886 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
887 if (!di->aligndesc_4k) {
888 if (direction == DMA_TX)
889 di->xmtptrbase = PHYSADDRLO(pa);
890 else
891 di->rcvptrbase = PHYSADDRLO(pa);
892 }
893
894 if ((di->ddoffsetlow == 0) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) {
895 if (direction == DMA_TX) {
896 W_REG(di->osh, &di->d64txregs->addrlow, (PHYSADDRLO(pa) +
897 di->ddoffsetlow));
898 W_REG(di->osh, &di->d64txregs->addrhigh, (PHYSADDRHI(pa) +
899 di->ddoffsethigh));
900 } else {
901 W_REG(di->osh, &di->d64rxregs->addrlow, (PHYSADDRLO(pa) +
902 di->ddoffsetlow));
903 W_REG(di->osh, &di->d64rxregs->addrhigh, (PHYSADDRHI(pa) +
904 di->ddoffsethigh));
905 }
906 } else {
907 /* DMA64 32bits address extension */
908 uint32 ae;
909 ASSERT(di->addrext);
910 ASSERT(PHYSADDRHI(pa) == 0);
911
912 /* shift the high bit(s) from pa to ae */
913 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
914 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH;
915
916 if (direction == DMA_TX) {
917 W_REG(di->osh, &di->d64txregs->addrlow, (PHYSADDRLO(pa) +
918 di->ddoffsetlow));
919 W_REG(di->osh, &di->d64txregs->addrhigh, di->ddoffsethigh);
920 SET_REG(di->osh, &di->d64txregs->control, D64_XC_AE,
921 (ae << D64_XC_AE_SHIFT));
922 } else {
923 W_REG(di->osh, &di->d64rxregs->addrlow, (PHYSADDRLO(pa) +
924 di->ddoffsetlow));
925 W_REG(di->osh, &di->d64rxregs->addrhigh, di->ddoffsethigh);
926 SET_REG(di->osh, &di->d64rxregs->control, D64_RC_AE,
927 (ae << D64_RC_AE_SHIFT));
928 }
929 }
930
931 } else if (DMA32_ENAB(di)) {
932 ASSERT(PHYSADDRHI(pa) == 0);
933 if ((di->ddoffsetlow == 0) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) {
934 if (direction == DMA_TX)
935 W_REG(di->osh, &di->d32txregs->addr, (PHYSADDRLO(pa) +
936 di->ddoffsetlow));
937 else
938 W_REG(di->osh, &di->d32rxregs->addr, (PHYSADDRLO(pa) +
939 di->ddoffsetlow));
940 } else {
941 /* dma32 address extension */
942 uint32 ae;
943 ASSERT(di->addrext);
944
945 /* shift the high bit(s) from pa to ae */
946 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
947 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH;
948
949 if (direction == DMA_TX) {
950 W_REG(di->osh, &di->d32txregs->addr, (PHYSADDRLO(pa) +
951 di->ddoffsetlow));
952 SET_REG(di->osh, &di->d32txregs->control, XC_AE, ae <<XC_AE_SHIFT);
953 } else {
954 W_REG(di->osh, &di->d32rxregs->addr, (PHYSADDRLO(pa) +
955 di->ddoffsetlow));
956 SET_REG(di->osh, &di->d32rxregs->control, RC_AE, ae <<RC_AE_SHIFT);
957 }
958 }
959 } else
960 ASSERT(0);
961 }
962
963 static void
964 _dma_fifoloopbackenable(dma_info_t *di)
965 {
966 DMA_TRACE(("%s: dma_fifoloopbackenable\n", di->name));
967
968 if (DMA64_ENAB(di) && DMA64_MODE(di))
969 OR_REG(di->osh, &di->d64txregs->control, D64_XC_LE);
970 else if (DMA32_ENAB(di))
971 OR_REG(di->osh, &di->d32txregs->control, XC_LE);
972 else
973 ASSERT(0);
974 }
975
976 static void
977 _dma_rxinit(dma_info_t *di)
978 {
979 DMA_TRACE(("%s: dma_rxinit\n", di->name));
980
981 if (di->nrxd == 0)
982 return;
983
984 di->rxin = di->rxout = 0;
985
986 /* clear rx descriptor ring */
987 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
988 BZERO_SM((void *)(uintptr)di->rxd64, (di->nrxd * sizeof(dma64dd_t)));
989
990 /* DMA engine with out alignment requirement requires table to be inited
991 * before enabling the engine
992 */
993 if (!di->aligndesc_4k)
994 _dma_ddtable_init(di, DMA_RX, di->rxdpa);
995
996 _dma_rxenable(di);
997
998 if (di->aligndesc_4k)
999 _dma_ddtable_init(di, DMA_RX, di->rxdpa);
1000 } else if (DMA32_ENAB(di)) {
1001 BZERO_SM((void *)(uintptr)di->rxd32, (di->nrxd * sizeof(dma32dd_t)));
1002 _dma_rxenable(di);
1003 _dma_ddtable_init(di, DMA_RX, di->rxdpa);
1004 } else
1005 ASSERT(0);
1006 }
1007
1008 static void
1009 _dma_rxenable(dma_info_t *di)
1010 {
1011 uint dmactrlflags = di->hnddma.dmactrlflags;
1012
1013 DMA_TRACE(("%s: dma_rxenable\n", di->name));
1014
1015 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1016 uint32 control = (R_REG(di->osh, &di->d64rxregs->control) & D64_RC_AE) | D64_RC_RE;
1017
1018 if ((dmactrlflags & DMA_CTRL_PEN) == 0)
1019 control |= D64_RC_PD;
1020
1021 if (dmactrlflags & DMA_CTRL_ROC)
1022 control |= D64_RC_OC;
1023
1024 W_REG(di->osh, &di->d64rxregs->control,
1025 ((di->rxoffset << D64_RC_RO_SHIFT) | control));
1026 } else if (DMA32_ENAB(di)) {
1027 uint32 control = (R_REG(di->osh, &di->d32rxregs->control) & RC_AE) | RC_RE;
1028
1029 if ((dmactrlflags & DMA_CTRL_PEN) == 0)
1030 control |= RC_PD;
1031
1032 if (dmactrlflags & DMA_CTRL_ROC)
1033 control |= RC_OC;
1034
1035 W_REG(di->osh, &di->d32rxregs->control,
1036 ((di->rxoffset << RC_RO_SHIFT) | control));
1037 } else
1038 ASSERT(0);
1039 }
1040
1041 static void
1042 _dma_rx_param_get(dma_info_t *di, uint16 *rxoffset, uint16 *rxbufsize)
1043 {
1044 /* the normal values fit into 16 bits */
1045 *rxoffset = (uint16)di->rxoffset;
1046 *rxbufsize = (uint16)di->rxbufsize;
1047 }
1048
1049 /* !! rx entry routine
1050 * returns a pointer to the next frame received, or NULL if there are no more
1051 * if DMA_CTRL_RXMULTI is defined, DMA scattering(multiple buffers) is supported
1052 * with pkts chain
1053 * otherwise, it's treated as giant pkt and will be tossed.
1054 * The DMA scattering starts with normal DMA header, followed by first buffer data.
1055 * After it reaches the max size of buffer, the data continues in next DMA descriptor
1056 * buffer WITHOUT DMA header
1057 */
1058 static void * BCMFASTPATH
1059 _dma_rx(dma_info_t *di)
1060 {
1061 void *p, *head, *tail;
1062 uint len;
1063 uint pkt_len;
1064 int resid = 0;
1065
1066 next_frame:
1067 head = _dma_getnextrxp(di, FALSE);
1068 if (head == NULL)
1069 return (NULL);
1070
1071 #if !defined(__mips__)
1072 len = ltoh16(*(uint16 *)(PKTDATA(di->osh, head)));
1073 #else
1074 {
1075 int read_count = 0;
1076 for (read_count = 200; (!(len = ltoh16(*(uint16 *)OSL_UNCACHED(PKTDATA(di->osh, head)))) &&
1077 read_count); read_count--) {
1078 if (CHIPID(di->sih->chip) == BCM5356_CHIP_ID)
1079 break;
1080 OSL_DELAY(1);
1081 }
1082
1083 if (!len) {
1084 DMA_ERROR(("%s: dma_rx: frame length (%d)\n", di->name, len));
1085 PKTFREE(di->osh, head, FALSE);
1086 goto next_frame;
1087 }
1088
1089 }
1090 #endif /* defined(__mips__) */
1091 DMA_TRACE(("%s: dma_rx len %d\n", di->name, len));
1092
1093 /* set actual length */
1094 pkt_len = MIN((di->rxoffset + len), di->rxbufsize);
1095 PKTSETLEN(di->osh, head, pkt_len);
1096 resid = len - (di->rxbufsize - di->rxoffset);
1097
1098 /* check for single or multi-buffer rx */
1099 if (resid > 0) {
1100 tail = head;
1101 while ((resid > 0) && (p = _dma_getnextrxp(di, FALSE))) {
1102 PKTSETNEXT(di->osh, tail, p);
1103 pkt_len = MIN(resid, (int)di->rxbufsize);
1104 PKTSETLEN(di->osh, p, pkt_len);
1105
1106 tail = p;
1107 resid -= di->rxbufsize;
1108 }
1109
1110 #ifdef BCMDBG
1111 if (resid > 0) {
1112 uint cur;
1113 ASSERT(p == NULL);
1114 cur = (DMA64_ENAB(di) && DMA64_MODE(di)) ?
1115 B2I(((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) -
1116 di->rcvptrbase) & D64_RS0_CD_MASK, dma64dd_t) :
1117 B2I(R_REG(di->osh, &di->d32rxregs->status) & RS_CD_MASK,
1118 dma32dd_t);
1119 DMA_ERROR(("_dma_rx, rxin %d rxout %d, hw_curr %d\n",
1120 di->rxin, di->rxout, cur));
1121 }
1122 #endif /* BCMDBG */
1123
1124 if ((di->hnddma.dmactrlflags & DMA_CTRL_RXMULTI) == 0) {
1125 DMA_ERROR(("%s: dma_rx: bad frame length (%d)\n", di->name, len));
1126 PKTFREE(di->osh, head, FALSE);
1127 di->hnddma.rxgiants++;
1128 goto next_frame;
1129 }
1130 }
1131
1132 return (head);
1133 }
1134
1135 /* post receive buffers
1136 * return FALSE is refill failed completely and ring is empty
1137 * this will stall the rx dma and user might want to call rxfill again asap
1138 * This unlikely happens on memory-rich NIC, but often on memory-constrained dongle
1139 */
1140 static bool BCMFASTPATH
1141 _dma_rxfill(dma_info_t *di)
1142 {
1143 void *p;
1144 uint16 rxin, rxout;
1145 uint32 flags = 0;
1146 uint n;
1147 uint i;
1148 dmaaddr_t pa;
1149 uint extra_offset = 0;
1150 bool ring_empty;
1151 uint alignment_req = (di->hnddma.dmactrlflags & DMA_CTRL_USB_BOUNDRY4KB_WAR) ?
1152 16 : 1; /* MUST BE POWER of 2 */
1153
1154 ring_empty = FALSE;
1155
1156 /*
1157 * Determine how many receive buffers we're lacking
1158 * from the full complement, allocate, initialize,
1159 * and post them, then update the chip rx lastdscr.
1160 */
1161
1162 rxin = di->rxin;
1163 rxout = di->rxout;
1164
1165 n = di->nrxpost - NRXDACTIVE(rxin, rxout);
1166
1167 DMA_TRACE(("%s: dma_rxfill: post %d\n", di->name, n));
1168
1169 for (i = 0; i < n; i++) {
1170 /* the di->rxbufsize doesn't include the extra headroom, we need to add it to the
1171 size to be allocated
1172 */
1173 if (POOL_ENAB(di) && di->rxbufsize > PKTPOOL_MIN_LEN) {
1174 ASSERT(di->pktpool);
1175 p = pktpool_get(di->pktpool);
1176 #ifdef BCMDBG_POOL
1177 if (p)
1178 PKTPOOLSETSTATE(p, POOL_RXFILL);
1179 #endif /* BCMDBG_POOL */
1180 }
1181 else {
1182 if (di->rxbufsize > BCMEXTRAHDROOM)
1183 extra_offset = di->rxextrahdrroom;
1184 else
1185 extra_offset = 0;
1186
1187 p = PKTGET(di->osh, (di->rxbufsize + extra_offset + alignment_req - 1),
1188 FALSE);
1189 }
1190 if (p == NULL) {
1191 DMA_ERROR(("%s: dma_rxfill: out of rxbufs\n", di->name));
1192 if (i == 0) {
1193 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1194 if (dma64_rxidle(di)) {
1195 DMA_ERROR(("%s: rxfill64: ring is empty !\n",
1196 di->name));
1197 ring_empty = TRUE;
1198 }
1199 } else if (DMA32_ENAB(di)) {
1200 if (dma32_rxidle(di)) {
1201 DMA_ERROR(("%s: rxfill32: ring is empty !\n",
1202 di->name));
1203 ring_empty = TRUE;
1204 }
1205 } else
1206 ASSERT(0);
1207 }
1208 di->hnddma.rxnobuf++;
1209 break;
1210 }
1211 /* reserve an extra headroom, if applicable */
1212 if (di->hnddma.dmactrlflags & DMA_CTRL_USB_BOUNDRY4KB_WAR) {
1213 extra_offset += ((alignment_req - (unsigned long)PKTDATA(di->osh, p))
1214 & (alignment_req - 1));
1215 }
1216 if (extra_offset)
1217 PKTPULL(di->osh, p, extra_offset);
1218
1219 #ifdef CTFMAP
1220 /* mark as ctf buffer for fast mapping */
1221 if(CTF_ENAB(kcih)) {
1222 ASSERT((((uint32)PKTDATA(di->osh, p)) & 31)==0);
1223 PKTSETCTF(di->osh, p);
1224 }
1225 #endif /* CTFMAP */
1226
1227 /* Do a cached write instead of uncached write since DMA_MAP
1228 * will flush the cache.
1229 */
1230 *(uint32 *)(PKTDATA(di->osh, p)) = 0;
1231
1232 if (DMASGLIST_ENAB)
1233 bzero(&di->rxp_dmah[rxout], sizeof(hnddma_seg_map_t));
1234
1235 pa = DMA_MAP(di->osh, PKTDATA(di->osh, p),
1236 di->rxbufsize, DMA_RX, p,
1237 &di->rxp_dmah[rxout]);
1238
1239 #ifdef __mips__
1240 /* DMA_MAP will invalidated the cache but may or maynot write-back
1241 * to cache. So update len uncached.
1242 */
1243 *(uint32 *)OSL_UNCACHED((PKTDATA(di->osh, p))) = 0;
1244 #endif /* __mips__ */
1245
1246 ASSERT(ISALIGNED(PHYSADDRLO(pa), 4));
1247
1248 /* save the free packet pointer */
1249 ASSERT(di->rxp[rxout] == NULL);
1250 di->rxp[rxout] = p;
1251
1252 /* reset flags for each descriptor */
1253 flags = 0;
1254 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1255 if (rxout == (di->nrxd - 1))
1256 flags = D64_CTRL1_EOT;
1257
1258 dma64_dd_upd(di, di->rxd64, pa, rxout, &flags, di->rxbufsize);
1259 } else if (DMA32_ENAB(di)) {
1260 if (rxout == (di->nrxd - 1))
1261 flags = CTRL_EOT;
1262
1263 ASSERT(PHYSADDRHI(pa) == 0);
1264 dma32_dd_upd(di, di->rxd32, pa, rxout, &flags, di->rxbufsize);
1265 } else
1266 ASSERT(0);
1267 rxout = NEXTRXD(rxout);
1268 }
1269
1270 di->rxout = rxout;
1271
1272 /* update the chip lastdscr pointer */
1273 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1274 W_REG(di->osh, &di->d64rxregs->ptr, di->rcvptrbase + I2B(rxout, dma64dd_t));
1275 } else if (DMA32_ENAB(di)) {
1276 W_REG(di->osh, &di->d32rxregs->ptr, I2B(rxout, dma32dd_t));
1277 } else
1278 ASSERT(0);
1279
1280 return ring_empty;
1281 }
1282
1283 /* like getnexttxp but no reclaim */
1284 static void *
1285 _dma_peeknexttxp(dma_info_t *di)
1286 {
1287 uint end, i;
1288
1289 if (di->ntxd == 0)
1290 return (NULL);
1291
1292 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1293 end = B2I(((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_CD_MASK) -
1294 di->xmtptrbase) & D64_XS0_CD_MASK, dma64dd_t);
1295 } else if (DMA32_ENAB(di)) {
1296 end = B2I(R_REG(di->osh, &di->d32txregs->status) & XS_CD_MASK, dma32dd_t);
1297 } else
1298 ASSERT(0);
1299
1300 for (i = di->txin; i != end; i = NEXTTXD(i))
1301 if (di->txp[i])
1302 return (di->txp[i]);
1303
1304 return (NULL);
1305 }
1306
1307 /* like getnextrxp but not take off the ring */
1308 static void *
1309 _dma_peeknextrxp(dma_info_t *di)
1310 {
1311 uint end, i;
1312
1313 if (di->nrxd == 0)
1314 return (NULL);
1315
1316 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1317 end = B2I(((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) -
1318 di->rcvptrbase) & D64_RS0_CD_MASK, dma64dd_t);
1319 } else if (DMA32_ENAB(di)) {
1320 end = B2I(R_REG(di->osh, &di->d32rxregs->status) & RS_CD_MASK, dma32dd_t);
1321 } else
1322 ASSERT(0);
1323
1324 for (i = di->rxin; i != end; i = NEXTRXD(i))
1325 if (di->rxp[i])
1326 return (di->rxp[i]);
1327
1328 return (NULL);
1329 }
1330
1331 static void
1332 _dma_rxreclaim(dma_info_t *di)
1333 {
1334 void *p;
1335
1336 /* "unused local" warning suppression for OSLs that
1337 * define PKTFREE() without using the di->osh arg
1338 */
1339 di = di;
1340
1341 DMA_TRACE(("%s: dma_rxreclaim\n", di->name));
1342
1343 while ((p = _dma_getnextrxp(di, TRUE)))
1344 PKTFREE(di->osh, p, FALSE);
1345 }
1346
1347 static void * BCMFASTPATH
1348 _dma_getnextrxp(dma_info_t *di, bool forceall)
1349 {
1350 if (di->nrxd == 0)
1351 return (NULL);
1352
1353 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1354 return dma64_getnextrxp(di, forceall);
1355 } else if (DMA32_ENAB(di)) {
1356 return dma32_getnextrxp(di, forceall);
1357 } else
1358 ASSERT(0);
1359 }
1360
1361 static void
1362 _dma_txblock(dma_info_t *di)
1363 {
1364 di->hnddma.txavail = 0;
1365 }
1366
1367 static void
1368 _dma_txunblock(dma_info_t *di)
1369 {
1370 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
1371 }
1372
1373 static uint
1374 _dma_txactive(dma_info_t *di)
1375 {
1376 return NTXDACTIVE(di->txin, di->txout);
1377 }
1378
1379 static uint
1380 _dma_txpending(dma_info_t *di)
1381 {
1382 uint curr;
1383
1384 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1385 curr = B2I(((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_CD_MASK) -
1386 di->xmtptrbase) & D64_XS0_CD_MASK, dma64dd_t);
1387 } else if (DMA32_ENAB(di)) {
1388 curr = B2I(R_REG(di->osh, &di->d32txregs->status) & XS_CD_MASK, dma32dd_t);
1389 } else
1390 ASSERT(0);
1391
1392 return NTXDACTIVE(curr, di->txout);
1393 }
1394
1395 static uint
1396 _dma_txcommitted(dma_info_t *di)
1397 {
1398 uint ptr;
1399 uint txin = di->txin;
1400
1401 if (txin == di->txout)
1402 return 0;
1403
1404 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1405 ptr = B2I(R_REG(di->osh, &di->d64txregs->ptr), dma64dd_t);
1406 } else if (DMA32_ENAB(di)) {
1407 ptr = B2I(R_REG(di->osh, &di->d32txregs->ptr), dma32dd_t);
1408 } else
1409 ASSERT(0);
1410
1411 return NTXDACTIVE(di->txin, ptr);
1412 }
1413
1414 static uint
1415 _dma_rxactive(dma_info_t *di)
1416 {
1417 return NRXDACTIVE(di->rxin, di->rxout);
1418 }
1419
1420 static void
1421 _dma_counterreset(dma_info_t *di)
1422 {
1423 /* reset all software counter */
1424 di->hnddma.rxgiants = 0;
1425 di->hnddma.rxnobuf = 0;
1426 di->hnddma.txnobuf = 0;
1427 }
1428
1429 static uint
1430 _dma_ctrlflags(dma_info_t *di, uint mask, uint flags)
1431 {
1432 uint dmactrlflags;
1433
1434 if (!di) {
1435 DMA_ERROR(("%s: _dma_ctrlflags: NULL dma handle\n", di->name));
1436 return (0);
1437 }
1438
1439 dmactrlflags = di->hnddma.dmactrlflags;
1440
1441 ASSERT((flags & ~mask) == 0);
1442
1443 dmactrlflags &= ~mask;
1444 dmactrlflags |= flags;
1445
1446 /* If trying to enable parity, check if parity is actually supported */
1447 if (dmactrlflags & DMA_CTRL_PEN) {
1448 uint32 control;
1449
1450 if (DMA64_ENAB(di) && DMA64_MODE(di)) {
1451 control = R_REG(di->osh, &di->d64txregs->control);
1452 W_REG(di->osh, &di->d64txregs->control, control | D64_XC_PD);
1453 if (R_REG(di->osh, &di->d64txregs->control) & D64_XC_PD) {
1454 /* We *can* disable it so it is supported,
1455 * restore control register
1456 */
1457 W_REG(di->osh, &di->d64txregs->control, control);
1458 } else {
1459 /* Not supported, don't allow it to be enabled */
1460 dmactrlflags &= ~DMA_CTRL_PEN;
1461 }
1462 } else if (DMA32_ENAB(di)) {
1463 control = R_REG(di->osh, &di->d32txregs->control);
1464 W_REG(di->osh, &di->d32txregs->control, control | XC_PD);
1465 if (R_REG(di->osh, &di->d32txregs->control) & XC_PD) {
1466 W_REG(di->osh, &di->d32txregs->control, control);
1467 } else {
1468 /* Not supported, don't allow it to be enabled */
1469 dmactrlflags &= ~DMA_CTRL_PEN;
1470 }
1471 } else
1472 ASSERT(0);
1473 }
1474
1475 di->hnddma.dmactrlflags = dmactrlflags;
1476
1477 return (dmactrlflags);
1478 }
1479
1480 /* get the address of the var in order to change later */
1481 static uintptr
1482 _dma_getvar(dma_info_t *di, const char *name)
1483 {
1484 if (!strcmp(name, "&txavail"))
1485 return ((uintptr) &(di->hnddma.txavail));
1486 else {
1487 ASSERT(0);
1488 }
1489 return (0);
1490 }
1491
1492 static uint
1493 _dma_avoidancecnt(dma_info_t *di)
1494 {
1495 return (di->dma_avoidance_cnt);
1496 }
1497
1498 void
1499 dma_txpioloopback(osl_t *osh, dma32regs_t *regs)
1500 {
1501 OR_REG(osh, &regs->control, XC_LE);
1502 }
1503
1504 static
1505 uint8 dma_align_sizetobits(uint size)
1506 {
1507 uint8 bitpos = 0;
1508 ASSERT(size);
1509 ASSERT(!(size & (size-1)));
1510 while (size >>= 1) {
1511 bitpos ++;
1512 }
1513 return (bitpos);
1514 }
1515
1516 /* This function ensures that the DMA descriptor ring will not get allocated
1517 * across Page boundary. If the allocation is done across the page boundary
1518 * at the first time, then it is freed and the allocation is done at
1519 * descriptor ring size aligned location. This will ensure that the ring will
1520 * not cross page boundary
1521 */
1522 static void *
1523 dma_ringalloc(osl_t *osh, uint32 boundary, uint size, uint16 *alignbits, uint* alloced,
1524 dmaaddr_t *descpa, osldma_t **dmah)
1525 {
1526 void * va;
1527 uint32 desc_strtaddr;
1528 uint32 alignbytes = 1 << *alignbits;
1529
1530 if (NULL == (va = DMA_ALLOC_CONSISTENT(osh, size, *alignbits, alloced, descpa, dmah)))
1531 return NULL;
1532
1533 desc_strtaddr = (uint32)ROUNDUP((uintptr)va, alignbytes);
1534 if (((desc_strtaddr + size - 1) & boundary) != (desc_strtaddr
1535 & boundary)) {
1536 *alignbits = dma_align_sizetobits(size);
1537 DMA_FREE_CONSISTENT(osh, va,
1538 size, *descpa, dmah);
1539 va = DMA_ALLOC_CONSISTENT(osh, size, *alignbits, alloced, descpa, dmah);
1540 }
1541 return va;
1542 }
1543
1544 #if defined(BCMDBG) || defined(BCMDBG_DUMP)
1545 static void
1546 dma32_dumpring(dma_info_t *di, struct bcmstrbuf *b, dma32dd_t *ring, uint start, uint end,
1547 uint max_num)
1548 {
1549 uint i;
1550
1551 for (i = start; i != end; i = XXD((i + 1), max_num)) {
1552 /* in the format of high->low 8 bytes */
1553 bcm_bprintf(b, "ring index %d: 0x%x %x\n",
1554 i, R_SM(&ring[i].addr), R_SM(&ring[i].ctrl));
1555 }
1556 }
1557
1558 static void
1559 dma32_dumptx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring)
1560 {
1561 if (di->ntxd == 0)
1562 return;
1563
1564 bcm_bprintf(b, "DMA32: txd32 %p txdpa 0x%lx txp %p txin %d txout %d "
1565 "txavail %d txnodesc %d\n", di->txd32, PHYSADDRLO(di->txdpa), di->txp, di->txin,
1566 di->txout, di->hnddma.txavail, di->hnddma.txnodesc);
1567
1568 bcm_bprintf(b, "xmtcontrol 0x%x xmtaddr 0x%x xmtptr 0x%x xmtstatus 0x%x\n",
1569 R_REG(di->osh, &di->d32txregs->control),
1570 R_REG(di->osh, &di->d32txregs->addr),
1571 R_REG(di->osh, &di->d32txregs->ptr),
1572 R_REG(di->osh, &di->d32txregs->status));
1573
1574 if (dumpring && di->txd32)
1575 dma32_dumpring(di, b, di->txd32, di->txin, di->txout, di->ntxd);
1576 }
1577
1578 static void
1579 dma32_dumprx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring)
1580 {
1581 if (di->nrxd == 0)
1582 return;
1583
1584 bcm_bprintf(b, "DMA32: rxd32 %p rxdpa 0x%lx rxp %p rxin %d rxout %d\n",
1585 di->rxd32, PHYSADDRLO(di->rxdpa), di->rxp, di->rxin, di->rxout);
1586
1587 bcm_bprintf(b, "rcvcontrol 0x%x rcvaddr 0x%x rcvptr 0x%x rcvstatus 0x%x\n",
1588 R_REG(di->osh, &di->d32rxregs->control),
1589 R_REG(di->osh, &di->d32rxregs->addr),
1590 R_REG(di->osh, &di->d32rxregs->ptr),
1591 R_REG(di->osh, &di->d32rxregs->status));
1592 if (di->rxd32 && dumpring)
1593 dma32_dumpring(di, b, di->rxd32, di->rxin, di->rxout, di->nrxd);
1594 }
1595
1596 static void
1597 dma32_dump(dma_info_t *di, struct bcmstrbuf *b, bool dumpring)
1598 {
1599 dma32_dumptx(di, b, dumpring);
1600 dma32_dumprx(di, b, dumpring);
1601 }
1602
1603 static void
1604 dma64_dumpring(dma_info_t *di, struct bcmstrbuf *b, dma64dd_t *ring, uint start, uint end,
1605 uint max_num)
1606 {
1607 uint i;
1608
1609 for (i = start; i != end; i = XXD((i + 1), max_num)) {
1610 /* in the format of high->low 16 bytes */
1611 bcm_bprintf(b, "ring index %d: 0x%x %x %x %x\n",
1612 i, R_SM(&ring[i].addrhigh), R_SM(&ring[i].addrlow),
1613 R_SM(&ring[i].ctrl2), R_SM(&ring[i].ctrl1));
1614 }
1615 }
1616
1617 static void
1618 dma64_dumptx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring)
1619 {
1620 if (di->ntxd == 0)
1621 return;
1622
1623 bcm_bprintf(b, "DMA64: txd64 %p txdpa 0x%lx txdpahi 0x%lx txp %p txin %d txout %d "
1624 "txavail %d txnodesc %d\n", di->txd64, PHYSADDRLO(di->txdpa),
1625 PHYSADDRHI(di->txdpaorig), di->txp, di->txin, di->txout, di->hnddma.txavail,
1626 di->hnddma.txnodesc);
1627
1628 bcm_bprintf(b, "xmtcontrol 0x%x xmtaddrlow 0x%x xmtaddrhigh 0x%x "
1629 "xmtptr 0x%x xmtstatus0 0x%x xmtstatus1 0x%x\n",
1630 R_REG(di->osh, &di->d64txregs->control),
1631 R_REG(di->osh, &di->d64txregs->addrlow),
1632 R_REG(di->osh, &di->d64txregs->addrhigh),
1633 R_REG(di->osh, &di->d64txregs->ptr),
1634 R_REG(di->osh, &di->d64txregs->status0),
1635 R_REG(di->osh, &di->d64txregs->status1));
1636
1637 bcm_bprintf(b, "DMA64: DMA avoidance applied %d\n", di->dma_avoidance_cnt);
1638
1639 if (dumpring && di->txd64) {
1640 dma64_dumpring(di, b, di->txd64, di->txin, di->txout, di->ntxd);
1641 }
1642 }
1643
1644 static void
1645 dma64_dumprx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring)
1646 {
1647 if (di->nrxd == 0)
1648 return;
1649
1650 bcm_bprintf(b, "DMA64: rxd64 %p rxdpa 0x%lx rxdpahi 0x%lx rxp %p rxin %d rxout %d\n",
1651 di->rxd64, PHYSADDRLO(di->rxdpa), PHYSADDRHI(di->rxdpaorig), di->rxp,
1652 di->rxin, di->rxout);
1653
1654 bcm_bprintf(b, "rcvcontrol 0x%x rcvaddrlow 0x%x rcvaddrhigh 0x%x rcvptr "
1655 "0x%x rcvstatus0 0x%x rcvstatus1 0x%x\n",
1656 R_REG(di->osh, &di->d64rxregs->control),
1657 R_REG(di->osh, &di->d64rxregs->addrlow),
1658 R_REG(di->osh, &di->d64rxregs->addrhigh),
1659 R_REG(di->osh, &di->d64rxregs->ptr),
1660 R_REG(di->osh, &di->d64rxregs->status0),
1661 R_REG(di->osh, &di->d64rxregs->status1));
1662 if (di->rxd64 && dumpring) {
1663 dma64_dumpring(di, b, di->rxd64, di->rxin, di->rxout, di->nrxd);
1664 }
1665 }
1666
1667 static void
1668 dma64_dump(dma_info_t *di, struct bcmstrbuf *b, bool dumpring)
1669 {
1670 dma64_dumptx(di, b, dumpring);
1671 dma64_dumprx(di, b, dumpring);
1672 }
1673
1674 #endif /* BCMDBG || BCMDBG_DUMP */
1675
1676
1677 /* 32-bit DMA functions */
1678
1679 static void
1680 dma32_txinit(dma_info_t *di)
1681 {
1682 uint32 control = XC_XE;
1683
1684 DMA_TRACE(("%s: dma_txinit\n", di->name));
1685
1686 if (di->ntxd == 0)
1687 return;
1688
1689 di->txin = di->txout = 0;
1690 di->hnddma.txavail = di->ntxd - 1;
1691
1692 /* clear tx descriptor ring */
1693 BZERO_SM((void *)(uintptr)di->txd32, (di->ntxd * sizeof(dma32dd_t)));
1694
1695 if ((di->hnddma.dmactrlflags & DMA_CTRL_PEN) == 0)
1696 control |= XC_PD;
1697 W_REG(di->osh, &di->d32txregs->control, control);
1698 _dma_ddtable_init(di, DMA_TX, di->txdpa);
1699 }
1700
1701 static bool
1702 dma32_txenabled(dma_info_t *di)
1703 {
1704 uint32 xc;
1705
1706 /* If the chip is dead, it is not enabled :-) */
1707 xc = R_REG(di->osh, &di->d32txregs->control);
1708 return ((xc != 0xffffffff) && (xc & XC_XE));
1709 }
1710
1711 static void
1712 dma32_txsuspend(dma_info_t *di)
1713 {
1714 DMA_TRACE(("%s: dma_txsuspend\n", di->name));
1715
1716 if (di->ntxd == 0)
1717 return;
1718
1719 OR_REG(di->osh, &di->d32txregs->control, XC_SE);
1720 }
1721
1722 static void
1723 dma32_txresume(dma_info_t *di)
1724 {
1725 DMA_TRACE(("%s: dma_txresume\n", di->name));
1726
1727 if (di->ntxd == 0)
1728 return;
1729
1730 AND_REG(di->osh, &di->d32txregs->control, ~XC_SE);
1731 }
1732
1733 static bool
1734 dma32_txsuspended(dma_info_t *di)
1735 {
1736 return (di->ntxd == 0) || ((R_REG(di->osh, &di->d32txregs->control) & XC_SE) == XC_SE);
1737 }
1738
1739 #ifdef WL_MULTIQUEUE
1740 static void
1741 dma32_txflush(dma_info_t *di)
1742 {
1743 DMA_TRACE(("%s: dma_txflush\n", di->name));
1744
1745 if (di->ntxd == 0)
1746 return;
1747
1748 OR_REG(di->osh, &di->d32txregs->control, XC_SE | XC_FL);
1749 }
1750
1751 static void
1752 dma32_txflush_clear(dma_info_t *di)
1753 {
1754 uint32 status;
1755
1756 DMA_TRACE(("%s: dma_txflush_clear\n", di->name));
1757
1758 if (di->ntxd == 0)
1759 return;
1760
1761 SPINWAIT(((status = (R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK))
1762 != XS_XS_DISABLED) &&
1763 (status != XS_XS_IDLE) &&
1764 (status != XS_XS_STOPPED),
1765 (10000));
1766 AND_REG(di->osh, &di->d32txregs->control, ~XC_FL);
1767 }
1768 #endif /* WL_MULTIQUEUE */
1769
1770 static void
1771 dma32_txreclaim(dma_info_t *di, txd_range_t range)
1772 {
1773 void *p;
1774
1775 DMA_TRACE(("%s: dma_txreclaim %s\n", di->name,
1776 (range == HNDDMA_RANGE_ALL) ? "all" :
1777 ((range == HNDDMA_RANGE_TRANSMITTED) ? "transmitted" : "transfered")));
1778
1779 if (di->txin == di->txout)
1780 return;
1781
1782 while ((p = dma32_getnexttxp(di, range)))
1783 PKTFREE(di->osh, p, TRUE);
1784 }
1785
1786 static bool
1787 dma32_txstopped(dma_info_t *di)
1788 {
1789 return ((R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK) == XS_XS_STOPPED);
1790 }
1791
1792 static bool
1793 dma32_rxstopped(dma_info_t *di)
1794 {
1795 return ((R_REG(di->osh, &di->d32rxregs->status) & RS_RS_MASK) == RS_RS_STOPPED);
1796 }
1797
1798 static bool
1799 dma32_alloc(dma_info_t *di, uint direction)
1800 {
1801 uint size;
1802 uint ddlen;
1803 void *va;
1804 uint alloced;
1805 uint16 align;
1806 uint16 align_bits;
1807
1808 ddlen = sizeof(dma32dd_t);
1809
1810 size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);
1811
1812 alloced = 0;
1813 align_bits = di->dmadesc_align;
1814 align = (1 << align_bits);
1815
1816 if (direction == DMA_TX) {
1817 if ((va = dma_ringalloc(di->osh, D32RINGALIGN, size, &align_bits, &alloced,
1818 &di->txdpaorig, &di->tx_dmah)) == NULL) {
1819 DMA_ERROR(("%s: dma_alloc: DMA_ALLOC_CONSISTENT(ntxd) failed\n",
1820 di->name));
1821 return FALSE;
1822 }
1823
1824 PHYSADDRHISET(di->txdpa, 0);
1825 ASSERT(PHYSADDRHI(di->txdpaorig) == 0);
1826 di->txd32 = (dma32dd_t *)ROUNDUP((uintptr)va, align);
1827 di->txdalign = (uint)((int8 *)(uintptr)di->txd32 - (int8 *)va);
1828
1829 PHYSADDRLOSET(di->txdpa, PHYSADDRLO(di->txdpaorig) + di->txdalign);
1830 /* Make sure that alignment didn't overflow */
1831 ASSERT(PHYSADDRLO(di->txdpa) >= PHYSADDRLO(di->txdpaorig));
1832
1833 di->txdalloc = alloced;
1834 ASSERT(ISALIGNED((uintptr)di->txd32, align));
1835 } else {
1836 if ((va = dma_ringalloc(di->osh, D32RINGALIGN, size, &align_bits, &alloced,
1837 &di->rxdpaorig, &di->rx_dmah)) == NULL) {
1838 DMA_ERROR(("%s: dma_alloc: DMA_ALLOC_CONSISTENT(nrxd) failed\n",
1839 di->name));
1840 return FALSE;
1841 }
1842
1843 PHYSADDRHISET(di->rxdpa, 0);
1844 ASSERT(PHYSADDRHI(di->rxdpaorig) == 0);
1845 di->rxd32 = (dma32dd_t *)ROUNDUP((uintptr)va, align);
1846 di->rxdalign = (uint)((int8 *)(uintptr)di->rxd32 - (int8 *)va);
1847
1848 PHYSADDRLOSET(di->rxdpa, PHYSADDRLO(di->rxdpaorig) + di->rxdalign);
1849 /* Make sure that alignment didn't overflow */
1850 ASSERT(PHYSADDRLO(di->rxdpa) >= PHYSADDRLO(di->rxdpaorig));
1851 di->rxdalloc = alloced;
1852 ASSERT(ISALIGNED((uintptr)di->rxd32, align));
1853 }
1854
1855 return TRUE;
1856 }
1857
1858 static bool
1859 dma32_txreset(dma_info_t *di)
1860 {
1861 uint32 status;
1862
1863 if (di->ntxd == 0)
1864 return TRUE;
1865
1866 /* suspend tx DMA first */
1867 W_REG(di->osh, &di->d32txregs->control, XC_SE);
1868 SPINWAIT(((status = (R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK))
1869 != XS_XS_DISABLED) &&
1870 (status != XS_XS_IDLE) &&
1871 (status != XS_XS_STOPPED),
1872 (10000));
1873
1874 W_REG(di->osh, &di->d32txregs->control, 0);
1875 SPINWAIT(((status = (R_REG(di->osh,
1876 &di->d32txregs->status) & XS_XS_MASK)) != XS_XS_DISABLED),
1877 10000);
1878
1879 /* wait for the last transaction to complete */
1880 OSL_DELAY(300);
1881
1882 return (status == XS_XS_DISABLED);
1883 }
1884
1885 static bool
1886 dma32_rxidle(dma_info_t *di)
1887 {
1888 DMA_TRACE(("%s: dma_rxidle\n", di->name));
1889
1890 if (di->nrxd == 0)
1891 return TRUE;
1892
1893 return ((R_REG(di->osh, &di->d32rxregs->status) & RS_CD_MASK) ==
1894 R_REG(di->osh, &di->d32rxregs->ptr));
1895 }
1896
1897 static bool
1898 dma32_rxreset(dma_info_t *di)
1899 {
1900 uint32 status;
1901
1902 if (di->nrxd == 0)
1903 return TRUE;
1904
1905 W_REG(di->osh, &di->d32rxregs->control, 0);
1906 SPINWAIT(((status = (R_REG(di->osh,
1907 &di->d32rxregs->status) & RS_RS_MASK)) != RS_RS_DISABLED),
1908 10000);
1909
1910 return (status == RS_RS_DISABLED);
1911 }
1912
1913 static bool
1914 dma32_rxenabled(dma_info_t *di)
1915 {
1916 uint32 rc;
1917
1918 rc = R_REG(di->osh, &di->d32rxregs->control);
1919 return ((rc != 0xffffffff) && (rc & RC_RE));
1920 }
1921
1922 static bool
1923 dma32_txsuspendedidle(dma_info_t *di)
1924 {
1925 if (di->ntxd == 0)
1926 return TRUE;
1927
1928 if (!(R_REG(di->osh, &di->d32txregs->control) & XC_SE))
1929 return 0;
1930
1931 if ((R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK) != XS_XS_IDLE)
1932 return 0;
1933
1934 OSL_DELAY(2);
1935 return ((R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK) == XS_XS_IDLE);
1936 }
1937
1938 /* !! tx entry routine
1939 * supports full 32bit dma engine buffer addressing so
1940 * dma buffers can cross 4 Kbyte page boundaries.
1941 *
1942 * WARNING: call must check the return value for error.
1943 * the error(toss frames) could be fatal and cause many subsequent hard to debug problems
1944 */
1945 static int
1946 dma32_txfast(dma_info_t *di, void *p0, bool commit)
1947 {
1948 void *p, *next;
1949 uchar *data;
1950 uint len;
1951 uint16 txout;
1952 uint32 flags = 0;
1953 dmaaddr_t pa;
1954
1955 DMA_TRACE(("%s: dma_txfast\n", di->name));
1956
1957 txout = di->txout;
1958
1959 /*
1960 * Walk the chain of packet buffers
1961 * allocating and initializing transmit descriptor entries.
1962 */
1963 for (p = p0; p; p = next) {
1964 uint nsegs, j;
1965 hnddma_seg_map_t *map;
1966
1967 data = PKTDATA(di->osh, p);
1968 len = PKTLEN(di->osh, p);
1969 #ifdef BCM_DMAPAD
1970 len += PKTDMAPAD(di->osh, p);
1971 #endif
1972 next = PKTNEXT(di->osh, p);
1973
1974 /* return nonzero if out of tx descriptors */
1975 if (NEXTTXD(txout) == di->txin)
1976 goto outoftxd;
1977
1978 if (len == 0)
1979 continue;
1980
1981 if (DMASGLIST_ENAB)
1982 bzero(&di->txp_dmah[txout], sizeof(hnddma_seg_map_t));
1983
1984 /* get physical address of buffer start */
1985 pa = DMA_MAP(di->osh, data, len, DMA_TX, p, &di->txp_dmah[txout]);
1986
1987 if (DMASGLIST_ENAB) {
1988 map = &di->txp_dmah[txout];
1989
1990 /* See if all the segments can be accounted for */
1991 if (map->nsegs > (uint)(di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1))
1992 goto outoftxd;
1993
1994 nsegs = map->nsegs;
1995 } else
1996 nsegs = 1;
1997
1998 for (j = 1; j <= nsegs; j++) {
1999 flags = 0;
2000 if (p == p0 && j == 1)
2001 flags |= CTRL_SOF;
2002
2003 /* With a DMA segment list, Descriptor table is filled
2004 * using the segment list instead of looping over
2005 * buffers in multi-chain DMA. Therefore, EOF for SGLIST is when
2006 * end of segment list is reached.
2007 */
2008 if ((!DMASGLIST_ENAB && next == NULL) ||
2009 (DMASGLIST_ENAB && j == nsegs))
2010 flags |= (CTRL_IOC | CTRL_EOF);
2011 if (txout == (di->ntxd - 1))
2012 flags |= CTRL_EOT;
2013
2014 if (DMASGLIST_ENAB) {
2015 len = map->segs[j - 1].length;
2016 pa = map->segs[j - 1].addr;
2017 }
2018 ASSERT(PHYSADDRHI(pa) == 0);
2019
2020 dma32_dd_upd(di, di->txd32, pa, txout, &flags, len);
2021 ASSERT(di->txp[txout] == NULL);
2022
2023 txout = NEXTTXD(txout);
2024 }
2025
2026 /* See above. No need to loop over individual buffers */
2027 if (DMASGLIST_ENAB)
2028 break;
2029 }
2030
2031 if (p0 != NULL) {
2032 /* if last txd eof not set, fix it */
2033 if (!(flags & CTRL_EOF))
2034 W_SM(&di->txd32[PREVTXD(txout)].ctrl,
2035 BUS_SWAP32(flags | CTRL_IOC | CTRL_EOF));
2036
2037 /* save the packet */
2038 di->txp[PREVTXD(txout)] = p0;
2039
2040 /* bump the tx descriptor index */
2041 di->txout = txout;
2042 }
2043
2044 /* kick the chip */
2045 if (commit)
2046 W_REG(di->osh, &di->d32txregs->ptr, I2B(txout, dma32dd_t));
2047
2048 /* tx flow control */
2049 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
2050
2051 return (0);
2052
2053 outoftxd:
2054 DMA_ERROR(("%s: dma_txfast: out of txds\n", di->name));
2055 PKTFREE(di->osh, p0, TRUE);
2056 di->hnddma.txavail = 0;
2057 di->hnddma.txnobuf++;
2058 di->hnddma.txnodesc++;
2059 return (-1);
2060 }
2061
2062 /*
2063 * Reclaim next completed txd (txds if using chained buffers) in the range
2064 * specified and return associated packet.
2065 * If range is HNDDMA_RANGE_TRANSMITTED, reclaim descriptors that have be
2066 * transmitted as noted by the hardware "CurrDescr" pointer.
2067 * If range is HNDDMA_RANGE_TRANSFERED, reclaim descriptors that have be
2068 * transfered by the DMA as noted by the hardware "ActiveDescr" pointer.
2069 * If range is HNDDMA_RANGE_ALL, reclaim all txd(s) posted to the ring and
2070 * return associated packet regardless of the value of hardware pointers.
2071 */
2072 static void *
2073 dma32_getnexttxp(dma_info_t *di, txd_range_t range)
2074 {
2075 uint16 start, end, i;
2076 uint16 active_desc;
2077 void *txp;
2078
2079 DMA_TRACE(("%s: dma_getnexttxp %s\n", di->name,
2080 (range == HNDDMA_RANGE_ALL) ? "all" :
2081 ((range == HNDDMA_RANGE_TRANSMITTED) ? "transmitted" : "transfered")));
2082
2083 if (di->ntxd == 0)
2084 return (NULL);
2085
2086 txp = NULL;
2087
2088 start = di->txin;
2089 if (range == HNDDMA_RANGE_ALL)
2090 end = di->txout;
2091 else {
2092 dma32regs_t *dregs = di->d32txregs;
2093
2094 end = (uint16)B2I(R_REG(di->osh, &dregs->status) & XS_CD_MASK, dma32dd_t);
2095
2096 if (range == HNDDMA_RANGE_TRANSFERED) {
2097 active_desc = (uint16)((R_REG(di->osh, &dregs->status) & XS_AD_MASK) >>
2098 XS_AD_SHIFT);
2099 active_desc = (uint16)B2I(active_desc, dma32dd_t);
2100 if (end != active_desc)
2101 end = PREVTXD(active_desc);
2102 }
2103 }
2104
2105 if ((start == 0) && (end > di->txout))
2106 goto bogus;
2107
2108 for (i = start; i != end && !txp; i = NEXTTXD(i)) {
2109 dmaaddr_t pa;
2110 hnddma_seg_map_t *map = NULL;
2111 uint size, j, nsegs;
2112
2113 PHYSADDRLOSET(pa, (BUS_SWAP32(R_SM(&di->txd32[i].addr)) - di->dataoffsetlow));
2114 PHYSADDRHISET(pa, 0);
2115
2116 if (DMASGLIST_ENAB) {
2117 map = &di->txp_dmah[i];
2118 size = map->origsize;
2119 nsegs = map->nsegs;
2120 } else {
2121 size = (BUS_SWAP32(R_SM(&di->txd32[i].ctrl)) & CTRL_BC_MASK);
2122 nsegs = 1;
2123 }
2124
2125 for (j = nsegs; j > 0; j--) {
2126 W_SM(&di->txd32[i].addr, 0xdeadbeef);
2127
2128 txp = di->txp[i];
2129 di->txp[i] = NULL;
2130 if (j > 1)
2131 i = NEXTTXD(i);
2132 }
2133
2134 DMA_UNMAP(di->osh, pa, size, DMA_TX, txp, map);
2135 }
2136
2137 di->txin = i;
2138
2139 /* tx flow control */
2140 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
2141
2142 return (txp);
2143
2144 bogus:
2145 DMA_NONE(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n",
2146 start, end, di->txout, forceall));
2147 return (NULL);
2148 }
2149
2150 static void *
2151 dma32_getnextrxp(dma_info_t *di, bool forceall)
2152 {
2153 uint i, curr;
2154 void *rxp;
2155 dmaaddr_t pa;
2156 /* if forcing, dma engine must be disabled */
2157 ASSERT(!forceall || !dma32_rxenabled(di));
2158
2159 i = di->rxin;
2160
2161 /* return if no packets posted */
2162 if (i == di->rxout)
2163 return (NULL);
2164
2165 curr = B2I(R_REG(di->osh, &di->d32rxregs->status) & RS_CD_MASK, dma32dd_t);
2166
2167 /* ignore curr if forceall */
2168 if (!forceall && (i == curr))
2169 return (NULL);
2170
2171 /* get the packet pointer that corresponds to the rx descriptor */
2172 rxp = di->rxp[i];
2173 ASSERT(rxp);
2174 di->rxp[i] = NULL;
2175
2176 PHYSADDRLOSET(pa, (BUS_SWAP32(R_SM(&di->rxd32[i].addr)) - di->dataoffsetlow));
2177 PHYSADDRHISET(pa, 0);
2178
2179 /* clear this packet from the descriptor ring */
2180 DMA_UNMAP(di->osh, pa,
2181 di->rxbufsize, DMA_RX, rxp, &di->rxp_dmah[i]);
2182
2183 W_SM(&di->rxd32[i].addr, 0xdeadbeef);
2184
2185 di->rxin = NEXTRXD(i);
2186
2187 return (rxp);
2188 }
2189
2190 /*
2191 * Rotate all active tx dma ring entries "forward" by (ActiveDescriptor - txin).
2192 */
2193 static void
2194 dma32_txrotate(dma_info_t *di)
2195 {
2196 uint16 ad;
2197 uint nactive;
2198 uint rot;
2199 uint16 old, new;
2200 uint32 w;
2201 uint16 first, last;
2202
2203 ASSERT(dma32_txsuspendedidle(di));
2204
2205 nactive = _dma_txactive(di);
2206 ad = (uint16) (B2I(((R_REG(di->osh, &di->d32txregs->status) & XS_AD_MASK) >> XS_AD_SHIFT),
2207 dma32dd_t));
2208 rot = TXD(ad - di->txin);
2209
2210 ASSERT(rot < di->ntxd);
2211
2212 /* full-ring case is a lot harder - don't worry about this */
2213 if (rot >= (di->ntxd - nactive)) {
2214 DMA_ERROR(("%s: dma_txrotate: ring full - punt\n", di->name));
2215 return;
2216 }
2217
2218 first = di->txin;
2219 last = PREVTXD(di->txout);
2220
2221 /* move entries starting at last and moving backwards to first */
2222 for (old = last; old != PREVTXD(first); old = PREVTXD(old)) {
2223 new = TXD(old + rot);
2224
2225 /*
2226 * Move the tx dma descriptor.
2227 * EOT is set only in the last entry in the ring.
2228 */
2229 w = BUS_SWAP32(R_SM(&di->txd32[old].ctrl)) & ~CTRL_EOT;
2230 if (new == (di->ntxd - 1))
2231 w |= CTRL_EOT;
2232 W_SM(&di->txd32[new].ctrl, BUS_SWAP32(w));
2233 W_SM(&di->txd32[new].addr, R_SM(&di->txd32[old].addr));
2234
2235 /* zap the old tx dma descriptor address field */
2236 W_SM(&di->txd32[old].addr, BUS_SWAP32(0xdeadbeef));
2237
2238 /* move the corresponding txp[] entry */
2239 ASSERT(di->txp[new] == NULL);
2240 di->txp[new] = di->txp[old];
2241
2242 /* Move the segment map as well */
2243 if (DMASGLIST_ENAB) {
2244 bcopy(&di->txp_dmah[old], &di->txp_dmah[new], sizeof(hnddma_seg_map_t));
2245 bzero(&di->txp_dmah[old], sizeof(hnddma_seg_map_t));
2246 }
2247
2248 di->txp[old] = NULL;
2249 }
2250
2251 /* update txin and txout */
2252 di->txin = ad;
2253 di->txout = TXD(di->txout + rot);
2254 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
2255
2256 /* kick the chip */
2257 W_REG(di->osh, &di->d32txregs->ptr, I2B(di->txout, dma32dd_t));
2258 }
2259
2260 /* 64-bit DMA functions */
2261
2262 static void
2263 dma64_txinit(dma_info_t *di)
2264 {
2265 uint32 control = D64_XC_XE;
2266
2267 DMA_TRACE(("%s: dma_txinit\n", di->name));
2268
2269 if (di->ntxd == 0)
2270 return;
2271
2272 di->txin = di->txout = 0;
2273 di->hnddma.txavail = di->ntxd - 1;
2274
2275 /* clear tx descriptor ring */
2276 BZERO_SM((void *)(uintptr)di->txd64, (di->ntxd * sizeof(dma64dd_t)));
2277
2278 /* DMA engine with out alignment requirement requires table to be inited
2279 * before enabling the engine
2280 */
2281 if (!di->aligndesc_4k)
2282 _dma_ddtable_init(di, DMA_TX, di->txdpa);
2283
2284 if ((di->hnddma.dmactrlflags & DMA_CTRL_PEN) == 0)
2285 control |= D64_XC_PD;
2286 OR_REG(di->osh, &di->d64txregs->control, control);
2287
2288 /* DMA engine with alignment requirement requires table to be inited
2289 * before enabling the engine
2290 */
2291 if (di->aligndesc_4k)
2292 _dma_ddtable_init(di, DMA_TX, di->txdpa);
2293 }
2294
2295 static bool
2296 dma64_txenabled(dma_info_t *di)
2297 {
2298 uint32 xc;
2299
2300 /* If the chip is dead, it is not enabled :-) */
2301 xc = R_REG(di->osh, &di->d64txregs->control);
2302 return ((xc != 0xffffffff) && (xc & D64_XC_XE));
2303 }
2304
2305 static void
2306 dma64_txsuspend(dma_info_t *di)
2307 {
2308 DMA_TRACE(("%s: dma_txsuspend\n", di->name));
2309
2310 if (di->ntxd == 0)
2311 return;
2312
2313 OR_REG(di->osh, &di->d64txregs->control, D64_XC_SE);
2314 }
2315
2316 static void
2317 dma64_txresume(dma_info_t *di)
2318 {
2319 DMA_TRACE(("%s: dma_txresume\n", di->name));
2320
2321 if (di->ntxd == 0)
2322 return;
2323
2324 AND_REG(di->osh, &di->d64txregs->control, ~D64_XC_SE);
2325 }
2326
2327 static bool
2328 dma64_txsuspended(dma_info_t *di)
2329 {
2330 return (di->ntxd == 0) ||
2331 ((R_REG(di->osh, &di->d64txregs->control) & D64_XC_SE) == D64_XC_SE);
2332 }
2333
2334 #ifdef WL_MULTIQUEUE
2335 static void
2336 dma64_txflush(dma_info_t *di)
2337 {
2338 DMA_TRACE(("%s: dma_txflush\n", di->name));
2339
2340 if (di->ntxd == 0)
2341 return;
2342
2343 OR_REG(di->osh, &di->d64txregs->control, D64_XC_SE | D64_XC_FL);
2344 }
2345
2346 static void
2347 dma64_txflush_clear(dma_info_t *di)
2348 {
2349 uint32 status;
2350
2351 DMA_TRACE(("%s: dma_txflush_clear\n", di->name));
2352
2353 if (di->ntxd == 0)
2354 return;
2355
2356 SPINWAIT(((status = (R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK)) !=
2357 D64_XS0_XS_DISABLED) &&
2358 (status != D64_XS0_XS_IDLE) &&
2359 (status != D64_XS0_XS_STOPPED),
2360 10000);
2361 AND_REG(di->osh, &di->d64txregs->control, ~D64_XC_FL);
2362 }
2363 #endif /* WL_MULTIQUEUE */
2364
2365 static void BCMFASTPATH
2366 dma64_txreclaim(dma_info_t *di, txd_range_t range)
2367 {
2368 void *p;
2369
2370 DMA_TRACE(("%s: dma_txreclaim %s\n", di->name,
2371 (range == HNDDMA_RANGE_ALL) ? "all" :
2372 ((range == HNDDMA_RANGE_TRANSMITTED) ? "transmitted" : "transfered")));
2373
2374 if (di->txin == di->txout)
2375 return;
2376
2377 while ((p = dma64_getnexttxp(di, range))) {
2378 /* For unframed data, we don't have any packets to free */
2379 if (!(di->hnddma.dmactrlflags & DMA_CTRL_UNFRAMED))
2380 PKTFREE(di->osh, p, TRUE);
2381 }
2382 }
2383
2384 static bool
2385 dma64_txstopped(dma_info_t *di)
2386 {
2387 return ((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK) == D64_XS0_XS_STOPPED);
2388 }
2389
2390 static bool
2391 dma64_rxstopped(dma_info_t *di)
2392 {
2393 return ((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_RS_MASK) == D64_RS0_RS_STOPPED);
2394 }
2395
2396 static bool
2397 dma64_alloc(dma_info_t *di, uint direction)
2398 {
2399 uint16 size;
2400 uint ddlen;
2401 void *va;
2402 uint alloced = 0;
2403 uint16 align;
2404 uint16 align_bits;
2405
2406 ddlen = sizeof(dma64dd_t);
2407
2408 size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);
2409 align_bits = di->dmadesc_align;
2410 align = (1 << align_bits);
2411
2412 if (direction == DMA_TX) {
2413 if ((va = dma_ringalloc(di->osh, D64RINGALIGN, size, &align_bits, &alloced,
2414 &di->txdpaorig, &di->tx_dmah)) == NULL) {
2415 DMA_ERROR(("%s: dma64_alloc: DMA_ALLOC_CONSISTENT(ntxd) failed\n",
2416 di->name));
2417 return FALSE;
2418 }
2419 align = (1 << align_bits);
2420 di->txd64 = (dma64dd_t *)ROUNDUP((uintptr)va, align);
2421 di->txdalign = (uint)((int8 *)(uintptr)di->txd64 - (int8 *)va);
2422 PHYSADDRLOSET(di->txdpa, PHYSADDRLO(di->txdpaorig) + di->txdalign);
2423 /* Make sure that alignment didn't overflow */
2424 ASSERT(PHYSADDRLO(di->txdpa) >= PHYSADDRLO(di->txdpaorig));
2425
2426 PHYSADDRHISET(di->txdpa, PHYSADDRHI(di->txdpaorig));
2427 di->txdalloc = alloced;
2428 ASSERT(ISALIGNED((uintptr)di->txd64, align));
2429 } else {
2430 if ((va = dma_ringalloc(di->osh, D64RINGALIGN, size, &align_bits, &alloced,
2431 &di->rxdpaorig, &di->rx_dmah)) == NULL) {
2432 DMA_ERROR(("%s: dma64_alloc: DMA_ALLOC_CONSISTENT(nrxd) failed\n",
2433 di->name));
2434 return FALSE;
2435 }
2436 align = (1 << align_bits);
2437 di->rxd64 = (dma64dd_t *)ROUNDUP((uintptr)va, align);
2438 di->rxdalign = (uint)((int8 *)(uintptr)di->rxd64 - (int8 *)va);
2439 PHYSADDRLOSET(di->rxdpa, PHYSADDRLO(di->rxdpaorig) + di->rxdalign);
2440 /* Make sure that alignment didn't overflow */
2441 ASSERT(PHYSADDRLO(di->rxdpa) >= PHYSADDRLO(di->rxdpaorig));
2442
2443 PHYSADDRHISET(di->rxdpa, PHYSADDRHI(di->rxdpaorig));
2444 di->rxdalloc = alloced;
2445 ASSERT(ISALIGNED((uintptr)di->rxd64, align));
2446 }
2447
2448 return TRUE;
2449 }
2450
2451 static bool
2452 dma64_txreset(dma_info_t *di)
2453 {
2454 uint32 status;
2455
2456 if (di->ntxd == 0)
2457 return TRUE;
2458
2459 /* suspend tx DMA first */
2460 W_REG(di->osh, &di->d64txregs->control, D64_XC_SE);
2461 SPINWAIT(((status = (R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK)) !=
2462 D64_XS0_XS_DISABLED) &&
2463 (status != D64_XS0_XS_IDLE) &&
2464 (status != D64_XS0_XS_STOPPED),
2465 10000);
2466
2467 W_REG(di->osh, &di->d64txregs->control, 0);
2468 SPINWAIT(((status = (R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK)) !=
2469 D64_XS0_XS_DISABLED),
2470 10000);
2471
2472 /* wait for the last transaction to complete */
2473 OSL_DELAY(300);
2474
2475 return (status == D64_XS0_XS_DISABLED);
2476 }
2477
2478 static bool
2479 dma64_rxidle(dma_info_t *di)
2480 {
2481 DMA_TRACE(("%s: dma_rxidle\n", di->name));
2482
2483 if (di->nrxd == 0)
2484 return TRUE;
2485
2486 return ((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) ==
2487 (R_REG(di->osh, &di->d64rxregs->ptr) & D64_RS0_CD_MASK));
2488 }
2489
2490 static bool
2491 dma64_rxreset(dma_info_t *di)
2492 {
2493 uint32 status;
2494
2495 if (di->nrxd == 0)
2496 return TRUE;
2497
2498 W_REG(di->osh, &di->d64rxregs->control, 0);
2499 SPINWAIT(((status = (R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_RS_MASK)) !=
2500 D64_RS0_RS_DISABLED), 10000);
2501
2502 return (status == D64_RS0_RS_DISABLED);
2503 }
2504
2505 static bool
2506 dma64_rxenabled(dma_info_t *di)
2507 {
2508 uint32 rc;
2509
2510 rc = R_REG(di->osh, &di->d64rxregs->control);
2511 return ((rc != 0xffffffff) && (rc & D64_RC_RE));
2512 }
2513
2514 static bool
2515 dma64_txsuspendedidle(dma_info_t *di)
2516 {
2517
2518 if (di->ntxd == 0)
2519 return TRUE;
2520
2521 if (!(R_REG(di->osh, &di->d64txregs->control) & D64_XC_SE))
2522 return 0;
2523
2524 if ((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK) == D64_XS0_XS_IDLE)
2525 return 1;
2526
2527 return 0;
2528 }
2529
2530 #if !defined(_CFE_)
2531 /* Useful when sending unframed data. This allows us to get a progress report from the DMA.
2532 * We return a pointer to the beginning of the DATA buffer of the current descriptor.
2533 * If DMA is idle, we return NULL.
2534 */
2535 static void*
2536 dma64_getpos(dma_info_t *di, bool direction)
2537 {
2538 void *va;
2539 bool idle;
2540 uint32 cur_idx;
2541
2542 if (direction == DMA_TX) {
2543 cur_idx = B2I(((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_CD_MASK) -
2544 di->xmtptrbase) & D64_XS0_CD_MASK, dma64dd_t);
2545 idle = !NTXDACTIVE(di->txin, di->txout);
2546 va = di->txp[cur_idx];
2547 } else {
2548 cur_idx = B2I(((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) -
2549 di->rcvptrbase) & D64_RS0_CD_MASK, dma64dd_t);
2550 idle = !NRXDACTIVE(di->rxin, di->rxout);
2551 va = di->rxp[cur_idx];
2552 }
2553
2554 /* If DMA is IDLE, return NULL */
2555 if (idle) {
2556 DMA_TRACE(("%s: DMA idle, return NULL\n", __FUNCTION__));
2557 va = NULL;
2558 }
2559
2560 return va;
2561 }
2562
2563 /* TX of unframed data
2564 *
2565 * Adds a DMA ring descriptor for the data pointed to by "buf".
2566 * This is for DMA of a buffer of data and is unlike other hnddma TX functions
2567 * that take a pointer to a "packet"
2568 * Each call to this is results in a single descriptor being added for "len" bytes of
2569 * data starting at "buf", it doesn't handle chained buffers.
2570 */
2571 static int
2572 dma64_txunframed(dma_info_t *di, void *buf, uint len, bool commit)
2573 {
2574 uint16 txout;
2575 uint32 flags = 0;
2576 dmaaddr_t pa; /* phys addr */
2577
2578 txout = di->txout;
2579
2580 /* return nonzero if out of tx descriptors */
2581 if (NEXTTXD(txout) == di->txin)
2582 goto outoftxd;
2583
2584 if (len == 0)
2585 return 0;
2586
2587 pa = DMA_MAP(di->osh, buf, len, DMA_TX, NULL, &di->txp_dmah[txout]);
2588
2589 flags = (D64_CTRL1_SOF | D64_CTRL1_IOC | D64_CTRL1_EOF);
2590
2591 if (txout == (di->ntxd - 1))
2592 flags |= D64_CTRL1_EOT;
2593
2594 dma64_dd_upd(di, di->txd64, pa, txout, &flags, len);
2595 ASSERT(di->txp[txout] == NULL);
2596
2597 /* save the buffer pointer - used by dma_getpos */
2598 di->txp[txout] = buf;
2599
2600 txout = NEXTTXD(txout);
2601 /* bump the tx descriptor index */
2602 di->txout = txout;
2603
2604 /* kick the chip */
2605 if (commit) {
2606 W_REG(di->osh, &di->d64txregs->ptr, di->xmtptrbase + I2B(txout, dma64dd_t));
2607 }
2608
2609 /* tx flow control */
2610 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
2611
2612 return (0);
2613
2614 outoftxd:
2615 DMA_ERROR(("%s: %s: out of txds !!!\n", di->name, __FUNCTION__));
2616 di->hnddma.txavail = 0;
2617 di->hnddma.txnobuf++;
2618 return (-1);
2619 }
2620 #endif /* !_CFE_ */
2621
2622
2623 /* !! tx entry routine
2624 * WARNING: call must check the return value for error.
2625 * the error(toss frames) could be fatal and cause many subsequent hard to debug problems
2626 */
2627 static int BCMFASTPATH
2628 dma64_txfast(dma_info_t *di, void *p0, bool commit)
2629 {
2630 void *p, *next;
2631 uchar *data;
2632 uint len;
2633 uint16 txout;
2634 uint32 flags = 0;
2635 dmaaddr_t pa;
2636 bool war;
2637
2638 DMA_TRACE(("%s: dma_txfast\n", di->name));
2639
2640 txout = di->txout;
2641 war = (di->hnddma.dmactrlflags & DMA_CTRL_DMA_AVOIDANCE_WAR) ? TRUE : FALSE;
2642
2643 /*
2644 * Walk the chain of packet buffers
2645 * allocating and initializing transmit descriptor entries.
2646 */
2647 for (p = p0; p; p = next) {
2648 uint nsegs, j, segsadd;
2649 hnddma_seg_map_t *map = NULL;
2650
2651 data = PKTDATA(di->osh, p);
2652 len = PKTLEN(di->osh, p);
2653 #ifdef BCM_DMAPAD
2654 len += PKTDMAPAD(di->osh, p);
2655 #endif /* BCM_DMAPAD */
2656 next = PKTNEXT(di->osh, p);
2657
2658 /* return nonzero if out of tx descriptors */
2659 if (NEXTTXD(txout) == di->txin)
2660 goto outoftxd;
2661
2662 if (len == 0)
2663 continue;
2664
2665 /* get physical address of buffer start */
2666 if (DMASGLIST_ENAB)
2667 bzero(&di->txp_dmah[txout], sizeof(hnddma_seg_map_t));
2668
2669 pa = DMA_MAP(di->osh, data, len, DMA_TX, p, &di->txp_dmah[txout]);
2670
2671 if (DMASGLIST_ENAB) {
2672 map = &di->txp_dmah[txout];
2673
2674 /* See if all the segments can be accounted for */
2675 if (map->nsegs > (uint)(di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1))
2676 goto outoftxd;
2677
2678 nsegs = map->nsegs;
2679 } else
2680 nsegs = 1;
2681
2682 segsadd = 0;
2683 for (j = 1; j <= nsegs; j++) {
2684 flags = 0;
2685 if (p == p0 && j == 1)
2686 flags |= D64_CTRL1_SOF;
2687
2688 /* With a DMA segment list, Descriptor table is filled
2689 * using the segment list instead of looping over
2690 * buffers in multi-chain DMA. Therefore, EOF for SGLIST is when
2691 * end of segment list is reached.
2692 */
2693 if ((!DMASGLIST_ENAB && next == NULL) ||
2694 (DMASGLIST_ENAB && j == nsegs))
2695 flags |= (D64_CTRL1_IOC | D64_CTRL1_EOF);
2696 if (txout == (di->ntxd - 1))
2697 flags |= D64_CTRL1_EOT;
2698
2699 if (DMASGLIST_ENAB) {
2700 len = map->segs[j - 1].length;
2701 pa = map->segs[j - 1].addr;
2702 if (len > 128 && war) {
2703 uint remain, new_len, align64;
2704 /* check for 64B aligned of pa */
2705 align64 = (uint)(PHYSADDRLO(pa) & 0x3f);
2706 align64 = (64 - align64) & 0x3f;
2707 new_len = len - align64;
2708 remain = new_len % 128;
2709 if (remain > 0 && remain <= 4) {
2710 uint32 buf_addr_lo;
2711 uint32 tmp_flags =
2712 flags & (~(D64_CTRL1_EOF | D64_CTRL1_IOC));
2713 flags &= ~(D64_CTRL1_SOF | D64_CTRL1_EOT);
2714 remain += 64;
2715 dma64_dd_upd(di, di->txd64, pa, txout,
2716 &tmp_flags, len-remain);
2717 ASSERT(di->txp[txout] == NULL);
2718 txout = NEXTTXD(txout);
2719 /* return nonzero if out of tx descriptors */
2720 if (txout == di->txin) {
2721 DMA_ERROR(("%s: dma_txfast: Out-of-DMA"
2722 " descriptors (txin %d txout %d"
2723 " nsegs %d)\n", __FUNCTION__,
2724 di->txin, di->txout, nsegs));
2725 goto outoftxd;
2726 }
2727 if (txout == (di->ntxd - 1))
2728 flags |= D64_CTRL1_EOT;
2729 buf_addr_lo = PHYSADDRLO(pa);
2730 PHYSADDRLOSET(pa, (PHYSADDRLO(pa) + (len-remain)));
2731 if (PHYSADDRLO(pa) < buf_addr_lo) {
2732 PHYSADDRHISET(pa, (PHYSADDRHI(pa) + 1));
2733 }
2734 len = remain;
2735 segsadd++;
2736 di->dma_avoidance_cnt++;
2737 }
2738 }
2739 }
2740 dma64_dd_upd(di, di->txd64, pa, txout, &flags, len);
2741 ASSERT(di->txp[txout] == NULL);
2742
2743 txout = NEXTTXD(txout);
2744 /* return nonzero if out of tx descriptors */
2745 if (txout == di->txin) {
2746 DMA_ERROR(("%s: dma_txfast: Out-of-DMA descriptors"
2747 " (txin %d txout %d nsegs %d)\n", __FUNCTION__,
2748 di->txin, di->txout, nsegs));
2749 goto outoftxd;
2750 }
2751 }
2752 if (segsadd && DMASGLIST_ENAB)
2753 map->nsegs += segsadd;
2754
2755 /* See above. No need to loop over individual buffers */
2756 if (DMASGLIST_ENAB)
2757 break;
2758 }
2759
2760 if (p0 != NULL) {
2761 /* if last txd eof not set, fix it */
2762 if (!(flags & D64_CTRL1_EOF))
2763 W_SM(&di->txd64[PREVTXD(txout)].ctrl1,
2764 BUS_SWAP32(flags | D64_CTRL1_IOC | D64_CTRL1_EOF));
2765
2766 /* save the packet */
2767 di->txp[PREVTXD(txout)] = p0;
2768
2769 /* bump the tx descriptor index */
2770 di->txout = txout;
2771 }
2772
2773 /* kick the chip */
2774 if (commit)
2775 W_REG(di->osh, &di->d64txregs->ptr, di->xmtptrbase + I2B(txout, dma64dd_t));
2776
2777 /* tx flow control */
2778 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
2779
2780 return (0);
2781
2782 outoftxd:
2783 DMA_ERROR(("%s: dma_txfast: out of txds !!!\n", di->name));
2784 PKTFREE(di->osh, p0, TRUE);
2785 di->hnddma.txavail = 0;
2786 di->hnddma.txnobuf++;
2787 return (-1);
2788 }
2789
2790 /*
2791 * Reclaim next completed txd (txds if using chained buffers) in the range
2792 * specified and return associated packet.
2793 * If range is HNDDMA_RANGE_TRANSMITTED, reclaim descriptors that have be
2794 * transmitted as noted by the hardware "CurrDescr" pointer.
2795 * If range is HNDDMA_RANGE_TRANSFERED, reclaim descriptors that have be
2796 * transfered by the DMA as noted by the hardware "ActiveDescr" pointer.
2797 * If range is HNDDMA_RANGE_ALL, reclaim all txd(s) posted to the ring and
2798 * return associated packet regardless of the value of hardware pointers.
2799 */
2800 static void * BCMFASTPATH
2801 dma64_getnexttxp(dma_info_t *di, txd_range_t range)
2802 {
2803 uint16 start, end, i;
2804 uint16 active_desc;
2805 void *txp;
2806
2807 DMA_TRACE(("%s: dma_getnexttxp %s\n", di->name,
2808 (range == HNDDMA_RANGE_ALL) ? "all" :
2809 ((range == HNDDMA_RANGE_TRANSMITTED) ? "transmitted" : "transfered")));
2810
2811 if (di->ntxd == 0)
2812 return (NULL);
2813
2814 txp = NULL;
2815
2816 start = di->txin;
2817 if (range == HNDDMA_RANGE_ALL)
2818 end = di->txout;
2819 else {
2820 dma64regs_t *dregs = di->d64txregs;
2821
2822 end = (uint16)(B2I(((R_REG(di->osh, &dregs->status0) & D64_XS0_CD_MASK) -
2823 di->xmtptrbase) & D64_XS0_CD_MASK, dma64dd_t));
2824
2825 if (range == HNDDMA_RANGE_TRANSFERED) {
2826 active_desc = (uint16)(R_REG(di->osh, &dregs->status1) & D64_XS1_AD_MASK);
2827 active_desc = (active_desc - di->xmtptrbase) & D64_XS0_CD_MASK;
2828 active_desc = B2I(active_desc, dma64dd_t);
2829 if (end != active_desc)
2830 end = PREVTXD(active_desc);
2831 }
2832 }
2833
2834 if ((start == 0) && (end > di->txout))
2835 goto bogus;
2836
2837 for (i = start; i != end && !txp; i = NEXTTXD(i)) {
2838 dmaaddr_t pa;
2839 hnddma_seg_map_t *map = NULL;
2840 uint size, j, nsegs;
2841
2842 PHYSADDRLOSET(pa, (BUS_SWAP32(R_SM(&di->txd64[i].addrlow)) - di->dataoffsetlow));
2843 PHYSADDRHISET(pa, (BUS_SWAP32(R_SM(&di->txd64[i].addrhigh)) - di->dataoffsethigh));
2844
2845 if (DMASGLIST_ENAB) {
2846 map = &di->txp_dmah[i];
2847 size = map->origsize;
2848 nsegs = map->nsegs;
2849 } else {
2850 size = (BUS_SWAP32(R_SM(&di->txd64[i].ctrl2)) & D64_CTRL2_BC_MASK);
2851 nsegs = 1;
2852 }
2853
2854 for (j = nsegs; j > 0; j--) {
2855 W_SM(&di->txd64[i].addrlow, 0xdeadbeef);
2856 W_SM(&di->txd64[i].addrhigh, 0xdeadbeef);
2857
2858 txp = di->txp[i];
2859 di->txp[i] = NULL;
2860 if (j > 1)
2861 i = NEXTTXD(i);
2862 }
2863
2864 DMA_UNMAP(di->osh, pa, size, DMA_TX, txp, map);
2865 }
2866
2867 di->txin = i;
2868
2869 /* tx flow control */
2870 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
2871
2872 return (txp);
2873
2874 bogus:
2875 DMA_NONE(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n",
2876 start, end, di->txout, forceall));
2877 return (NULL);
2878 }
2879
2880 static void * BCMFASTPATH
2881 dma64_getnextrxp(dma_info_t *di, bool forceall)
2882 {
2883 uint i, curr;
2884 void *rxp;
2885 dmaaddr_t pa;
2886
2887 /* if forcing, dma engine must be disabled */
2888 ASSERT(!forceall || !dma64_rxenabled(di));
2889
2890 i = di->rxin;
2891
2892 /* return if no packets posted */
2893 if (i == di->rxout)
2894 return (NULL);
2895
2896 curr = B2I(((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) -
2897 di->rcvptrbase) & D64_RS0_CD_MASK, dma64dd_t);
2898
2899 /* ignore curr if forceall */
2900 if (!forceall && (i == curr))
2901 return (NULL);
2902
2903 /* get the packet pointer that corresponds to the rx descriptor */
2904 rxp = di->rxp[i];
2905 ASSERT(rxp);
2906 di->rxp[i] = NULL;
2907
2908 PHYSADDRLOSET(pa, (BUS_SWAP32(R_SM(&di->rxd64[i].addrlow)) - di->dataoffsetlow));
2909 PHYSADDRHISET(pa, (BUS_SWAP32(R_SM(&di->rxd64[i].addrhigh)) - di->dataoffsethigh));
2910
2911 /* clear this packet from the descriptor ring */
2912 DMA_UNMAP(di->osh, pa,
2913 di->rxbufsize, DMA_RX, rxp, &di->rxp_dmah[i]);
2914
2915 W_SM(&di->rxd64[i].addrlow, 0xdeadbeef);
2916 W_SM(&di->rxd64[i].addrhigh, 0xdeadbeef);
2917
2918 di->rxin = NEXTRXD(i);
2919
2920 return (rxp);
2921 }
2922
2923 static bool
2924 _dma64_addrext(osl_t *osh, dma64regs_t *dma64regs)
2925 {
2926 uint32 w;
2927 OR_REG(osh, &dma64regs->control, D64_XC_AE);
2928 w = R_REG(osh, &dma64regs->control);
2929 AND_REG(osh, &dma64regs->control, ~D64_XC_AE);
2930 return ((w & D64_XC_AE) == D64_XC_AE);
2931 }
2932
2933 /*
2934 * Rotate all active tx dma ring entries "forward" by (ActiveDescriptor - txin).
2935 */
2936 static void
2937 dma64_txrotate(dma_info_t *di)
2938 {
2939 uint16 ad;
2940 uint nactive;
2941 uint rot;
2942 uint16 old, new;
2943 uint32 w;
2944 uint16 first, last;
2945
2946 ASSERT(dma64_txsuspendedidle(di));
2947
2948 nactive = _dma_txactive(di);
2949 ad = (uint16)(B2I((((R_REG(di->osh, &di->d64txregs->status1) & D64_XS1_AD_MASK)
2950 - di->xmtptrbase) & D64_XS1_AD_MASK), dma64dd_t));
2951 rot = TXD(ad - di->txin);
2952
2953 ASSERT(rot < di->ntxd);
2954
2955 /* full-ring case is a lot harder - don't worry about this */
2956 if (rot >= (di->ntxd - nactive)) {
2957 DMA_ERROR(("%s: dma_txrotate: ring full - punt\n", di->name));
2958 return;
2959 }
2960
2961 first = di->txin;
2962 last = PREVTXD(di->txout);
2963
2964 /* move entries starting at last and moving backwards to first */
2965 for (old = last; old != PREVTXD(first); old = PREVTXD(old)) {
2966 new = TXD(old + rot);
2967
2968 /*
2969 * Move the tx dma descriptor.
2970 * EOT is set only in the last entry in the ring.
2971 */
2972 w = BUS_SWAP32(R_SM(&di->txd64[old].ctrl1)) & ~D64_CTRL1_EOT;
2973 if (new == (di->ntxd - 1))
2974 w |= D64_CTRL1_EOT;
2975 W_SM(&di->txd64[new].ctrl1, BUS_SWAP32(w));
2976
2977 w = BUS_SWAP32(R_SM(&di->txd64[old].ctrl2));
2978 W_SM(&di->txd64[new].ctrl2, BUS_SWAP32(w));
2979
2980 W_SM(&di->txd64[new].addrlow, R_SM(&di->txd64[old].addrlow));
2981 W_SM(&di->txd64[new].addrhigh, R_SM(&di->txd64[old].addrhigh));
2982
2983 /* zap the old tx dma descriptor address field */
2984 W_SM(&di->txd64[old].addrlow, BUS_SWAP32(0xdeadbeef));
2985 W_SM(&di->txd64[old].addrhigh, BUS_SWAP32(0xdeadbeef));
2986
2987 /* move the corresponding txp[] entry */
2988 ASSERT(di->txp[new] == NULL);
2989 di->txp[new] = di->txp[old];
2990
2991 /* Move the map */
2992 if (DMASGLIST_ENAB) {
2993 bcopy(&di->txp_dmah[old], &di->txp_dmah[new], sizeof(hnddma_seg_map_t));
2994 bzero(&di->txp_dmah[old], sizeof(hnddma_seg_map_t));
2995 }
2996
2997 di->txp[old] = NULL;
2998 }
2999
3000 /* update txin and txout */
3001 di->txin = ad;
3002 di->txout = TXD(di->txout + rot);
3003 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1;
3004
3005 /* kick the chip */
3006 W_REG(di->osh, &di->d64txregs->ptr, di->xmtptrbase + I2B(di->txout, dma64dd_t));
3007 }
3008
3009 uint
3010 dma_addrwidth(si_t *sih, void *dmaregs)
3011 {
3012 dma32regs_t *dma32regs;
3013 osl_t *osh;
3014
3015 osh = si_osh(sih);
3016
3017 /* Perform 64-bit checks only if we want to advertise 64-bit (> 32bit) capability) */
3018 /* DMA engine is 64-bit capable */
3019 if ((si_core_sflags(sih, 0, 0) & SISF_DMA64) == SISF_DMA64) {
3020 /* backplane are 64-bit capable */
3021 if (si_backplane64(sih))
3022 /* If bus is System Backplane or PCIE then we can access 64-bits */
3023 if ((BUSTYPE(sih->bustype) == SI_BUS) ||
3024 ((BUSTYPE(sih->bustype) == PCI_BUS) &&
3025 (sih->buscoretype == PCIE_CORE_ID)))
3026 return (DMADDRWIDTH_64);
3027
3028 /* DMA64 is always 32-bit capable, AE is always TRUE */
3029 ASSERT(_dma64_addrext(osh, (dma64regs_t *)dmaregs));
3030
3031 return (DMADDRWIDTH_32);
3032 }
3033
3034 /* Start checking for 32-bit / 30-bit addressing */
3035 dma32regs = (dma32regs_t *)dmaregs;
3036
3037 /* For System Backplane, PCIE bus or addrext feature, 32-bits ok */
3038 if ((BUSTYPE(sih->bustype) == SI_BUS) ||
3039 ((BUSTYPE(sih->bustype) == PCI_BUS) && sih->buscoretype == PCIE_CORE_ID) ||
3040 (_dma32_addrext(osh, dma32regs)))
3041 return (DMADDRWIDTH_32);
3042
3043 /* Fallthru */
3044 return (DMADDRWIDTH_30);
3045 }
Tomato firmware and modifications.