?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
dmaengine/dw_dmac: Replace spin_lock* with irqsave variants and enable submission...
[linux-2.6.git] / drivers / dma / dw_dmac.c
blob442b98b81e7cae27f8c888a5cf0d65fb36c5192b
1 /*
2 * Driver for the Synopsys DesignWare DMA Controller (aka DMACA on
3 * AVR32 systems.)
4 *
5 * Copyright (C) 2007-2008 Atmel Corporation
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11 #include <linux/clk.h>
12 #include <linux/delay.h>
13 #include <linux/dmaengine.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/init.h>
16 #include <linux/interrupt.h>
17 #include <linux/io.h>
18 #include <linux/mm.h>
19 #include <linux/module.h>
20 #include <linux/platform_device.h>
21 #include <linux/slab.h>
22
23 #include "dw_dmac_regs.h"
24
25 /*
26 * This supports the Synopsys "DesignWare AHB Central DMA Controller",
27 * (DW_ahb_dmac) which is used with various AMBA 2.0 systems (not all
28 * of which use ARM any more). See the "Databook" from Synopsys for
29 * information beyond what licensees probably provide.
30 *
31 * The driver has currently been tested only with the Atmel AT32AP7000,
32 * which does not support descriptor writeback.
33 */
34
35 #define DWC_DEFAULT_CTLLO(private) ({ \
36 struct dw_dma_slave *__slave = (private); \
37 int dms = __slave ? __slave->dst_master : 0; \
38 int sms = __slave ? __slave->src_master : 1; \
39 u8 smsize = __slave ? __slave->src_msize : DW_DMA_MSIZE_16; \
40 u8 dmsize = __slave ? __slave->dst_msize : DW_DMA_MSIZE_16; \
41 \
42 (DWC_CTLL_DST_MSIZE(dmsize) \
43 | DWC_CTLL_SRC_MSIZE(smsize) \
44 | DWC_CTLL_LLP_D_EN \
45 | DWC_CTLL_LLP_S_EN \
46 | DWC_CTLL_DMS(dms) \
47 | DWC_CTLL_SMS(sms)); \
48 })
49
50 /*
51 * This is configuration-dependent and usually a funny size like 4095.
52 *
53 * Note that this is a transfer count, i.e. if we transfer 32-bit
54 * words, we can do 16380 bytes per descriptor.
55 *
56 * This parameter is also system-specific.
57 */
58 #define DWC_MAX_COUNT 4095U
59
60 /*
61 * Number of descriptors to allocate for each channel. This should be
62 * made configurable somehow; preferably, the clients (at least the
63 * ones using slave transfers) should be able to give us a hint.
64 */
65 #define NR_DESCS_PER_CHANNEL 64
66
67 /*----------------------------------------------------------------------*/
68
69 /*
70 * Because we're not relying on writeback from the controller (it may not
71 * even be configured into the core!) we don't need to use dma_pool. These
72 * descriptors -- and associated data -- are cacheable. We do need to make
73 * sure their dcache entries are written back before handing them off to
74 * the controller, though.
75 */
76
77 static struct device *chan2dev(struct dma_chan *chan)
78 {
79 return &chan->dev->device;
80 }
81 static struct device *chan2parent(struct dma_chan *chan)
82 {
83 return chan->dev->device.parent;
84 }
85
86 static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc)
87 {
88 return list_entry(dwc->active_list.next, struct dw_desc, desc_node);
89 }
90
91 static struct dw_desc *dwc_desc_get(struct dw_dma_chan *dwc)
92 {
93 struct dw_desc *desc, *_desc;
94 struct dw_desc *ret = NULL;
95 unsigned int i = 0;
96 unsigned long flags;
97
98 spin_lock_irqsave(&dwc->lock, flags);
99 list_for_each_entry_safe(desc, _desc, &dwc->free_list, desc_node) {
100 if (async_tx_test_ack(&desc->txd)) {
101 list_del(&desc->desc_node);
102 ret = desc;
103 break;
104 }
105 dev_dbg(chan2dev(&dwc->chan), "desc %p not ACKed\n", desc);
106 i++;
107 }
108 spin_unlock_irqrestore(&dwc->lock, flags);
109
110 dev_vdbg(chan2dev(&dwc->chan), "scanned %u descriptors on freelist\n", i);
111
112 return ret;
113 }
114
115 static void dwc_sync_desc_for_cpu(struct dw_dma_chan *dwc, struct dw_desc *desc)
116 {
117 struct dw_desc *child;
118
119 list_for_each_entry(child, &desc->tx_list, desc_node)
120 dma_sync_single_for_cpu(chan2parent(&dwc->chan),
121 child->txd.phys, sizeof(child->lli),
122 DMA_TO_DEVICE);
123 dma_sync_single_for_cpu(chan2parent(&dwc->chan),
124 desc->txd.phys, sizeof(desc->lli),
125 DMA_TO_DEVICE);
126 }
127
128 /*
129 * Move a descriptor, including any children, to the free list.
130 * `desc' must not be on any lists.
131 */
132 static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
133 {
134 unsigned long flags;
135
136 if (desc) {
137 struct dw_desc *child;
138
139 dwc_sync_desc_for_cpu(dwc, desc);
140
141 spin_lock_irqsave(&dwc->lock, flags);
142 list_for_each_entry(child, &desc->tx_list, desc_node)
143 dev_vdbg(chan2dev(&dwc->chan),
144 "moving child desc %p to freelist\n",
145 child);
146 list_splice_init(&desc->tx_list, &dwc->free_list);
147 dev_vdbg(chan2dev(&dwc->chan), "moving desc %p to freelist\n", desc);
148 list_add(&desc->desc_node, &dwc->free_list);
149 spin_unlock_irqrestore(&dwc->lock, flags);
150 }
151 }
152
153 /* Called with dwc->lock held and bh disabled */
154 static dma_cookie_t
155 dwc_assign_cookie(struct dw_dma_chan *dwc, struct dw_desc *desc)
156 {
157 dma_cookie_t cookie = dwc->chan.cookie;
158
159 if (++cookie < 0)
160 cookie = 1;
161
162 dwc->chan.cookie = cookie;
163 desc->txd.cookie = cookie;
164
165 return cookie;
166 }
167
168 /*----------------------------------------------------------------------*/
169
170 /* Called with dwc->lock held and bh disabled */
171 static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first)
172 {
173 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
174
175 /* ASSERT: channel is idle */
176 if (dma_readl(dw, CH_EN) & dwc->mask) {
177 dev_err(chan2dev(&dwc->chan),
178 "BUG: Attempted to start non-idle channel\n");
179 dev_err(chan2dev(&dwc->chan),
180 " SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
181 channel_readl(dwc, SAR),
182 channel_readl(dwc, DAR),
183 channel_readl(dwc, LLP),
184 channel_readl(dwc, CTL_HI),
185 channel_readl(dwc, CTL_LO));
186
187 /* The tasklet will hopefully advance the queue... */
188 return;
189 }
190
191 channel_writel(dwc, LLP, first->txd.phys);
192 channel_writel(dwc, CTL_LO,
193 DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
194 channel_writel(dwc, CTL_HI, 0);
195 channel_set_bit(dw, CH_EN, dwc->mask);
196 }
197
198 /*----------------------------------------------------------------------*/
199
200 static void
201 dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,
202 bool callback_required)
203 {
204 dma_async_tx_callback callback = NULL;
205 void *param = NULL;
206 struct dma_async_tx_descriptor *txd = &desc->txd;
207 struct dw_desc *child;
208 unsigned long flags;
209
210 dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie);
211
212 spin_lock_irqsave(&dwc->lock, flags);
213 dwc->completed = txd->cookie;
214 if (callback_required) {
215 callback = txd->callback;
216 param = txd->callback_param;
217 }
218
219 dwc_sync_desc_for_cpu(dwc, desc);
220
221 /* async_tx_ack */
222 list_for_each_entry(child, &desc->tx_list, desc_node)
223 async_tx_ack(&child->txd);
224 async_tx_ack(&desc->txd);
225
226 list_splice_init(&desc->tx_list, &dwc->free_list);
227 list_move(&desc->desc_node, &dwc->free_list);
228
229 if (!dwc->chan.private) {
230 struct device *parent = chan2parent(&dwc->chan);
231 if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
232 if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
233 dma_unmap_single(parent, desc->lli.dar,
234 desc->len, DMA_FROM_DEVICE);
235 else
236 dma_unmap_page(parent, desc->lli.dar,
237 desc->len, DMA_FROM_DEVICE);
238 }
239 if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
240 if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
241 dma_unmap_single(parent, desc->lli.sar,
242 desc->len, DMA_TO_DEVICE);
243 else
244 dma_unmap_page(parent, desc->lli.sar,
245 desc->len, DMA_TO_DEVICE);
246 }
247 }
248
249 spin_unlock_irqrestore(&dwc->lock, flags);
250
251 if (callback_required && callback)
252 callback(param);
253 }
254
255 static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc)
256 {
257 struct dw_desc *desc, *_desc;
258 LIST_HEAD(list);
259 unsigned long flags;
260
261 spin_lock_irqsave(&dwc->lock, flags);
262 if (dma_readl(dw, CH_EN) & dwc->mask) {
263 dev_err(chan2dev(&dwc->chan),
264 "BUG: XFER bit set, but channel not idle!\n");
265
266 /* Try to continue after resetting the channel... */
267 channel_clear_bit(dw, CH_EN, dwc->mask);
268 while (dma_readl(dw, CH_EN) & dwc->mask)
269 cpu_relax();
270 }
271
272 /*
273 * Submit queued descriptors ASAP, i.e. before we go through
274 * the completed ones.
275 */
276 list_splice_init(&dwc->active_list, &list);
277 if (!list_empty(&dwc->queue)) {
278 list_move(dwc->queue.next, &dwc->active_list);
279 dwc_dostart(dwc, dwc_first_active(dwc));
280 }
281
282 spin_unlock_irqrestore(&dwc->lock, flags);
283
284 list_for_each_entry_safe(desc, _desc, &list, desc_node)
285 dwc_descriptor_complete(dwc, desc, true);
286 }
287
288 static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
289 {
290 dma_addr_t llp;
291 struct dw_desc *desc, *_desc;
292 struct dw_desc *child;
293 u32 status_xfer;
294 unsigned long flags;
295
296 spin_lock_irqsave(&dwc->lock, flags);
297 /*
298 * Clear block interrupt flag before scanning so that we don't
299 * miss any, and read LLP before RAW_XFER to ensure it is
300 * valid if we decide to scan the list.
301 */
302 dma_writel(dw, CLEAR.BLOCK, dwc->mask);
303 llp = channel_readl(dwc, LLP);
304 status_xfer = dma_readl(dw, RAW.XFER);
305
306 if (status_xfer & dwc->mask) {
307 /* Everything we've submitted is done */
308 dma_writel(dw, CLEAR.XFER, dwc->mask);
309 spin_unlock_irqrestore(&dwc->lock, flags);
310
311 dwc_complete_all(dw, dwc);
312 return;
313 }
314
315 if (list_empty(&dwc->active_list)) {
316 spin_unlock_irqrestore(&dwc->lock, flags);
317 return;
318 }
319
320 dev_vdbg(chan2dev(&dwc->chan), "scan_descriptors: llp=0x%x\n", llp);
321
322 list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) {
323 /* check first descriptors addr */
324 if (desc->txd.phys == llp) {
325 spin_unlock_irqrestore(&dwc->lock, flags);
326 return;
327 }
328
329 /* check first descriptors llp */
330 if (desc->lli.llp == llp) {
331 /* This one is currently in progress */
332 spin_unlock_irqrestore(&dwc->lock, flags);
333 return;
334 }
335
336 list_for_each_entry(child, &desc->tx_list, desc_node)
337 if (child->lli.llp == llp) {
338 /* Currently in progress */
339 spin_unlock_irqrestore(&dwc->lock, flags);
340 return;
341 }
342
343 /*
344 * No descriptors so far seem to be in progress, i.e.
345 * this one must be done.
346 */
347 spin_unlock_irqrestore(&dwc->lock, flags);
348 dwc_descriptor_complete(dwc, desc, true);
349 spin_lock_irqsave(&dwc->lock, flags);
350 }
351
352 dev_err(chan2dev(&dwc->chan),
353 "BUG: All descriptors done, but channel not idle!\n");
354
355 /* Try to continue after resetting the channel... */
356 channel_clear_bit(dw, CH_EN, dwc->mask);
357 while (dma_readl(dw, CH_EN) & dwc->mask)
358 cpu_relax();
359
360 if (!list_empty(&dwc->queue)) {
361 list_move(dwc->queue.next, &dwc->active_list);
362 dwc_dostart(dwc, dwc_first_active(dwc));
363 }
364 spin_unlock_irqrestore(&dwc->lock, flags);
365 }
366
367 static void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_lli *lli)
368 {
369 dev_printk(KERN_CRIT, chan2dev(&dwc->chan),
370 " desc: s0x%x d0x%x l0x%x c0x%x:%x\n",
371 lli->sar, lli->dar, lli->llp,
372 lli->ctlhi, lli->ctllo);
373 }
374
375 static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
376 {
377 struct dw_desc *bad_desc;
378 struct dw_desc *child;
379 unsigned long flags;
380
381 dwc_scan_descriptors(dw, dwc);
382
383 spin_lock_irqsave(&dwc->lock, flags);
384
385 /*
386 * The descriptor currently at the head of the active list is
387 * borked. Since we don't have any way to report errors, we'll
388 * just have to scream loudly and try to carry on.
389 */
390 bad_desc = dwc_first_active(dwc);
391 list_del_init(&bad_desc->desc_node);
392 list_move(dwc->queue.next, dwc->active_list.prev);
393
394 /* Clear the error flag and try to restart the controller */
395 dma_writel(dw, CLEAR.ERROR, dwc->mask);
396 if (!list_empty(&dwc->active_list))
397 dwc_dostart(dwc, dwc_first_active(dwc));
398
399 /*
400 * KERN_CRITICAL may seem harsh, but since this only happens
401 * when someone submits a bad physical address in a
402 * descriptor, we should consider ourselves lucky that the
403 * controller flagged an error instead of scribbling over
404 * random memory locations.
405 */
406 dev_printk(KERN_CRIT, chan2dev(&dwc->chan),
407 "Bad descriptor submitted for DMA!\n");
408 dev_printk(KERN_CRIT, chan2dev(&dwc->chan),
409 " cookie: %d\n", bad_desc->txd.cookie);
410 dwc_dump_lli(dwc, &bad_desc->lli);
411 list_for_each_entry(child, &bad_desc->tx_list, desc_node)
412 dwc_dump_lli(dwc, &child->lli);
413
414 spin_unlock_irqrestore(&dwc->lock, flags);
415
416 /* Pretend the descriptor completed successfully */
417 dwc_descriptor_complete(dwc, bad_desc, true);
418 }
419
420 /* --------------------- Cyclic DMA API extensions -------------------- */
421
422 inline dma_addr_t dw_dma_get_src_addr(struct dma_chan *chan)
423 {
424 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
425 return channel_readl(dwc, SAR);
426 }
427 EXPORT_SYMBOL(dw_dma_get_src_addr);
428
429 inline dma_addr_t dw_dma_get_dst_addr(struct dma_chan *chan)
430 {
431 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
432 return channel_readl(dwc, DAR);
433 }
434 EXPORT_SYMBOL(dw_dma_get_dst_addr);
435
436 /* called with dwc->lock held and all DMAC interrupts disabled */
437 static void dwc_handle_cyclic(struct dw_dma *dw, struct dw_dma_chan *dwc,
438 u32 status_block, u32 status_err, u32 status_xfer)
439 {
440 unsigned long flags;
441
442 if (status_block & dwc->mask) {
443 void (*callback)(void *param);
444 void *callback_param;
445
446 dev_vdbg(chan2dev(&dwc->chan), "new cyclic period llp 0x%08x\n",
447 channel_readl(dwc, LLP));
448 dma_writel(dw, CLEAR.BLOCK, dwc->mask);
449
450 callback = dwc->cdesc->period_callback;
451 callback_param = dwc->cdesc->period_callback_param;
452
453 if (callback)
454 callback(callback_param);
455 }
456
457 /*
458 * Error and transfer complete are highly unlikely, and will most
459 * likely be due to a configuration error by the user.
460 */
461 if (unlikely(status_err & dwc->mask) ||
462 unlikely(status_xfer & dwc->mask)) {
463 int i;
464
465 dev_err(chan2dev(&dwc->chan), "cyclic DMA unexpected %s "
466 "interrupt, stopping DMA transfer\n",
467 status_xfer ? "xfer" : "error");
468
469 spin_lock_irqsave(&dwc->lock, flags);
470
471 dev_err(chan2dev(&dwc->chan),
472 " SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
473 channel_readl(dwc, SAR),
474 channel_readl(dwc, DAR),
475 channel_readl(dwc, LLP),
476 channel_readl(dwc, CTL_HI),
477 channel_readl(dwc, CTL_LO));
478
479 channel_clear_bit(dw, CH_EN, dwc->mask);
480 while (dma_readl(dw, CH_EN) & dwc->mask)
481 cpu_relax();
482
483 /* make sure DMA does not restart by loading a new list */
484 channel_writel(dwc, LLP, 0);
485 channel_writel(dwc, CTL_LO, 0);
486 channel_writel(dwc, CTL_HI, 0);
487
488 dma_writel(dw, CLEAR.BLOCK, dwc->mask);
489 dma_writel(dw, CLEAR.ERROR, dwc->mask);
490 dma_writel(dw, CLEAR.XFER, dwc->mask);
491
492 for (i = 0; i < dwc->cdesc->periods; i++)
493 dwc_dump_lli(dwc, &dwc->cdesc->desc[i]->lli);
494
495 spin_unlock_irqrestore(&dwc->lock, flags);
496 }
497 }
498
499 /* ------------------------------------------------------------------------- */
500
501 static void dw_dma_tasklet(unsigned long data)
502 {
503 struct dw_dma *dw = (struct dw_dma *)data;
504 struct dw_dma_chan *dwc;
505 u32 status_block;
506 u32 status_xfer;
507 u32 status_err;
508 int i;
509
510 status_block = dma_readl(dw, RAW.BLOCK);
511 status_xfer = dma_readl(dw, RAW.XFER);
512 status_err = dma_readl(dw, RAW.ERROR);
513
514 dev_vdbg(dw->dma.dev, "tasklet: status_block=%x status_err=%x\n",
515 status_block, status_err);
516
517 for (i = 0; i < dw->dma.chancnt; i++) {
518 dwc = &dw->chan[i];
519 if (test_bit(DW_DMA_IS_CYCLIC, &dwc->flags))
520 dwc_handle_cyclic(dw, dwc, status_block, status_err,
521 status_xfer);
522 else if (status_err & (1 << i))
523 dwc_handle_error(dw, dwc);
524 else if ((status_block | status_xfer) & (1 << i))
525 dwc_scan_descriptors(dw, dwc);
526 }
527
528 /*
529 * Re-enable interrupts. Block Complete interrupts are only
530 * enabled if the INT_EN bit in the descriptor is set. This
531 * will trigger a scan before the whole list is done.
532 */
533 channel_set_bit(dw, MASK.XFER, dw->all_chan_mask);
534 channel_set_bit(dw, MASK.BLOCK, dw->all_chan_mask);
535 channel_set_bit(dw, MASK.ERROR, dw->all_chan_mask);
536 }
537
538 static irqreturn_t dw_dma_interrupt(int irq, void *dev_id)
539 {
540 struct dw_dma *dw = dev_id;
541 u32 status;
542
543 dev_vdbg(dw->dma.dev, "interrupt: status=0x%x\n",
544 dma_readl(dw, STATUS_INT));
545
546 /*
547 * Just disable the interrupts. We'll turn them back on in the
548 * softirq handler.
549 */
550 channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
551 channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
552 channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
553
554 status = dma_readl(dw, STATUS_INT);
555 if (status) {
556 dev_err(dw->dma.dev,
557 "BUG: Unexpected interrupts pending: 0x%x\n",
558 status);
559
560 /* Try to recover */
561 channel_clear_bit(dw, MASK.XFER, (1 << 8) - 1);
562 channel_clear_bit(dw, MASK.BLOCK, (1 << 8) - 1);
563 channel_clear_bit(dw, MASK.SRC_TRAN, (1 << 8) - 1);
564 channel_clear_bit(dw, MASK.DST_TRAN, (1 << 8) - 1);
565 channel_clear_bit(dw, MASK.ERROR, (1 << 8) - 1);
566 }
567
568 tasklet_schedule(&dw->tasklet);
569
570 return IRQ_HANDLED;
571 }
572
573 /*----------------------------------------------------------------------*/
574
575 static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx)
576 {
577 struct dw_desc *desc = txd_to_dw_desc(tx);
578 struct dw_dma_chan *dwc = to_dw_dma_chan(tx->chan);
579 dma_cookie_t cookie;
580 unsigned long flags;
581
582 spin_lock_irqsave(&dwc->lock, flags);
583 cookie = dwc_assign_cookie(dwc, desc);
584
585 /*
586 * REVISIT: We should attempt to chain as many descriptors as
587 * possible, perhaps even appending to those already submitted
588 * for DMA. But this is hard to do in a race-free manner.
589 */
590 if (list_empty(&dwc->active_list)) {
591 dev_vdbg(chan2dev(tx->chan), "tx_submit: started %u\n",
592 desc->txd.cookie);
593 list_add_tail(&desc->desc_node, &dwc->active_list);
594 dwc_dostart(dwc, dwc_first_active(dwc));
595 } else {
596 dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u\n",
597 desc->txd.cookie);
598
599 list_add_tail(&desc->desc_node, &dwc->queue);
600 }
601
602 spin_unlock_irqrestore(&dwc->lock, flags);
603
604 return cookie;
605 }
606
607 static struct dma_async_tx_descriptor *
608 dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
609 size_t len, unsigned long flags)
610 {
611 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
612 struct dw_desc *desc;
613 struct dw_desc *first;
614 struct dw_desc *prev;
615 size_t xfer_count;
616 size_t offset;
617 unsigned int src_width;
618 unsigned int dst_width;
619 u32 ctllo;
620
621 dev_vdbg(chan2dev(chan), "prep_dma_memcpy d0x%x s0x%x l0x%zx f0x%lx\n",
622 dest, src, len, flags);
623
624 if (unlikely(!len)) {
625 dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
626 return NULL;
627 }
628
629 /*
630 * We can be a lot more clever here, but this should take care
631 * of the most common optimization.
632 */
633 if (!((src | dest | len) & 7))
634 src_width = dst_width = 3;
635 else if (!((src | dest | len) & 3))
636 src_width = dst_width = 2;
637 else if (!((src | dest | len) & 1))
638 src_width = dst_width = 1;
639 else
640 src_width = dst_width = 0;
641
642 ctllo = DWC_DEFAULT_CTLLO(chan->private)
643 | DWC_CTLL_DST_WIDTH(dst_width)
644 | DWC_CTLL_SRC_WIDTH(src_width)
645 | DWC_CTLL_DST_INC
646 | DWC_CTLL_SRC_INC
647 | DWC_CTLL_FC_M2M;
648 prev = first = NULL;
649
650 for (offset = 0; offset < len; offset += xfer_count << src_width) {
651 xfer_count = min_t(size_t, (len - offset) >> src_width,
652 DWC_MAX_COUNT);
653
654 desc = dwc_desc_get(dwc);
655 if (!desc)
656 goto err_desc_get;
657
658 desc->lli.sar = src + offset;
659 desc->lli.dar = dest + offset;
660 desc->lli.ctllo = ctllo;
661 desc->lli.ctlhi = xfer_count;
662
663 if (!first) {
664 first = desc;
665 } else {
666 prev->lli.llp = desc->txd.phys;
667 dma_sync_single_for_device(chan2parent(chan),
668 prev->txd.phys, sizeof(prev->lli),
669 DMA_TO_DEVICE);
670 list_add_tail(&desc->desc_node,
671 &first->tx_list);
672 }
673 prev = desc;
674 }
675
676
677 if (flags & DMA_PREP_INTERRUPT)
678 /* Trigger interrupt after last block */
679 prev->lli.ctllo |= DWC_CTLL_INT_EN;
680
681 prev->lli.llp = 0;
682 dma_sync_single_for_device(chan2parent(chan),
683 prev->txd.phys, sizeof(prev->lli),
684 DMA_TO_DEVICE);
685
686 first->txd.flags = flags;
687 first->len = len;
688
689 return &first->txd;
690
691 err_desc_get:
692 dwc_desc_put(dwc, first);
693 return NULL;
694 }
695
696 static struct dma_async_tx_descriptor *
697 dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
698 unsigned int sg_len, enum dma_data_direction direction,
699 unsigned long flags)
700 {
701 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
702 struct dw_dma_slave *dws = chan->private;
703 struct dw_desc *prev;
704 struct dw_desc *first;
705 u32 ctllo;
706 dma_addr_t reg;
707 unsigned int reg_width;
708 unsigned int mem_width;
709 unsigned int i;
710 struct scatterlist *sg;
711 size_t total_len = 0;
712
713 dev_vdbg(chan2dev(chan), "prep_dma_slave\n");
714
715 if (unlikely(!dws || !sg_len))
716 return NULL;
717
718 reg_width = dws->reg_width;
719 prev = first = NULL;
720
721 switch (direction) {
722 case DMA_TO_DEVICE:
723 ctllo = (DWC_DEFAULT_CTLLO(chan->private)
724 | DWC_CTLL_DST_WIDTH(reg_width)
725 | DWC_CTLL_DST_FIX
726 | DWC_CTLL_SRC_INC
727 | DWC_CTLL_FC(dws->fc));
728 reg = dws->tx_reg;
729 for_each_sg(sgl, sg, sg_len, i) {
730 struct dw_desc *desc;
731 u32 len, dlen, mem;
732
733 mem = sg_phys(sg);
734 len = sg_dma_len(sg);
735 mem_width = 2;
736 if (unlikely(mem & 3 || len & 3))
737 mem_width = 0;
738
739 slave_sg_todev_fill_desc:
740 desc = dwc_desc_get(dwc);
741 if (!desc) {
742 dev_err(chan2dev(chan),
743 "not enough descriptors available\n");
744 goto err_desc_get;
745 }
746
747 desc->lli.sar = mem;
748 desc->lli.dar = reg;
749 desc->lli.ctllo = ctllo | DWC_CTLL_SRC_WIDTH(mem_width);
750 if ((len >> mem_width) > DWC_MAX_COUNT) {
751 dlen = DWC_MAX_COUNT << mem_width;
752 mem += dlen;
753 len -= dlen;
754 } else {
755 dlen = len;
756 len = 0;
757 }
758
759 desc->lli.ctlhi = dlen >> mem_width;
760
761 if (!first) {
762 first = desc;
763 } else {
764 prev->lli.llp = desc->txd.phys;
765 dma_sync_single_for_device(chan2parent(chan),
766 prev->txd.phys,
767 sizeof(prev->lli),
768 DMA_TO_DEVICE);
769 list_add_tail(&desc->desc_node,
770 &first->tx_list);
771 }
772 prev = desc;
773 total_len += dlen;
774
775 if (len)
776 goto slave_sg_todev_fill_desc;
777 }
778 break;
779 case DMA_FROM_DEVICE:
780 ctllo = (DWC_DEFAULT_CTLLO(chan->private)
781 | DWC_CTLL_SRC_WIDTH(reg_width)
782 | DWC_CTLL_DST_INC
783 | DWC_CTLL_SRC_FIX
784 | DWC_CTLL_FC(dws->fc));
785
786 reg = dws->rx_reg;
787 for_each_sg(sgl, sg, sg_len, i) {
788 struct dw_desc *desc;
789 u32 len, dlen, mem;
790
791 mem = sg_phys(sg);
792 len = sg_dma_len(sg);
793 mem_width = 2;
794 if (unlikely(mem & 3 || len & 3))
795 mem_width = 0;
796
797 slave_sg_fromdev_fill_desc:
798 desc = dwc_desc_get(dwc);
799 if (!desc) {
800 dev_err(chan2dev(chan),
801 "not enough descriptors available\n");
802 goto err_desc_get;
803 }
804
805 desc->lli.sar = reg;
806 desc->lli.dar = mem;
807 desc->lli.ctllo = ctllo | DWC_CTLL_DST_WIDTH(mem_width);
808 if ((len >> reg_width) > DWC_MAX_COUNT) {
809 dlen = DWC_MAX_COUNT << reg_width;
810 mem += dlen;
811 len -= dlen;
812 } else {
813 dlen = len;
814 len = 0;
815 }
816 desc->lli.ctlhi = dlen >> reg_width;
817
818 if (!first) {
819 first = desc;
820 } else {
821 prev->lli.llp = desc->txd.phys;
822 dma_sync_single_for_device(chan2parent(chan),
823 prev->txd.phys,
824 sizeof(prev->lli),
825 DMA_TO_DEVICE);
826 list_add_tail(&desc->desc_node,
827 &first->tx_list);
828 }
829 prev = desc;
830 total_len += dlen;
831
832 if (len)
833 goto slave_sg_fromdev_fill_desc;
834 }
835 break;
836 default:
837 return NULL;
838 }
839
840 if (flags & DMA_PREP_INTERRUPT)
841 /* Trigger interrupt after last block */
842 prev->lli.ctllo |= DWC_CTLL_INT_EN;
843
844 prev->lli.llp = 0;
845 dma_sync_single_for_device(chan2parent(chan),
846 prev->txd.phys, sizeof(prev->lli),
847 DMA_TO_DEVICE);
848
849 first->len = total_len;
850
851 return &first->txd;
852
853 err_desc_get:
854 dwc_desc_put(dwc, first);
855 return NULL;
856 }
857
858 static int dwc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
859 unsigned long arg)
860 {
861 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
862 struct dw_dma *dw = to_dw_dma(chan->device);
863 struct dw_desc *desc, *_desc;
864 unsigned long flags;
865 LIST_HEAD(list);
866
867 /* Only supports DMA_TERMINATE_ALL */
868 if (cmd != DMA_TERMINATE_ALL)
869 return -ENXIO;
870
871 /*
872 * This is only called when something went wrong elsewhere, so
873 * we don't really care about the data. Just disable the
874 * channel. We still have to poll the channel enable bit due
875 * to AHB/HSB limitations.
876 */
877 spin_lock_irqsave(&dwc->lock, flags);
878
879 channel_clear_bit(dw, CH_EN, dwc->mask);
880
881 while (dma_readl(dw, CH_EN) & dwc->mask)
882 cpu_relax();
883
884 /* active_list entries will end up before queued entries */
885 list_splice_init(&dwc->queue, &list);
886 list_splice_init(&dwc->active_list, &list);
887
888 spin_unlock_irqrestore(&dwc->lock, flags);
889
890 /* Flush all pending and queued descriptors */
891 list_for_each_entry_safe(desc, _desc, &list, desc_node)
892 dwc_descriptor_complete(dwc, desc, false);
893
894 return 0;
895 }
896
897 static enum dma_status
898 dwc_tx_status(struct dma_chan *chan,
899 dma_cookie_t cookie,
900 struct dma_tx_state *txstate)
901 {
902 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
903 dma_cookie_t last_used;
904 dma_cookie_t last_complete;
905 int ret;
906
907 last_complete = dwc->completed;
908 last_used = chan->cookie;
909
910 ret = dma_async_is_complete(cookie, last_complete, last_used);
911 if (ret != DMA_SUCCESS) {
912 dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
913
914 last_complete = dwc->completed;
915 last_used = chan->cookie;
916
917 ret = dma_async_is_complete(cookie, last_complete, last_used);
918 }
919
920 if (ret != DMA_SUCCESS)
921 dma_set_tx_state(txstate, last_complete, last_used,
922 dwc_first_active(dwc)->len);
923 else
924 dma_set_tx_state(txstate, last_complete, last_used, 0);
925
926 return ret;
927 }
928
929 static void dwc_issue_pending(struct dma_chan *chan)
930 {
931 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
932
933 if (!list_empty(&dwc->queue))
934 dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
935 }
936
937 static int dwc_alloc_chan_resources(struct dma_chan *chan)
938 {
939 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
940 struct dw_dma *dw = to_dw_dma(chan->device);
941 struct dw_desc *desc;
942 struct dw_dma_slave *dws;
943 int i;
944 u32 cfghi;
945 u32 cfglo;
946 unsigned long flags;
947
948 dev_vdbg(chan2dev(chan), "alloc_chan_resources\n");
949
950 /* ASSERT: channel is idle */
951 if (dma_readl(dw, CH_EN) & dwc->mask) {
952 dev_dbg(chan2dev(chan), "DMA channel not idle?\n");
953 return -EIO;
954 }
955
956 dwc->completed = chan->cookie = 1;
957
958 cfghi = DWC_CFGH_FIFO_MODE;
959 cfglo = 0;
960
961 dws = chan->private;
962 if (dws) {
963 /*
964 * We need controller-specific data to set up slave
965 * transfers.
966 */
967 BUG_ON(!dws->dma_dev || dws->dma_dev != dw->dma.dev);
968
969 cfghi = dws->cfg_hi;
970 cfglo = dws->cfg_lo & ~DWC_CFGL_CH_PRIOR_MASK;
971 }
972
973 cfglo |= DWC_CFGL_CH_PRIOR(dwc->priority);
974
975 channel_writel(dwc, CFG_LO, cfglo);
976 channel_writel(dwc, CFG_HI, cfghi);
977
978 /*
979 * NOTE: some controllers may have additional features that we
980 * need to initialize here, like "scatter-gather" (which
981 * doesn't mean what you think it means), and status writeback.
982 */
983
984 spin_lock_irqsave(&dwc->lock, flags);
985 i = dwc->descs_allocated;
986 while (dwc->descs_allocated < NR_DESCS_PER_CHANNEL) {
987 spin_unlock_irqrestore(&dwc->lock, flags);
988
989 desc = kzalloc(sizeof(struct dw_desc), GFP_KERNEL);
990 if (!desc) {
991 dev_info(chan2dev(chan),
992 "only allocated %d descriptors\n", i);
993 spin_lock_irqsave(&dwc->lock, flags);
994 break;
995 }
996
997 INIT_LIST_HEAD(&desc->tx_list);
998 dma_async_tx_descriptor_init(&desc->txd, chan);
999 desc->txd.tx_submit = dwc_tx_submit;
1000 desc->txd.flags = DMA_CTRL_ACK;
1001 desc->txd.phys = dma_map_single(chan2parent(chan), &desc->lli,
1002 sizeof(desc->lli), DMA_TO_DEVICE);
1003 dwc_desc_put(dwc, desc);
1004
1005 spin_lock_irqsave(&dwc->lock, flags);
1006 i = ++dwc->descs_allocated;
1007 }
1008
1009 /* Enable interrupts */
1010 channel_set_bit(dw, MASK.XFER, dwc->mask);
1011 channel_set_bit(dw, MASK.BLOCK, dwc->mask);
1012 channel_set_bit(dw, MASK.ERROR, dwc->mask);
1013
1014 spin_unlock_irqrestore(&dwc->lock, flags);
1015
1016 dev_dbg(chan2dev(chan),
1017 "alloc_chan_resources allocated %d descriptors\n", i);
1018
1019 return i;
1020 }
1021
1022 static void dwc_free_chan_resources(struct dma_chan *chan)
1023 {
1024 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1025 struct dw_dma *dw = to_dw_dma(chan->device);
1026 struct dw_desc *desc, *_desc;
1027 unsigned long flags;
1028 LIST_HEAD(list);
1029
1030 dev_dbg(chan2dev(chan), "free_chan_resources (descs allocated=%u)\n",
1031 dwc->descs_allocated);
1032
1033 /* ASSERT: channel is idle */
1034 BUG_ON(!list_empty(&dwc->active_list));
1035 BUG_ON(!list_empty(&dwc->queue));
1036 BUG_ON(dma_readl(to_dw_dma(chan->device), CH_EN) & dwc->mask);
1037
1038 spin_lock_irqsave(&dwc->lock, flags);
1039 list_splice_init(&dwc->free_list, &list);
1040 dwc->descs_allocated = 0;
1041
1042 /* Disable interrupts */
1043 channel_clear_bit(dw, MASK.XFER, dwc->mask);
1044 channel_clear_bit(dw, MASK.BLOCK, dwc->mask);
1045 channel_clear_bit(dw, MASK.ERROR, dwc->mask);
1046
1047 spin_unlock_irqrestore(&dwc->lock, flags);
1048
1049 list_for_each_entry_safe(desc, _desc, &list, desc_node) {
1050 dev_vdbg(chan2dev(chan), " freeing descriptor %p\n", desc);
1051 dma_unmap_single(chan2parent(chan), desc->txd.phys,
1052 sizeof(desc->lli), DMA_TO_DEVICE);
1053 kfree(desc);
1054 }
1055
1056 dev_vdbg(chan2dev(chan), "free_chan_resources done\n");
1057 }
1058
1059 /* --------------------- Cyclic DMA API extensions -------------------- */
1060
1061 /**
1062 * dw_dma_cyclic_start - start the cyclic DMA transfer
1063 * @chan: the DMA channel to start
1064 *
1065 * Must be called with soft interrupts disabled. Returns zero on success or
1066 * -errno on failure.
1067 */
1068 int dw_dma_cyclic_start(struct dma_chan *chan)
1069 {
1070 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1071 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
1072 unsigned long flags;
1073
1074 if (!test_bit(DW_DMA_IS_CYCLIC, &dwc->flags)) {
1075 dev_err(chan2dev(&dwc->chan), "missing prep for cyclic DMA\n");
1076 return -ENODEV;
1077 }
1078
1079 spin_lock_irqsave(&dwc->lock, flags);
1080
1081 /* assert channel is idle */
1082 if (dma_readl(dw, CH_EN) & dwc->mask) {
1083 dev_err(chan2dev(&dwc->chan),
1084 "BUG: Attempted to start non-idle channel\n");
1085 dev_err(chan2dev(&dwc->chan),
1086 " SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
1087 channel_readl(dwc, SAR),
1088 channel_readl(dwc, DAR),
1089 channel_readl(dwc, LLP),
1090 channel_readl(dwc, CTL_HI),
1091 channel_readl(dwc, CTL_LO));
1092 spin_unlock_irqrestore(&dwc->lock, flags);
1093 return -EBUSY;
1094 }
1095
1096 dma_writel(dw, CLEAR.BLOCK, dwc->mask);
1097 dma_writel(dw, CLEAR.ERROR, dwc->mask);
1098 dma_writel(dw, CLEAR.XFER, dwc->mask);
1099
1100 /* setup DMAC channel registers */
1101 channel_writel(dwc, LLP, dwc->cdesc->desc[0]->txd.phys);
1102 channel_writel(dwc, CTL_LO, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
1103 channel_writel(dwc, CTL_HI, 0);
1104
1105 channel_set_bit(dw, CH_EN, dwc->mask);
1106
1107 spin_unlock_irqrestore(&dwc->lock, flags);
1108
1109 return 0;
1110 }
1111 EXPORT_SYMBOL(dw_dma_cyclic_start);
1112
1113 /**
1114 * dw_dma_cyclic_stop - stop the cyclic DMA transfer
1115 * @chan: the DMA channel to stop
1116 *
1117 * Must be called with soft interrupts disabled.
1118 */
1119 void dw_dma_cyclic_stop(struct dma_chan *chan)
1120 {
1121 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1122 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
1123 unsigned long flags;
1124
1125 spin_lock_irqsave(&dwc->lock, flags);
1126
1127 channel_clear_bit(dw, CH_EN, dwc->mask);
1128 while (dma_readl(dw, CH_EN) & dwc->mask)
1129 cpu_relax();
1130
1131 spin_unlock_irqrestore(&dwc->lock, flags);
1132 }
1133 EXPORT_SYMBOL(dw_dma_cyclic_stop);
1134
1135 /**
1136 * dw_dma_cyclic_prep - prepare the cyclic DMA transfer
1137 * @chan: the DMA channel to prepare
1138 * @buf_addr: physical DMA address where the buffer starts
1139 * @buf_len: total number of bytes for the entire buffer
1140 * @period_len: number of bytes for each period
1141 * @direction: transfer direction, to or from device
1142 *
1143 * Must be called before trying to start the transfer. Returns a valid struct
1144 * dw_cyclic_desc if successful or an ERR_PTR(-errno) if not successful.
1145 */
1146 struct dw_cyclic_desc *dw_dma_cyclic_prep(struct dma_chan *chan,
1147 dma_addr_t buf_addr, size_t buf_len, size_t period_len,
1148 enum dma_data_direction direction)
1149 {
1150 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1151 struct dw_cyclic_desc *cdesc;
1152 struct dw_cyclic_desc *retval = NULL;
1153 struct dw_desc *desc;
1154 struct dw_desc *last = NULL;
1155 struct dw_dma_slave *dws = chan->private;
1156 unsigned long was_cyclic;
1157 unsigned int reg_width;
1158 unsigned int periods;
1159 unsigned int i;
1160 unsigned long flags;
1161
1162 spin_lock_irqsave(&dwc->lock, flags);
1163 if (!list_empty(&dwc->queue) || !list_empty(&dwc->active_list)) {
1164 spin_unlock_irqrestore(&dwc->lock, flags);
1165 dev_dbg(chan2dev(&dwc->chan),
1166 "queue and/or active list are not empty\n");
1167 return ERR_PTR(-EBUSY);
1168 }
1169
1170 was_cyclic = test_and_set_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1171 spin_unlock_irqrestore(&dwc->lock, flags);
1172 if (was_cyclic) {
1173 dev_dbg(chan2dev(&dwc->chan),
1174 "channel already prepared for cyclic DMA\n");
1175 return ERR_PTR(-EBUSY);
1176 }
1177
1178 retval = ERR_PTR(-EINVAL);
1179 reg_width = dws->reg_width;
1180 periods = buf_len / period_len;
1181
1182 /* Check for too big/unaligned periods and unaligned DMA buffer. */
1183 if (period_len > (DWC_MAX_COUNT << reg_width))
1184 goto out_err;
1185 if (unlikely(period_len & ((1 << reg_width) - 1)))
1186 goto out_err;
1187 if (unlikely(buf_addr & ((1 << reg_width) - 1)))
1188 goto out_err;
1189 if (unlikely(!(direction & (DMA_TO_DEVICE | DMA_FROM_DEVICE))))
1190 goto out_err;
1191
1192 retval = ERR_PTR(-ENOMEM);
1193
1194 if (periods > NR_DESCS_PER_CHANNEL)
1195 goto out_err;
1196
1197 cdesc = kzalloc(sizeof(struct dw_cyclic_desc), GFP_KERNEL);
1198 if (!cdesc)
1199 goto out_err;
1200
1201 cdesc->desc = kzalloc(sizeof(struct dw_desc *) * periods, GFP_KERNEL);
1202 if (!cdesc->desc)
1203 goto out_err_alloc;
1204
1205 for (i = 0; i < periods; i++) {
1206 desc = dwc_desc_get(dwc);
1207 if (!desc)
1208 goto out_err_desc_get;
1209
1210 switch (direction) {
1211 case DMA_TO_DEVICE:
1212 desc->lli.dar = dws->tx_reg;
1213 desc->lli.sar = buf_addr + (period_len * i);
1214 desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan->private)
1215 | DWC_CTLL_DST_WIDTH(reg_width)
1216 | DWC_CTLL_SRC_WIDTH(reg_width)
1217 | DWC_CTLL_DST_FIX
1218 | DWC_CTLL_SRC_INC
1219 | DWC_CTLL_FC(dws->fc)
1220 | DWC_CTLL_INT_EN);
1221 break;
1222 case DMA_FROM_DEVICE:
1223 desc->lli.dar = buf_addr + (period_len * i);
1224 desc->lli.sar = dws->rx_reg;
1225 desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan->private)
1226 | DWC_CTLL_SRC_WIDTH(reg_width)
1227 | DWC_CTLL_DST_WIDTH(reg_width)
1228 | DWC_CTLL_DST_INC
1229 | DWC_CTLL_SRC_FIX
1230 | DWC_CTLL_FC(dws->fc)
1231 | DWC_CTLL_INT_EN);
1232 break;
1233 default:
1234 break;
1235 }
1236
1237 desc->lli.ctlhi = (period_len >> reg_width);
1238 cdesc->desc[i] = desc;
1239
1240 if (last) {
1241 last->lli.llp = desc->txd.phys;
1242 dma_sync_single_for_device(chan2parent(chan),
1243 last->txd.phys, sizeof(last->lli),
1244 DMA_TO_DEVICE);
1245 }
1246
1247 last = desc;
1248 }
1249
1250 /* lets make a cyclic list */
1251 last->lli.llp = cdesc->desc[0]->txd.phys;
1252 dma_sync_single_for_device(chan2parent(chan), last->txd.phys,
1253 sizeof(last->lli), DMA_TO_DEVICE);
1254
1255 dev_dbg(chan2dev(&dwc->chan), "cyclic prepared buf 0x%08x len %zu "
1256 "period %zu periods %d\n", buf_addr, buf_len,
1257 period_len, periods);
1258
1259 cdesc->periods = periods;
1260 dwc->cdesc = cdesc;
1261
1262 return cdesc;
1263
1264 out_err_desc_get:
1265 while (i--)
1266 dwc_desc_put(dwc, cdesc->desc[i]);
1267 out_err_alloc:
1268 kfree(cdesc);
1269 out_err:
1270 clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1271 return (struct dw_cyclic_desc *)retval;
1272 }
1273 EXPORT_SYMBOL(dw_dma_cyclic_prep);
1274
1275 /**
1276 * dw_dma_cyclic_free - free a prepared cyclic DMA transfer
1277 * @chan: the DMA channel to free
1278 */
1279 void dw_dma_cyclic_free(struct dma_chan *chan)
1280 {
1281 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1282 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
1283 struct dw_cyclic_desc *cdesc = dwc->cdesc;
1284 int i;
1285 unsigned long flags;
1286
1287 dev_dbg(chan2dev(&dwc->chan), "cyclic free\n");
1288
1289 if (!cdesc)
1290 return;
1291
1292 spin_lock_irqsave(&dwc->lock, flags);
1293
1294 channel_clear_bit(dw, CH_EN, dwc->mask);
1295 while (dma_readl(dw, CH_EN) & dwc->mask)
1296 cpu_relax();
1297
1298 dma_writel(dw, CLEAR.BLOCK, dwc->mask);
1299 dma_writel(dw, CLEAR.ERROR, dwc->mask);
1300 dma_writel(dw, CLEAR.XFER, dwc->mask);
1301
1302 spin_unlock_irqrestore(&dwc->lock, flags);
1303
1304 for (i = 0; i < cdesc->periods; i++)
1305 dwc_desc_put(dwc, cdesc->desc[i]);
1306
1307 kfree(cdesc->desc);
1308 kfree(cdesc);
1309
1310 clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1311 }
1312 EXPORT_SYMBOL(dw_dma_cyclic_free);
1313
1314 /*----------------------------------------------------------------------*/
1315
1316 static void dw_dma_off(struct dw_dma *dw)
1317 {
1318 dma_writel(dw, CFG, 0);
1319
1320 channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
1321 channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
1322 channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);
1323 channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);
1324 channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
1325
1326 while (dma_readl(dw, CFG) & DW_CFG_DMA_EN)
1327 cpu_relax();
1328 }
1329
1330 static int __init dw_probe(struct platform_device *pdev)
1331 {
1332 struct dw_dma_platform_data *pdata;
1333 struct resource *io;
1334 struct dw_dma *dw;
1335 size_t size;
1336 int irq;
1337 int err;
1338 int i;
1339
1340 pdata = pdev->dev.platform_data;
1341 if (!pdata || pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS)
1342 return -EINVAL;
1343
1344 io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1345 if (!io)
1346 return -EINVAL;
1347
1348 irq = platform_get_irq(pdev, 0);
1349 if (irq < 0)
1350 return irq;
1351
1352 size = sizeof(struct dw_dma);
1353 size += pdata->nr_channels * sizeof(struct dw_dma_chan);
1354 dw = kzalloc(size, GFP_KERNEL);
1355 if (!dw)
1356 return -ENOMEM;
1357
1358 if (!request_mem_region(io->start, DW_REGLEN, pdev->dev.driver->name)) {
1359 err = -EBUSY;
1360 goto err_kfree;
1361 }
1362
1363 dw->regs = ioremap(io->start, DW_REGLEN);
1364 if (!dw->regs) {
1365 err = -ENOMEM;
1366 goto err_release_r;
1367 }
1368
1369 dw->clk = clk_get(&pdev->dev, "hclk");
1370 if (IS_ERR(dw->clk)) {
1371 err = PTR_ERR(dw->clk);
1372 goto err_clk;
1373 }
1374 clk_enable(dw->clk);
1375
1376 /* force dma off, just in case */
1377 dw_dma_off(dw);
1378
1379 err = request_irq(irq, dw_dma_interrupt, 0, "dw_dmac", dw);
1380 if (err)
1381 goto err_irq;
1382
1383 platform_set_drvdata(pdev, dw);
1384
1385 tasklet_init(&dw->tasklet, dw_dma_tasklet, (unsigned long)dw);
1386
1387 dw->all_chan_mask = (1 << pdata->nr_channels) - 1;
1388
1389 INIT_LIST_HEAD(&dw->dma.channels);
1390 for (i = 0; i < pdata->nr_channels; i++, dw->dma.chancnt++) {
1391 struct dw_dma_chan *dwc = &dw->chan[i];
1392
1393 dwc->chan.device = &dw->dma;
1394 dwc->chan.cookie = dwc->completed = 1;
1395 dwc->chan.chan_id = i;
1396 if (pdata->chan_allocation_order == CHAN_ALLOCATION_ASCENDING)
1397 list_add_tail(&dwc->chan.device_node,
1398 &dw->dma.channels);
1399 else
1400 list_add(&dwc->chan.device_node, &dw->dma.channels);
1401
1402 /* 7 is highest priority & 0 is lowest. */
1403 if (pdata->chan_priority == CHAN_PRIORITY_ASCENDING)
1404 dwc->priority = 7 - i;
1405 else
1406 dwc->priority = i;
1407
1408 dwc->ch_regs = &__dw_regs(dw)->CHAN[i];
1409 spin_lock_init(&dwc->lock);
1410 dwc->mask = 1 << i;
1411
1412 INIT_LIST_HEAD(&dwc->active_list);
1413 INIT_LIST_HEAD(&dwc->queue);
1414 INIT_LIST_HEAD(&dwc->free_list);
1415
1416 channel_clear_bit(dw, CH_EN, dwc->mask);
1417 }
1418
1419 /* Clear/disable all interrupts on all channels. */
1420 dma_writel(dw, CLEAR.XFER, dw->all_chan_mask);
1421 dma_writel(dw, CLEAR.BLOCK, dw->all_chan_mask);
1422 dma_writel(dw, CLEAR.SRC_TRAN, dw->all_chan_mask);
1423 dma_writel(dw, CLEAR.DST_TRAN, dw->all_chan_mask);
1424 dma_writel(dw, CLEAR.ERROR, dw->all_chan_mask);
1425
1426 channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
1427 channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
1428 channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);
1429 channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);
1430 channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
1431
1432 dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask);
1433 dma_cap_set(DMA_SLAVE, dw->dma.cap_mask);
1434 if (pdata->is_private)
1435 dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask);
1436 dw->dma.dev = &pdev->dev;
1437 dw->dma.device_alloc_chan_resources = dwc_alloc_chan_resources;
1438 dw->dma.device_free_chan_resources = dwc_free_chan_resources;
1439
1440 dw->dma.device_prep_dma_memcpy = dwc_prep_dma_memcpy;
1441
1442 dw->dma.device_prep_slave_sg = dwc_prep_slave_sg;
1443 dw->dma.device_control = dwc_control;
1444
1445 dw->dma.device_tx_status = dwc_tx_status;
1446 dw->dma.device_issue_pending = dwc_issue_pending;
1447
1448 dma_writel(dw, CFG, DW_CFG_DMA_EN);
1449
1450 printk(KERN_INFO "%s: DesignWare DMA Controller, %d channels\n",
1451 dev_name(&pdev->dev), dw->dma.chancnt);
1452
1453 dma_async_device_register(&dw->dma);
1454
1455 return 0;
1456
1457 err_irq:
1458 clk_disable(dw->clk);
1459 clk_put(dw->clk);
1460 err_clk:
1461 iounmap(dw->regs);
1462 dw->regs = NULL;
1463 err_release_r:
1464 release_resource(io);
1465 err_kfree:
1466 kfree(dw);
1467 return err;
1468 }
1469
1470 static int __exit dw_remove(struct platform_device *pdev)
1471 {
1472 struct dw_dma *dw = platform_get_drvdata(pdev);
1473 struct dw_dma_chan *dwc, *_dwc;
1474 struct resource *io;
1475
1476 dw_dma_off(dw);
1477 dma_async_device_unregister(&dw->dma);
1478
1479 free_irq(platform_get_irq(pdev, 0), dw);
1480 tasklet_kill(&dw->tasklet);
1481
1482 list_for_each_entry_safe(dwc, _dwc, &dw->dma.channels,
1483 chan.device_node) {
1484 list_del(&dwc->chan.device_node);
1485 channel_clear_bit(dw, CH_EN, dwc->mask);
1486 }
1487
1488 clk_disable(dw->clk);
1489 clk_put(dw->clk);
1490
1491 iounmap(dw->regs);
1492 dw->regs = NULL;
1493
1494 io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1495 release_mem_region(io->start, DW_REGLEN);
1496
1497 kfree(dw);
1498
1499 return 0;
1500 }
1501
1502 static void dw_shutdown(struct platform_device *pdev)
1503 {
1504 struct dw_dma *dw = platform_get_drvdata(pdev);
1505
1506 dw_dma_off(platform_get_drvdata(pdev));
1507 clk_disable(dw->clk);
1508 }
1509
1510 static int dw_suspend_noirq(struct device *dev)
1511 {
1512 struct platform_device *pdev = to_platform_device(dev);
1513 struct dw_dma *dw = platform_get_drvdata(pdev);
1514
1515 dw_dma_off(platform_get_drvdata(pdev));
1516 clk_disable(dw->clk);
1517 return 0;
1518 }
1519
1520 static int dw_resume_noirq(struct device *dev)
1521 {
1522 struct platform_device *pdev = to_platform_device(dev);
1523 struct dw_dma *dw = platform_get_drvdata(pdev);
1524
1525 clk_enable(dw->clk);
1526 dma_writel(dw, CFG, DW_CFG_DMA_EN);
1527 return 0;
1528 }
1529
1530 static const struct dev_pm_ops dw_dev_pm_ops = {
1531 .suspend_noirq = dw_suspend_noirq,
1532 .resume_noirq = dw_resume_noirq,
1533 };
1534
1535 static struct platform_driver dw_driver = {
1536 .remove = __exit_p(dw_remove),
1537 .shutdown = dw_shutdown,
1538 .driver = {
1539 .name = "dw_dmac",
1540 .pm = &dw_dev_pm_ops,
1541 },
1542 };
1543
1544 static int __init dw_init(void)
1545 {
1546 return platform_driver_probe(&dw_driver, dw_probe);
1547 }
1548 subsys_initcall(dw_init);
1549
1550 static void __exit dw_exit(void)
1551 {
1552 platform_driver_unregister(&dw_driver);
1553 }
1554 module_exit(dw_exit);
1555
1556 MODULE_LICENSE("GPL v2");
1557 MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller driver");
1558 MODULE_AUTHOR("Haavard Skinnemoen <haavard.skinnemoen@atmel.com>");
Linus' kernel tree