2 * Driver for the Atmel AHB DMA Controller (aka HDMA or DMAC on AT91 systems)
4 * Copyright (C) 2008 Atmel Corporation
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
12 * This supports the Atmel AHB DMA Controller,
14 * The driver has currently been tested with the Atmel AT91SAM9RL
15 * and AT91SAM9G45 series.
18 #include <linux/clk.h>
19 #include <linux/dmaengine.h>
20 #include <linux/dma-mapping.h>
21 #include <linux/dmapool.h>
22 #include <linux/interrupt.h>
23 #include <linux/module.h>
24 #include <linux/platform_device.h>
25 #include <linux/slab.h>
27 #include "at_hdmac_regs.h"
33 * at_hdmac : Name of the ATmel AHB DMA Controller
34 * at_dma_ / atdma : ATmel DMA controller entity related
35 * atc_ / atchan : ATmel DMA Channel entity related
38 #define ATC_DEFAULT_CFG (ATC_FIFOCFG_HALFFIFO)
39 #define ATC_DEFAULT_CTRLA (0)
40 #define ATC_DEFAULT_CTRLB (ATC_SIF(AT_DMA_MEM_IF) \
41 |ATC_DIF(AT_DMA_MEM_IF))
44 * Initial number of descriptors to allocate for each channel. This could
45 * be increased during dma usage.
47 static unsigned int init_nr_desc_per_channel
= 64;
48 module_param(init_nr_desc_per_channel
, uint
, 0644);
49 MODULE_PARM_DESC(init_nr_desc_per_channel
,
50 "initial descriptors per channel (default: 64)");
54 static dma_cookie_t
atc_tx_submit(struct dma_async_tx_descriptor
*tx
);
57 /*----------------------------------------------------------------------*/
59 static struct at_desc
*atc_first_active(struct at_dma_chan
*atchan
)
61 return list_first_entry(&atchan
->active_list
,
62 struct at_desc
, desc_node
);
65 static struct at_desc
*atc_first_queued(struct at_dma_chan
*atchan
)
67 return list_first_entry(&atchan
->queue
,
68 struct at_desc
, desc_node
);
72 * atc_alloc_descriptor - allocate and return an initialized descriptor
73 * @chan: the channel to allocate descriptors for
74 * @gfp_flags: GFP allocation flags
76 * Note: The ack-bit is positioned in the descriptor flag at creation time
77 * to make initial allocation more convenient. This bit will be cleared
78 * and control will be given to client at usage time (during
79 * preparation functions).
81 static struct at_desc
*atc_alloc_descriptor(struct dma_chan
*chan
,
84 struct at_desc
*desc
= NULL
;
85 struct at_dma
*atdma
= to_at_dma(chan
->device
);
88 desc
= dma_pool_alloc(atdma
->dma_desc_pool
, gfp_flags
, &phys
);
90 memset(desc
, 0, sizeof(struct at_desc
));
91 INIT_LIST_HEAD(&desc
->tx_list
);
92 dma_async_tx_descriptor_init(&desc
->txd
, chan
);
93 /* txd.flags will be overwritten in prep functions */
94 desc
->txd
.flags
= DMA_CTRL_ACK
;
95 desc
->txd
.tx_submit
= atc_tx_submit
;
96 desc
->txd
.phys
= phys
;
103 * atc_desc_get - get an unused descriptor from free_list
104 * @atchan: channel we want a new descriptor for
106 static struct at_desc
*atc_desc_get(struct at_dma_chan
*atchan
)
108 struct at_desc
*desc
, *_desc
;
109 struct at_desc
*ret
= NULL
;
113 spin_lock_bh(&atchan
->lock
);
114 list_for_each_entry_safe(desc
, _desc
, &atchan
->free_list
, desc_node
) {
116 if (async_tx_test_ack(&desc
->txd
)) {
117 list_del(&desc
->desc_node
);
121 dev_dbg(chan2dev(&atchan
->chan_common
),
122 "desc %p not ACKed\n", desc
);
124 spin_unlock_bh(&atchan
->lock
);
125 dev_vdbg(chan2dev(&atchan
->chan_common
),
126 "scanned %u descriptors on freelist\n", i
);
128 /* no more descriptor available in initial pool: create one more */
130 ret
= atc_alloc_descriptor(&atchan
->chan_common
, GFP_ATOMIC
);
132 spin_lock_bh(&atchan
->lock
);
133 atchan
->descs_allocated
++;
134 spin_unlock_bh(&atchan
->lock
);
136 dev_err(chan2dev(&atchan
->chan_common
),
137 "not enough descriptors available\n");
145 * atc_desc_put - move a descriptor, including any children, to the free list
146 * @atchan: channel we work on
147 * @desc: descriptor, at the head of a chain, to move to free list
149 static void atc_desc_put(struct at_dma_chan
*atchan
, struct at_desc
*desc
)
152 struct at_desc
*child
;
154 spin_lock_bh(&atchan
->lock
);
155 list_for_each_entry(child
, &desc
->tx_list
, desc_node
)
156 dev_vdbg(chan2dev(&atchan
->chan_common
),
157 "moving child desc %p to freelist\n",
159 list_splice_init(&desc
->tx_list
, &atchan
->free_list
);
160 dev_vdbg(chan2dev(&atchan
->chan_common
),
161 "moving desc %p to freelist\n", desc
);
162 list_add(&desc
->desc_node
, &atchan
->free_list
);
163 spin_unlock_bh(&atchan
->lock
);
168 * atc_desc_chain - build chain adding a descripor
169 * @first: address of first descripor of the chain
170 * @prev: address of previous descripor of the chain
171 * @desc: descriptor to queue
173 * Called from prep_* functions
175 static void atc_desc_chain(struct at_desc
**first
, struct at_desc
**prev
,
176 struct at_desc
*desc
)
181 /* inform the HW lli about chaining */
182 (*prev
)->lli
.dscr
= desc
->txd
.phys
;
183 /* insert the link descriptor to the LD ring */
184 list_add_tail(&desc
->desc_node
,
191 * atc_assign_cookie - compute and assign new cookie
192 * @atchan: channel we work on
193 * @desc: descriptor to asign cookie for
195 * Called with atchan->lock held and bh disabled
198 atc_assign_cookie(struct at_dma_chan
*atchan
, struct at_desc
*desc
)
200 dma_cookie_t cookie
= atchan
->chan_common
.cookie
;
205 atchan
->chan_common
.cookie
= cookie
;
206 desc
->txd
.cookie
= cookie
;
212 * atc_dostart - starts the DMA engine for real
213 * @atchan: the channel we want to start
214 * @first: first descriptor in the list we want to begin with
216 * Called with atchan->lock held and bh disabled
218 static void atc_dostart(struct at_dma_chan
*atchan
, struct at_desc
*first
)
220 struct at_dma
*atdma
= to_at_dma(atchan
->chan_common
.device
);
222 /* ASSERT: channel is idle */
223 if (atc_chan_is_enabled(atchan
)) {
224 dev_err(chan2dev(&atchan
->chan_common
),
225 "BUG: Attempted to start non-idle channel\n");
226 dev_err(chan2dev(&atchan
->chan_common
),
227 " channel: s0x%x d0x%x ctrl0x%x:0x%x l0x%x\n",
228 channel_readl(atchan
, SADDR
),
229 channel_readl(atchan
, DADDR
),
230 channel_readl(atchan
, CTRLA
),
231 channel_readl(atchan
, CTRLB
),
232 channel_readl(atchan
, DSCR
));
234 /* The tasklet will hopefully advance the queue... */
238 vdbg_dump_regs(atchan
);
240 /* clear any pending interrupt */
241 while (dma_readl(atdma
, EBCISR
))
244 channel_writel(atchan
, SADDR
, 0);
245 channel_writel(atchan
, DADDR
, 0);
246 channel_writel(atchan
, CTRLA
, 0);
247 channel_writel(atchan
, CTRLB
, 0);
248 channel_writel(atchan
, DSCR
, first
->txd
.phys
);
249 dma_writel(atdma
, CHER
, atchan
->mask
);
251 vdbg_dump_regs(atchan
);
255 * atc_chain_complete - finish work for one transaction chain
256 * @atchan: channel we work on
257 * @desc: descriptor at the head of the chain we want do complete
259 * Called with atchan->lock held and bh disabled */
261 atc_chain_complete(struct at_dma_chan
*atchan
, struct at_desc
*desc
)
263 struct dma_async_tx_descriptor
*txd
= &desc
->txd
;
265 dev_vdbg(chan2dev(&atchan
->chan_common
),
266 "descriptor %u complete\n", txd
->cookie
);
268 atchan
->completed_cookie
= txd
->cookie
;
270 /* move children to free_list */
271 list_splice_init(&desc
->tx_list
, &atchan
->free_list
);
272 /* move myself to free_list */
273 list_move(&desc
->desc_node
, &atchan
->free_list
);
275 /* unmap dma addresses (not on slave channels) */
276 if (!atchan
->chan_common
.private) {
277 struct device
*parent
= chan2parent(&atchan
->chan_common
);
278 if (!(txd
->flags
& DMA_COMPL_SKIP_DEST_UNMAP
)) {
279 if (txd
->flags
& DMA_COMPL_DEST_UNMAP_SINGLE
)
280 dma_unmap_single(parent
,
282 desc
->len
, DMA_FROM_DEVICE
);
284 dma_unmap_page(parent
,
286 desc
->len
, DMA_FROM_DEVICE
);
288 if (!(txd
->flags
& DMA_COMPL_SKIP_SRC_UNMAP
)) {
289 if (txd
->flags
& DMA_COMPL_SRC_UNMAP_SINGLE
)
290 dma_unmap_single(parent
,
292 desc
->len
, DMA_TO_DEVICE
);
294 dma_unmap_page(parent
,
296 desc
->len
, DMA_TO_DEVICE
);
300 /* for cyclic transfers,
301 * no need to replay callback function while stopping */
302 if (!test_bit(ATC_IS_CYCLIC
, &atchan
->status
)) {
303 dma_async_tx_callback callback
= txd
->callback
;
304 void *param
= txd
->callback_param
;
307 * The API requires that no submissions are done from a
308 * callback, so we don't need to drop the lock here
314 dma_run_dependencies(txd
);
318 * atc_complete_all - finish work for all transactions
319 * @atchan: channel to complete transactions for
321 * Eventually submit queued descriptors if any
323 * Assume channel is idle while calling this function
324 * Called with atchan->lock held and bh disabled
326 static void atc_complete_all(struct at_dma_chan
*atchan
)
328 struct at_desc
*desc
, *_desc
;
331 dev_vdbg(chan2dev(&atchan
->chan_common
), "complete all\n");
333 BUG_ON(atc_chan_is_enabled(atchan
));
336 * Submit queued descriptors ASAP, i.e. before we go through
337 * the completed ones.
339 if (!list_empty(&atchan
->queue
))
340 atc_dostart(atchan
, atc_first_queued(atchan
));
341 /* empty active_list now it is completed */
342 list_splice_init(&atchan
->active_list
, &list
);
343 /* empty queue list by moving descriptors (if any) to active_list */
344 list_splice_init(&atchan
->queue
, &atchan
->active_list
);
346 list_for_each_entry_safe(desc
, _desc
, &list
, desc_node
)
347 atc_chain_complete(atchan
, desc
);
351 * atc_cleanup_descriptors - cleanup up finished descriptors in active_list
352 * @atchan: channel to be cleaned up
354 * Called with atchan->lock held and bh disabled
356 static void atc_cleanup_descriptors(struct at_dma_chan
*atchan
)
358 struct at_desc
*desc
, *_desc
;
359 struct at_desc
*child
;
361 dev_vdbg(chan2dev(&atchan
->chan_common
), "cleanup descriptors\n");
363 list_for_each_entry_safe(desc
, _desc
, &atchan
->active_list
, desc_node
) {
364 if (!(desc
->lli
.ctrla
& ATC_DONE
))
365 /* This one is currently in progress */
368 list_for_each_entry(child
, &desc
->tx_list
, desc_node
)
369 if (!(child
->lli
.ctrla
& ATC_DONE
))
370 /* Currently in progress */
374 * No descriptors so far seem to be in progress, i.e.
375 * this chain must be done.
377 atc_chain_complete(atchan
, desc
);
382 * atc_advance_work - at the end of a transaction, move forward
383 * @atchan: channel where the transaction ended
385 * Called with atchan->lock held and bh disabled
387 static void atc_advance_work(struct at_dma_chan
*atchan
)
389 dev_vdbg(chan2dev(&atchan
->chan_common
), "advance_work\n");
391 if (list_empty(&atchan
->active_list
) ||
392 list_is_singular(&atchan
->active_list
)) {
393 atc_complete_all(atchan
);
395 atc_chain_complete(atchan
, atc_first_active(atchan
));
397 atc_dostart(atchan
, atc_first_active(atchan
));
403 * atc_handle_error - handle errors reported by DMA controller
404 * @atchan: channel where error occurs
406 * Called with atchan->lock held and bh disabled
408 static void atc_handle_error(struct at_dma_chan
*atchan
)
410 struct at_desc
*bad_desc
;
411 struct at_desc
*child
;
414 * The descriptor currently at the head of the active list is
415 * broked. Since we don't have any way to report errors, we'll
416 * just have to scream loudly and try to carry on.
418 bad_desc
= atc_first_active(atchan
);
419 list_del_init(&bad_desc
->desc_node
);
421 /* As we are stopped, take advantage to push queued descriptors
423 list_splice_init(&atchan
->queue
, atchan
->active_list
.prev
);
425 /* Try to restart the controller */
426 if (!list_empty(&atchan
->active_list
))
427 atc_dostart(atchan
, atc_first_active(atchan
));
430 * KERN_CRITICAL may seem harsh, but since this only happens
431 * when someone submits a bad physical address in a
432 * descriptor, we should consider ourselves lucky that the
433 * controller flagged an error instead of scribbling over
434 * random memory locations.
436 dev_crit(chan2dev(&atchan
->chan_common
),
437 "Bad descriptor submitted for DMA!\n");
438 dev_crit(chan2dev(&atchan
->chan_common
),
439 " cookie: %d\n", bad_desc
->txd
.cookie
);
440 atc_dump_lli(atchan
, &bad_desc
->lli
);
441 list_for_each_entry(child
, &bad_desc
->tx_list
, desc_node
)
442 atc_dump_lli(atchan
, &child
->lli
);
444 /* Pretend the descriptor completed successfully */
445 atc_chain_complete(atchan
, bad_desc
);
449 * atc_handle_cyclic - at the end of a period, run callback function
450 * @atchan: channel used for cyclic operations
452 * Called with atchan->lock held and bh disabled
454 static void atc_handle_cyclic(struct at_dma_chan
*atchan
)
456 struct at_desc
*first
= atc_first_active(atchan
);
457 struct dma_async_tx_descriptor
*txd
= &first
->txd
;
458 dma_async_tx_callback callback
= txd
->callback
;
459 void *param
= txd
->callback_param
;
461 dev_vdbg(chan2dev(&atchan
->chan_common
),
462 "new cyclic period llp 0x%08x\n",
463 channel_readl(atchan
, DSCR
));
469 /*-- IRQ & Tasklet ---------------------------------------------------*/
471 static void atc_tasklet(unsigned long data
)
473 struct at_dma_chan
*atchan
= (struct at_dma_chan
*)data
;
475 spin_lock(&atchan
->lock
);
476 if (test_and_clear_bit(ATC_IS_ERROR
, &atchan
->status
))
477 atc_handle_error(atchan
);
478 else if (test_bit(ATC_IS_CYCLIC
, &atchan
->status
))
479 atc_handle_cyclic(atchan
);
481 atc_advance_work(atchan
);
483 spin_unlock(&atchan
->lock
);
486 static irqreturn_t
at_dma_interrupt(int irq
, void *dev_id
)
488 struct at_dma
*atdma
= (struct at_dma
*)dev_id
;
489 struct at_dma_chan
*atchan
;
491 u32 status
, pending
, imr
;
495 imr
= dma_readl(atdma
, EBCIMR
);
496 status
= dma_readl(atdma
, EBCISR
);
497 pending
= status
& imr
;
502 dev_vdbg(atdma
->dma_common
.dev
,
503 "interrupt: status = 0x%08x, 0x%08x, 0x%08x\n",
504 status
, imr
, pending
);
506 for (i
= 0; i
< atdma
->dma_common
.chancnt
; i
++) {
507 atchan
= &atdma
->chan
[i
];
508 if (pending
& (AT_DMA_BTC(i
) | AT_DMA_ERR(i
))) {
509 if (pending
& AT_DMA_ERR(i
)) {
510 /* Disable channel on AHB error */
511 dma_writel(atdma
, CHDR
, atchan
->mask
);
512 /* Give information to tasklet */
513 set_bit(ATC_IS_ERROR
, &atchan
->status
);
515 tasklet_schedule(&atchan
->tasklet
);
526 /*-- DMA Engine API --------------------------------------------------*/
529 * atc_tx_submit - set the prepared descriptor(s) to be executed by the engine
530 * @desc: descriptor at the head of the transaction chain
532 * Queue chain if DMA engine is working already
534 * Cookie increment and adding to active_list or queue must be atomic
536 static dma_cookie_t
atc_tx_submit(struct dma_async_tx_descriptor
*tx
)
538 struct at_desc
*desc
= txd_to_at_desc(tx
);
539 struct at_dma_chan
*atchan
= to_at_dma_chan(tx
->chan
);
542 spin_lock_bh(&atchan
->lock
);
543 cookie
= atc_assign_cookie(atchan
, desc
);
545 if (list_empty(&atchan
->active_list
)) {
546 dev_vdbg(chan2dev(tx
->chan
), "tx_submit: started %u\n",
548 atc_dostart(atchan
, desc
);
549 list_add_tail(&desc
->desc_node
, &atchan
->active_list
);
551 dev_vdbg(chan2dev(tx
->chan
), "tx_submit: queued %u\n",
553 list_add_tail(&desc
->desc_node
, &atchan
->queue
);
556 spin_unlock_bh(&atchan
->lock
);
562 * atc_prep_dma_memcpy - prepare a memcpy operation
563 * @chan: the channel to prepare operation on
564 * @dest: operation virtual destination address
565 * @src: operation virtual source address
566 * @len: operation length
567 * @flags: tx descriptor status flags
569 static struct dma_async_tx_descriptor
*
570 atc_prep_dma_memcpy(struct dma_chan
*chan
, dma_addr_t dest
, dma_addr_t src
,
571 size_t len
, unsigned long flags
)
573 struct at_dma_chan
*atchan
= to_at_dma_chan(chan
);
574 struct at_desc
*desc
= NULL
;
575 struct at_desc
*first
= NULL
;
576 struct at_desc
*prev
= NULL
;
579 unsigned int src_width
;
580 unsigned int dst_width
;
584 dev_vdbg(chan2dev(chan
), "prep_dma_memcpy: d0x%x s0x%x l0x%zx f0x%lx\n",
585 dest
, src
, len
, flags
);
587 if (unlikely(!len
)) {
588 dev_dbg(chan2dev(chan
), "prep_dma_memcpy: length is zero!\n");
592 ctrla
= ATC_DEFAULT_CTRLA
;
593 ctrlb
= ATC_DEFAULT_CTRLB
| ATC_IEN
594 | ATC_SRC_ADDR_MODE_INCR
595 | ATC_DST_ADDR_MODE_INCR
599 * We can be a lot more clever here, but this should take care
600 * of the most common optimization.
602 if (!((src
| dest
| len
) & 3)) {
603 ctrla
|= ATC_SRC_WIDTH_WORD
| ATC_DST_WIDTH_WORD
;
604 src_width
= dst_width
= 2;
605 } else if (!((src
| dest
| len
) & 1)) {
606 ctrla
|= ATC_SRC_WIDTH_HALFWORD
| ATC_DST_WIDTH_HALFWORD
;
607 src_width
= dst_width
= 1;
609 ctrla
|= ATC_SRC_WIDTH_BYTE
| ATC_DST_WIDTH_BYTE
;
610 src_width
= dst_width
= 0;
613 for (offset
= 0; offset
< len
; offset
+= xfer_count
<< src_width
) {
614 xfer_count
= min_t(size_t, (len
- offset
) >> src_width
,
617 desc
= atc_desc_get(atchan
);
621 desc
->lli
.saddr
= src
+ offset
;
622 desc
->lli
.daddr
= dest
+ offset
;
623 desc
->lli
.ctrla
= ctrla
| xfer_count
;
624 desc
->lli
.ctrlb
= ctrlb
;
626 desc
->txd
.cookie
= 0;
631 /* inform the HW lli about chaining */
632 prev
->lli
.dscr
= desc
->txd
.phys
;
633 /* insert the link descriptor to the LD ring */
634 list_add_tail(&desc
->desc_node
,
640 /* First descriptor of the chain embedds additional information */
641 first
->txd
.cookie
= -EBUSY
;
644 /* set end-of-link to the last link descriptor of list*/
647 first
->txd
.flags
= flags
; /* client is in control of this ack */
652 atc_desc_put(atchan
, first
);
658 * atc_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
660 * @sgl: scatterlist to transfer to/from
661 * @sg_len: number of entries in @scatterlist
662 * @direction: DMA direction
663 * @flags: tx descriptor status flags
665 static struct dma_async_tx_descriptor
*
666 atc_prep_slave_sg(struct dma_chan
*chan
, struct scatterlist
*sgl
,
667 unsigned int sg_len
, enum dma_data_direction direction
,
670 struct at_dma_chan
*atchan
= to_at_dma_chan(chan
);
671 struct at_dma_slave
*atslave
= chan
->private;
672 struct at_desc
*first
= NULL
;
673 struct at_desc
*prev
= NULL
;
677 unsigned int reg_width
;
678 unsigned int mem_width
;
680 struct scatterlist
*sg
;
681 size_t total_len
= 0;
683 dev_vdbg(chan2dev(chan
), "prep_slave_sg (%d): %s f0x%lx\n",
685 direction
== DMA_TO_DEVICE
? "TO DEVICE" : "FROM DEVICE",
688 if (unlikely(!atslave
|| !sg_len
)) {
689 dev_dbg(chan2dev(chan
), "prep_dma_memcpy: length is zero!\n");
693 reg_width
= atslave
->reg_width
;
695 ctrla
= ATC_DEFAULT_CTRLA
| atslave
->ctrla
;
700 ctrla
|= ATC_DST_WIDTH(reg_width
);
701 ctrlb
|= ATC_DST_ADDR_MODE_FIXED
702 | ATC_SRC_ADDR_MODE_INCR
704 | ATC_SIF(AT_DMA_MEM_IF
) | ATC_DIF(AT_DMA_PER_IF
);
705 reg
= atslave
->tx_reg
;
706 for_each_sg(sgl
, sg
, sg_len
, i
) {
707 struct at_desc
*desc
;
711 desc
= atc_desc_get(atchan
);
715 mem
= sg_dma_address(sg
);
716 len
= sg_dma_len(sg
);
718 if (unlikely(mem
& 3 || len
& 3))
721 desc
->lli
.saddr
= mem
;
722 desc
->lli
.daddr
= reg
;
723 desc
->lli
.ctrla
= ctrla
724 | ATC_SRC_WIDTH(mem_width
)
726 desc
->lli
.ctrlb
= ctrlb
;
731 /* inform the HW lli about chaining */
732 prev
->lli
.dscr
= desc
->txd
.phys
;
733 /* insert the link descriptor to the LD ring */
734 list_add_tail(&desc
->desc_node
,
741 case DMA_FROM_DEVICE
:
742 ctrla
|= ATC_SRC_WIDTH(reg_width
);
743 ctrlb
|= ATC_DST_ADDR_MODE_INCR
744 | ATC_SRC_ADDR_MODE_FIXED
746 | ATC_SIF(AT_DMA_PER_IF
) | ATC_DIF(AT_DMA_MEM_IF
);
748 reg
= atslave
->rx_reg
;
749 for_each_sg(sgl
, sg
, sg_len
, i
) {
750 struct at_desc
*desc
;
754 desc
= atc_desc_get(atchan
);
758 mem
= sg_dma_address(sg
);
759 len
= sg_dma_len(sg
);
761 if (unlikely(mem
& 3 || len
& 3))
764 desc
->lli
.saddr
= reg
;
765 desc
->lli
.daddr
= mem
;
766 desc
->lli
.ctrla
= ctrla
767 | ATC_DST_WIDTH(mem_width
)
769 desc
->lli
.ctrlb
= ctrlb
;
774 /* inform the HW lli about chaining */
775 prev
->lli
.dscr
= desc
->txd
.phys
;
776 /* insert the link descriptor to the LD ring */
777 list_add_tail(&desc
->desc_node
,
788 /* set end-of-link to the last link descriptor of list*/
791 /* First descriptor of the chain embedds additional information */
792 first
->txd
.cookie
= -EBUSY
;
793 first
->len
= total_len
;
795 /* first link descriptor of list is responsible of flags */
796 first
->txd
.flags
= flags
; /* client is in control of this ack */
801 dev_err(chan2dev(chan
), "not enough descriptors available\n");
802 atc_desc_put(atchan
, first
);
807 * atc_dma_cyclic_check_values
808 * Check for too big/unaligned periods and unaligned DMA buffer
811 atc_dma_cyclic_check_values(unsigned int reg_width
, dma_addr_t buf_addr
,
812 size_t period_len
, enum dma_data_direction direction
)
814 if (period_len
> (ATC_BTSIZE_MAX
<< reg_width
))
816 if (unlikely(period_len
& ((1 << reg_width
) - 1)))
818 if (unlikely(buf_addr
& ((1 << reg_width
) - 1)))
820 if (unlikely(!(direction
& (DMA_TO_DEVICE
| DMA_FROM_DEVICE
))))
830 * atc_dma_cyclic_fill_desc - Fill one period decriptor
833 atc_dma_cyclic_fill_desc(struct at_dma_slave
*atslave
, struct at_desc
*desc
,
834 unsigned int period_index
, dma_addr_t buf_addr
,
835 size_t period_len
, enum dma_data_direction direction
)
838 unsigned int reg_width
= atslave
->reg_width
;
840 /* prepare common CRTLA value */
841 ctrla
= ATC_DEFAULT_CTRLA
| atslave
->ctrla
842 | ATC_DST_WIDTH(reg_width
)
843 | ATC_SRC_WIDTH(reg_width
)
844 | period_len
>> reg_width
;
848 desc
->lli
.saddr
= buf_addr
+ (period_len
* period_index
);
849 desc
->lli
.daddr
= atslave
->tx_reg
;
850 desc
->lli
.ctrla
= ctrla
;
851 desc
->lli
.ctrlb
= ATC_DST_ADDR_MODE_FIXED
852 | ATC_SRC_ADDR_MODE_INCR
854 | ATC_SIF(AT_DMA_MEM_IF
)
855 | ATC_DIF(AT_DMA_PER_IF
);
858 case DMA_FROM_DEVICE
:
859 desc
->lli
.saddr
= atslave
->rx_reg
;
860 desc
->lli
.daddr
= buf_addr
+ (period_len
* period_index
);
861 desc
->lli
.ctrla
= ctrla
;
862 desc
->lli
.ctrlb
= ATC_DST_ADDR_MODE_INCR
863 | ATC_SRC_ADDR_MODE_FIXED
865 | ATC_SIF(AT_DMA_PER_IF
)
866 | ATC_DIF(AT_DMA_MEM_IF
);
877 * atc_prep_dma_cyclic - prepare the cyclic DMA transfer
878 * @chan: the DMA channel to prepare
879 * @buf_addr: physical DMA address where the buffer starts
880 * @buf_len: total number of bytes for the entire buffer
881 * @period_len: number of bytes for each period
882 * @direction: transfer direction, to or from device
884 static struct dma_async_tx_descriptor
*
885 atc_prep_dma_cyclic(struct dma_chan
*chan
, dma_addr_t buf_addr
, size_t buf_len
,
886 size_t period_len
, enum dma_data_direction direction
)
888 struct at_dma_chan
*atchan
= to_at_dma_chan(chan
);
889 struct at_dma_slave
*atslave
= chan
->private;
890 struct at_desc
*first
= NULL
;
891 struct at_desc
*prev
= NULL
;
892 unsigned long was_cyclic
;
893 unsigned int periods
= buf_len
/ period_len
;
896 dev_vdbg(chan2dev(chan
), "prep_dma_cyclic: %s buf@0x%08x - %d (%d/%d)\n",
897 direction
== DMA_TO_DEVICE
? "TO DEVICE" : "FROM DEVICE",
899 periods
, buf_len
, period_len
);
901 if (unlikely(!atslave
|| !buf_len
|| !period_len
)) {
902 dev_dbg(chan2dev(chan
), "prep_dma_cyclic: length is zero!\n");
906 was_cyclic
= test_and_set_bit(ATC_IS_CYCLIC
, &atchan
->status
);
908 dev_dbg(chan2dev(chan
), "prep_dma_cyclic: channel in use!\n");
912 /* Check for too big/unaligned periods and unaligned DMA buffer */
913 if (atc_dma_cyclic_check_values(atslave
->reg_width
, buf_addr
,
914 period_len
, direction
))
917 /* build cyclic linked list */
918 for (i
= 0; i
< periods
; i
++) {
919 struct at_desc
*desc
;
921 desc
= atc_desc_get(atchan
);
925 if (atc_dma_cyclic_fill_desc(atslave
, desc
, i
, buf_addr
,
926 period_len
, direction
))
929 atc_desc_chain(&first
, &prev
, desc
);
932 /* lets make a cyclic list */
933 prev
->lli
.dscr
= first
->txd
.phys
;
935 /* First descriptor of the chain embedds additional information */
936 first
->txd
.cookie
= -EBUSY
;
937 first
->len
= buf_len
;
942 dev_err(chan2dev(chan
), "not enough descriptors available\n");
943 atc_desc_put(atchan
, first
);
945 clear_bit(ATC_IS_CYCLIC
, &atchan
->status
);
950 static int atc_control(struct dma_chan
*chan
, enum dma_ctrl_cmd cmd
,
953 struct at_dma_chan
*atchan
= to_at_dma_chan(chan
);
954 struct at_dma
*atdma
= to_at_dma(chan
->device
);
955 struct at_desc
*desc
, *_desc
;
958 /* Only supports DMA_TERMINATE_ALL */
959 if (cmd
!= DMA_TERMINATE_ALL
)
963 * This is only called when something went wrong elsewhere, so
964 * we don't really care about the data. Just disable the
965 * channel. We still have to poll the channel enable bit due
966 * to AHB/HSB limitations.
968 spin_lock_bh(&atchan
->lock
);
970 dma_writel(atdma
, CHDR
, atchan
->mask
);
972 /* confirm that this channel is disabled */
973 while (dma_readl(atdma
, CHSR
) & atchan
->mask
)
976 /* active_list entries will end up before queued entries */
977 list_splice_init(&atchan
->queue
, &list
);
978 list_splice_init(&atchan
->active_list
, &list
);
980 /* Flush all pending and queued descriptors */
981 list_for_each_entry_safe(desc
, _desc
, &list
, desc_node
)
982 atc_chain_complete(atchan
, desc
);
984 /* if channel dedicated to cyclic operations, free it */
985 clear_bit(ATC_IS_CYCLIC
, &atchan
->status
);
987 spin_unlock_bh(&atchan
->lock
);
993 * atc_tx_status - poll for transaction completion
995 * @cookie: transaction identifier to check status of
996 * @txstate: if not %NULL updated with transaction state
998 * If @txstate is passed in, upon return it reflect the driver
999 * internal state and can be used with dma_async_is_complete() to check
1000 * the status of multiple cookies without re-checking hardware state.
1002 static enum dma_status
1003 atc_tx_status(struct dma_chan
*chan
,
1004 dma_cookie_t cookie
,
1005 struct dma_tx_state
*txstate
)
1007 struct at_dma_chan
*atchan
= to_at_dma_chan(chan
);
1008 dma_cookie_t last_used
;
1009 dma_cookie_t last_complete
;
1010 enum dma_status ret
;
1012 spin_lock_bh(&atchan
->lock
);
1014 last_complete
= atchan
->completed_cookie
;
1015 last_used
= chan
->cookie
;
1017 ret
= dma_async_is_complete(cookie
, last_complete
, last_used
);
1018 if (ret
!= DMA_SUCCESS
) {
1019 atc_cleanup_descriptors(atchan
);
1021 last_complete
= atchan
->completed_cookie
;
1022 last_used
= chan
->cookie
;
1024 ret
= dma_async_is_complete(cookie
, last_complete
, last_used
);
1027 spin_unlock_bh(&atchan
->lock
);
1029 if (ret
!= DMA_SUCCESS
)
1030 dma_set_tx_state(txstate
, last_complete
, last_used
,
1031 atc_first_active(atchan
)->len
);
1033 dma_set_tx_state(txstate
, last_complete
, last_used
, 0);
1035 dev_vdbg(chan2dev(chan
), "tx_status: %d (d%d, u%d)\n",
1036 cookie
, last_complete
? last_complete
: 0,
1037 last_used
? last_used
: 0);
1043 * atc_issue_pending - try to finish work
1044 * @chan: target DMA channel
1046 static void atc_issue_pending(struct dma_chan
*chan
)
1048 struct at_dma_chan
*atchan
= to_at_dma_chan(chan
);
1050 dev_vdbg(chan2dev(chan
), "issue_pending\n");
1052 /* Not needed for cyclic transfers */
1053 if (test_bit(ATC_IS_CYCLIC
, &atchan
->status
))
1056 spin_lock_bh(&atchan
->lock
);
1057 if (!atc_chan_is_enabled(atchan
)) {
1058 atc_advance_work(atchan
);
1060 spin_unlock_bh(&atchan
->lock
);
1064 * atc_alloc_chan_resources - allocate resources for DMA channel
1065 * @chan: allocate descriptor resources for this channel
1066 * @client: current client requesting the channel be ready for requests
1068 * return - the number of allocated descriptors
1070 static int atc_alloc_chan_resources(struct dma_chan
*chan
)
1072 struct at_dma_chan
*atchan
= to_at_dma_chan(chan
);
1073 struct at_dma
*atdma
= to_at_dma(chan
->device
);
1074 struct at_desc
*desc
;
1075 struct at_dma_slave
*atslave
;
1078 LIST_HEAD(tmp_list
);
1080 dev_vdbg(chan2dev(chan
), "alloc_chan_resources\n");
1082 /* ASSERT: channel is idle */
1083 if (atc_chan_is_enabled(atchan
)) {
1084 dev_dbg(chan2dev(chan
), "DMA channel not idle ?\n");
1088 cfg
= ATC_DEFAULT_CFG
;
1090 atslave
= chan
->private;
1093 * We need controller-specific data to set up slave
1096 BUG_ON(!atslave
->dma_dev
|| atslave
->dma_dev
!= atdma
->dma_common
.dev
);
1098 /* if cfg configuration specified take it instad of default */
1103 /* have we already been set up?
1104 * reconfigure channel but no need to reallocate descriptors */
1105 if (!list_empty(&atchan
->free_list
))
1106 return atchan
->descs_allocated
;
1108 /* Allocate initial pool of descriptors */
1109 for (i
= 0; i
< init_nr_desc_per_channel
; i
++) {
1110 desc
= atc_alloc_descriptor(chan
, GFP_KERNEL
);
1112 dev_err(atdma
->dma_common
.dev
,
1113 "Only %d initial descriptors\n", i
);
1116 list_add_tail(&desc
->desc_node
, &tmp_list
);
1119 spin_lock_bh(&atchan
->lock
);
1120 atchan
->descs_allocated
= i
;
1121 list_splice(&tmp_list
, &atchan
->free_list
);
1122 atchan
->completed_cookie
= chan
->cookie
= 1;
1123 spin_unlock_bh(&atchan
->lock
);
1125 /* channel parameters */
1126 channel_writel(atchan
, CFG
, cfg
);
1128 dev_dbg(chan2dev(chan
),
1129 "alloc_chan_resources: allocated %d descriptors\n",
1130 atchan
->descs_allocated
);
1132 return atchan
->descs_allocated
;
1136 * atc_free_chan_resources - free all channel resources
1137 * @chan: DMA channel
1139 static void atc_free_chan_resources(struct dma_chan
*chan
)
1141 struct at_dma_chan
*atchan
= to_at_dma_chan(chan
);
1142 struct at_dma
*atdma
= to_at_dma(chan
->device
);
1143 struct at_desc
*desc
, *_desc
;
1146 dev_dbg(chan2dev(chan
), "free_chan_resources: (descs allocated=%u)\n",
1147 atchan
->descs_allocated
);
1149 /* ASSERT: channel is idle */
1150 BUG_ON(!list_empty(&atchan
->active_list
));
1151 BUG_ON(!list_empty(&atchan
->queue
));
1152 BUG_ON(atc_chan_is_enabled(atchan
));
1154 list_for_each_entry_safe(desc
, _desc
, &atchan
->free_list
, desc_node
) {
1155 dev_vdbg(chan2dev(chan
), " freeing descriptor %p\n", desc
);
1156 list_del(&desc
->desc_node
);
1157 /* free link descriptor */
1158 dma_pool_free(atdma
->dma_desc_pool
, desc
, desc
->txd
.phys
);
1160 list_splice_init(&atchan
->free_list
, &list
);
1161 atchan
->descs_allocated
= 0;
1164 dev_vdbg(chan2dev(chan
), "free_chan_resources: done\n");
1168 /*-- Module Management -----------------------------------------------*/
1171 * at_dma_off - disable DMA controller
1172 * @atdma: the Atmel HDAMC device
1174 static void at_dma_off(struct at_dma
*atdma
)
1176 dma_writel(atdma
, EN
, 0);
1178 /* disable all interrupts */
1179 dma_writel(atdma
, EBCIDR
, -1L);
1181 /* confirm that all channels are disabled */
1182 while (dma_readl(atdma
, CHSR
) & atdma
->all_chan_mask
)
1186 static int __init
at_dma_probe(struct platform_device
*pdev
)
1188 struct at_dma_platform_data
*pdata
;
1189 struct resource
*io
;
1190 struct at_dma
*atdma
;
1196 /* get DMA Controller parameters from platform */
1197 pdata
= pdev
->dev
.platform_data
;
1198 if (!pdata
|| pdata
->nr_channels
> AT_DMA_MAX_NR_CHANNELS
)
1201 io
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1205 irq
= platform_get_irq(pdev
, 0);
1209 size
= sizeof(struct at_dma
);
1210 size
+= pdata
->nr_channels
* sizeof(struct at_dma_chan
);
1211 atdma
= kzalloc(size
, GFP_KERNEL
);
1215 /* discover transaction capabilites from the platform data */
1216 atdma
->dma_common
.cap_mask
= pdata
->cap_mask
;
1217 atdma
->all_chan_mask
= (1 << pdata
->nr_channels
) - 1;
1219 size
= io
->end
- io
->start
+ 1;
1220 if (!request_mem_region(io
->start
, size
, pdev
->dev
.driver
->name
)) {
1225 atdma
->regs
= ioremap(io
->start
, size
);
1231 atdma
->clk
= clk_get(&pdev
->dev
, "dma_clk");
1232 if (IS_ERR(atdma
->clk
)) {
1233 err
= PTR_ERR(atdma
->clk
);
1236 clk_enable(atdma
->clk
);
1238 /* force dma off, just in case */
1241 err
= request_irq(irq
, at_dma_interrupt
, 0, "at_hdmac", atdma
);
1245 platform_set_drvdata(pdev
, atdma
);
1247 /* create a pool of consistent memory blocks for hardware descriptors */
1248 atdma
->dma_desc_pool
= dma_pool_create("at_hdmac_desc_pool",
1249 &pdev
->dev
, sizeof(struct at_desc
),
1250 4 /* word alignment */, 0);
1251 if (!atdma
->dma_desc_pool
) {
1252 dev_err(&pdev
->dev
, "No memory for descriptors dma pool\n");
1254 goto err_pool_create
;
1257 /* clear any pending interrupt */
1258 while (dma_readl(atdma
, EBCISR
))
1261 /* initialize channels related values */
1262 INIT_LIST_HEAD(&atdma
->dma_common
.channels
);
1263 for (i
= 0; i
< pdata
->nr_channels
; i
++, atdma
->dma_common
.chancnt
++) {
1264 struct at_dma_chan
*atchan
= &atdma
->chan
[i
];
1266 atchan
->chan_common
.device
= &atdma
->dma_common
;
1267 atchan
->chan_common
.cookie
= atchan
->completed_cookie
= 1;
1268 atchan
->chan_common
.chan_id
= i
;
1269 list_add_tail(&atchan
->chan_common
.device_node
,
1270 &atdma
->dma_common
.channels
);
1272 atchan
->ch_regs
= atdma
->regs
+ ch_regs(i
);
1273 spin_lock_init(&atchan
->lock
);
1274 atchan
->mask
= 1 << i
;
1276 INIT_LIST_HEAD(&atchan
->active_list
);
1277 INIT_LIST_HEAD(&atchan
->queue
);
1278 INIT_LIST_HEAD(&atchan
->free_list
);
1280 tasklet_init(&atchan
->tasklet
, atc_tasklet
,
1281 (unsigned long)atchan
);
1282 atc_enable_irq(atchan
);
1285 /* set base routines */
1286 atdma
->dma_common
.device_alloc_chan_resources
= atc_alloc_chan_resources
;
1287 atdma
->dma_common
.device_free_chan_resources
= atc_free_chan_resources
;
1288 atdma
->dma_common
.device_tx_status
= atc_tx_status
;
1289 atdma
->dma_common
.device_issue_pending
= atc_issue_pending
;
1290 atdma
->dma_common
.dev
= &pdev
->dev
;
1292 /* set prep routines based on capability */
1293 if (dma_has_cap(DMA_MEMCPY
, atdma
->dma_common
.cap_mask
))
1294 atdma
->dma_common
.device_prep_dma_memcpy
= atc_prep_dma_memcpy
;
1296 if (dma_has_cap(DMA_SLAVE
, atdma
->dma_common
.cap_mask
))
1297 atdma
->dma_common
.device_prep_slave_sg
= atc_prep_slave_sg
;
1299 if (dma_has_cap(DMA_CYCLIC
, atdma
->dma_common
.cap_mask
))
1300 atdma
->dma_common
.device_prep_dma_cyclic
= atc_prep_dma_cyclic
;
1302 if (dma_has_cap(DMA_SLAVE
, atdma
->dma_common
.cap_mask
) ||
1303 dma_has_cap(DMA_CYCLIC
, atdma
->dma_common
.cap_mask
))
1304 atdma
->dma_common
.device_control
= atc_control
;
1306 dma_writel(atdma
, EN
, AT_DMA_ENABLE
);
1308 dev_info(&pdev
->dev
, "Atmel AHB DMA Controller ( %s%s), %d channels\n",
1309 dma_has_cap(DMA_MEMCPY
, atdma
->dma_common
.cap_mask
) ? "cpy " : "",
1310 dma_has_cap(DMA_SLAVE
, atdma
->dma_common
.cap_mask
) ? "slave " : "",
1311 atdma
->dma_common
.chancnt
);
1313 dma_async_device_register(&atdma
->dma_common
);
1318 platform_set_drvdata(pdev
, NULL
);
1319 free_irq(platform_get_irq(pdev
, 0), atdma
);
1321 clk_disable(atdma
->clk
);
1322 clk_put(atdma
->clk
);
1324 iounmap(atdma
->regs
);
1327 release_mem_region(io
->start
, size
);
1333 static int __exit
at_dma_remove(struct platform_device
*pdev
)
1335 struct at_dma
*atdma
= platform_get_drvdata(pdev
);
1336 struct dma_chan
*chan
, *_chan
;
1337 struct resource
*io
;
1340 dma_async_device_unregister(&atdma
->dma_common
);
1342 dma_pool_destroy(atdma
->dma_desc_pool
);
1343 platform_set_drvdata(pdev
, NULL
);
1344 free_irq(platform_get_irq(pdev
, 0), atdma
);
1346 list_for_each_entry_safe(chan
, _chan
, &atdma
->dma_common
.channels
,
1348 struct at_dma_chan
*atchan
= to_at_dma_chan(chan
);
1350 /* Disable interrupts */
1351 atc_disable_irq(atchan
);
1352 tasklet_disable(&atchan
->tasklet
);
1354 tasklet_kill(&atchan
->tasklet
);
1355 list_del(&chan
->device_node
);
1358 clk_disable(atdma
->clk
);
1359 clk_put(atdma
->clk
);
1361 iounmap(atdma
->regs
);
1364 io
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1365 release_mem_region(io
->start
, io
->end
- io
->start
+ 1);
1372 static void at_dma_shutdown(struct platform_device
*pdev
)
1374 struct at_dma
*atdma
= platform_get_drvdata(pdev
);
1376 at_dma_off(platform_get_drvdata(pdev
));
1377 clk_disable(atdma
->clk
);
1380 static int at_dma_suspend_noirq(struct device
*dev
)
1382 struct platform_device
*pdev
= to_platform_device(dev
);
1383 struct at_dma
*atdma
= platform_get_drvdata(pdev
);
1385 at_dma_off(platform_get_drvdata(pdev
));
1386 clk_disable(atdma
->clk
);
1390 static int at_dma_resume_noirq(struct device
*dev
)
1392 struct platform_device
*pdev
= to_platform_device(dev
);
1393 struct at_dma
*atdma
= platform_get_drvdata(pdev
);
1395 clk_enable(atdma
->clk
);
1396 dma_writel(atdma
, EN
, AT_DMA_ENABLE
);
1400 static const struct dev_pm_ops at_dma_dev_pm_ops
= {
1401 .suspend_noirq
= at_dma_suspend_noirq
,
1402 .resume_noirq
= at_dma_resume_noirq
,
1405 static struct platform_driver at_dma_driver
= {
1406 .remove
= __exit_p(at_dma_remove
),
1407 .shutdown
= at_dma_shutdown
,
1410 .pm
= &at_dma_dev_pm_ops
,
1414 static int __init
at_dma_init(void)
1416 return platform_driver_probe(&at_dma_driver
, at_dma_probe
);
1418 subsys_initcall(at_dma_init
);
1420 static void __exit
at_dma_exit(void)
1422 platform_driver_unregister(&at_dma_driver
);
1424 module_exit(at_dma_exit
);
1426 MODULE_DESCRIPTION("Atmel AHB DMA Controller driver");
1427 MODULE_AUTHOR("Nicolas Ferre <nicolas.ferre@atmel.com>");
1428 MODULE_LICENSE("GPL");
1429 MODULE_ALIAS("platform:at_hdmac");