2 * Freescale MPC85xx, MPC83xx DMA Engine support
4 * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved.
7 * Zhang Wei <wei.zhang@freescale.com>, Jul 2007
8 * Ebony Zhu <ebony.zhu@freescale.com>, May 2007
11 * DMA engine driver for Freescale MPC8540 DMA controller, which is
12 * also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc.
13 * The support for MPC8349 DMA contorller is also added.
15 * This driver instructs the DMA controller to issue the PCI Read Multiple
16 * command for PCI read operations, instead of using the default PCI Read Line
17 * command. Please be aware that this setting may result in read pre-fetching
20 * This is free software; you can redistribute it and/or modify
21 * it under the terms of the GNU General Public License as published by
22 * the Free Software Foundation; either version 2 of the License, or
23 * (at your option) any later version.
27 #include <linux/init.h>
28 #include <linux/module.h>
29 #include <linux/pci.h>
30 #include <linux/interrupt.h>
31 #include <linux/dmaengine.h>
32 #include <linux/delay.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/dmapool.h>
35 #include <linux/of_platform.h>
37 #include <asm/fsldma.h>
40 static void dma_init(struct fsldma_chan
*chan
)
42 /* Reset the channel */
43 DMA_OUT(chan
, &chan
->regs
->mr
, 0, 32);
45 switch (chan
->feature
& FSL_DMA_IP_MASK
) {
47 /* Set the channel to below modes:
48 * EIE - Error interrupt enable
49 * EOSIE - End of segments interrupt enable (basic mode)
50 * EOLNIE - End of links interrupt enable
52 DMA_OUT(chan
, &chan
->regs
->mr
, FSL_DMA_MR_EIE
53 | FSL_DMA_MR_EOLNIE
| FSL_DMA_MR_EOSIE
, 32);
56 /* Set the channel to below modes:
57 * EOTIE - End-of-transfer interrupt enable
58 * PRC_RM - PCI read multiple
60 DMA_OUT(chan
, &chan
->regs
->mr
, FSL_DMA_MR_EOTIE
61 | FSL_DMA_MR_PRC_RM
, 32);
66 static void set_sr(struct fsldma_chan
*chan
, u32 val
)
68 DMA_OUT(chan
, &chan
->regs
->sr
, val
, 32);
71 static u32
get_sr(struct fsldma_chan
*chan
)
73 return DMA_IN(chan
, &chan
->regs
->sr
, 32);
76 static void set_desc_cnt(struct fsldma_chan
*chan
,
77 struct fsl_dma_ld_hw
*hw
, u32 count
)
79 hw
->count
= CPU_TO_DMA(chan
, count
, 32);
82 static void set_desc_src(struct fsldma_chan
*chan
,
83 struct fsl_dma_ld_hw
*hw
, dma_addr_t src
)
87 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_85XX
)
88 ? ((u64
)FSL_DMA_SATR_SREADTYPE_SNOOP_READ
<< 32) : 0;
89 hw
->src_addr
= CPU_TO_DMA(chan
, snoop_bits
| src
, 64);
92 static void set_desc_dst(struct fsldma_chan
*chan
,
93 struct fsl_dma_ld_hw
*hw
, dma_addr_t dst
)
97 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_85XX
)
98 ? ((u64
)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE
<< 32) : 0;
99 hw
->dst_addr
= CPU_TO_DMA(chan
, snoop_bits
| dst
, 64);
102 static void set_desc_next(struct fsldma_chan
*chan
,
103 struct fsl_dma_ld_hw
*hw
, dma_addr_t next
)
107 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_83XX
)
109 hw
->next_ln_addr
= CPU_TO_DMA(chan
, snoop_bits
| next
, 64);
112 static void set_cdar(struct fsldma_chan
*chan
, dma_addr_t addr
)
114 DMA_OUT(chan
, &chan
->regs
->cdar
, addr
| FSL_DMA_SNEN
, 64);
117 static dma_addr_t
get_cdar(struct fsldma_chan
*chan
)
119 return DMA_IN(chan
, &chan
->regs
->cdar
, 64) & ~FSL_DMA_SNEN
;
122 static dma_addr_t
get_ndar(struct fsldma_chan
*chan
)
124 return DMA_IN(chan
, &chan
->regs
->ndar
, 64);
127 static u32
get_bcr(struct fsldma_chan
*chan
)
129 return DMA_IN(chan
, &chan
->regs
->bcr
, 32);
132 static int dma_is_idle(struct fsldma_chan
*chan
)
134 u32 sr
= get_sr(chan
);
135 return (!(sr
& FSL_DMA_SR_CB
)) || (sr
& FSL_DMA_SR_CH
);
138 static void dma_start(struct fsldma_chan
*chan
)
142 mode
= DMA_IN(chan
, &chan
->regs
->mr
, 32);
144 if ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_85XX
) {
145 if (chan
->feature
& FSL_DMA_CHAN_PAUSE_EXT
) {
146 DMA_OUT(chan
, &chan
->regs
->bcr
, 0, 32);
147 mode
|= FSL_DMA_MR_EMP_EN
;
149 mode
&= ~FSL_DMA_MR_EMP_EN
;
153 if (chan
->feature
& FSL_DMA_CHAN_START_EXT
)
154 mode
|= FSL_DMA_MR_EMS_EN
;
156 mode
|= FSL_DMA_MR_CS
;
158 DMA_OUT(chan
, &chan
->regs
->mr
, mode
, 32);
161 static void dma_halt(struct fsldma_chan
*chan
)
166 mode
= DMA_IN(chan
, &chan
->regs
->mr
, 32);
167 mode
|= FSL_DMA_MR_CA
;
168 DMA_OUT(chan
, &chan
->regs
->mr
, mode
, 32);
170 mode
&= ~(FSL_DMA_MR_CS
| FSL_DMA_MR_EMS_EN
| FSL_DMA_MR_CA
);
171 DMA_OUT(chan
, &chan
->regs
->mr
, mode
, 32);
173 for (i
= 0; i
< 100; i
++) {
174 if (dma_is_idle(chan
))
180 if (!dma_is_idle(chan
))
181 dev_err(chan
->dev
, "DMA halt timeout!\n");
184 static void set_ld_eol(struct fsldma_chan
*chan
,
185 struct fsl_desc_sw
*desc
)
189 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_83XX
)
192 desc
->hw
.next_ln_addr
= CPU_TO_DMA(chan
,
193 DMA_TO_CPU(chan
, desc
->hw
.next_ln_addr
, 64) | FSL_DMA_EOL
198 * fsl_chan_set_src_loop_size - Set source address hold transfer size
199 * @chan : Freescale DMA channel
200 * @size : Address loop size, 0 for disable loop
202 * The set source address hold transfer size. The source
203 * address hold or loop transfer size is when the DMA transfer
204 * data from source address (SA), if the loop size is 4, the DMA will
205 * read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA,
206 * SA + 1 ... and so on.
208 static void fsl_chan_set_src_loop_size(struct fsldma_chan
*chan
, int size
)
212 mode
= DMA_IN(chan
, &chan
->regs
->mr
, 32);
216 mode
&= ~FSL_DMA_MR_SAHE
;
222 mode
|= FSL_DMA_MR_SAHE
| (__ilog2(size
) << 14);
226 DMA_OUT(chan
, &chan
->regs
->mr
, mode
, 32);
230 * fsl_chan_set_dst_loop_size - Set destination address hold transfer size
231 * @chan : Freescale DMA channel
232 * @size : Address loop size, 0 for disable loop
234 * The set destination address hold transfer size. The destination
235 * address hold or loop transfer size is when the DMA transfer
236 * data to destination address (TA), if the loop size is 4, the DMA will
237 * write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA,
238 * TA + 1 ... and so on.
240 static void fsl_chan_set_dst_loop_size(struct fsldma_chan
*chan
, int size
)
244 mode
= DMA_IN(chan
, &chan
->regs
->mr
, 32);
248 mode
&= ~FSL_DMA_MR_DAHE
;
254 mode
|= FSL_DMA_MR_DAHE
| (__ilog2(size
) << 16);
258 DMA_OUT(chan
, &chan
->regs
->mr
, mode
, 32);
262 * fsl_chan_set_request_count - Set DMA Request Count for external control
263 * @chan : Freescale DMA channel
264 * @size : Number of bytes to transfer in a single request
266 * The Freescale DMA channel can be controlled by the external signal DREQ#.
267 * The DMA request count is how many bytes are allowed to transfer before
268 * pausing the channel, after which a new assertion of DREQ# resumes channel
271 * A size of 0 disables external pause control. The maximum size is 1024.
273 static void fsl_chan_set_request_count(struct fsldma_chan
*chan
, int size
)
279 mode
= DMA_IN(chan
, &chan
->regs
->mr
, 32);
280 mode
|= (__ilog2(size
) << 24) & 0x0f000000;
282 DMA_OUT(chan
, &chan
->regs
->mr
, mode
, 32);
286 * fsl_chan_toggle_ext_pause - Toggle channel external pause status
287 * @chan : Freescale DMA channel
288 * @enable : 0 is disabled, 1 is enabled.
290 * The Freescale DMA channel can be controlled by the external signal DREQ#.
291 * The DMA Request Count feature should be used in addition to this feature
292 * to set the number of bytes to transfer before pausing the channel.
294 static void fsl_chan_toggle_ext_pause(struct fsldma_chan
*chan
, int enable
)
297 chan
->feature
|= FSL_DMA_CHAN_PAUSE_EXT
;
299 chan
->feature
&= ~FSL_DMA_CHAN_PAUSE_EXT
;
303 * fsl_chan_toggle_ext_start - Toggle channel external start status
304 * @chan : Freescale DMA channel
305 * @enable : 0 is disabled, 1 is enabled.
307 * If enable the external start, the channel can be started by an
308 * external DMA start pin. So the dma_start() does not start the
309 * transfer immediately. The DMA channel will wait for the
310 * control pin asserted.
312 static void fsl_chan_toggle_ext_start(struct fsldma_chan
*chan
, int enable
)
315 chan
->feature
|= FSL_DMA_CHAN_START_EXT
;
317 chan
->feature
&= ~FSL_DMA_CHAN_START_EXT
;
320 static void append_ld_queue(struct fsldma_chan
*chan
,
321 struct fsl_desc_sw
*desc
)
323 struct fsl_desc_sw
*tail
= to_fsl_desc(chan
->ld_pending
.prev
);
325 if (list_empty(&chan
->ld_pending
))
329 * Add the hardware descriptor to the chain of hardware descriptors
330 * that already exists in memory.
332 * This will un-set the EOL bit of the existing transaction, and the
333 * last link in this transaction will become the EOL descriptor.
335 set_desc_next(chan
, &tail
->hw
, desc
->async_tx
.phys
);
338 * Add the software descriptor and all children to the list
339 * of pending transactions
342 list_splice_tail_init(&desc
->tx_list
, &chan
->ld_pending
);
345 static dma_cookie_t
fsl_dma_tx_submit(struct dma_async_tx_descriptor
*tx
)
347 struct fsldma_chan
*chan
= to_fsl_chan(tx
->chan
);
348 struct fsl_desc_sw
*desc
= tx_to_fsl_desc(tx
);
349 struct fsl_desc_sw
*child
;
353 spin_lock_irqsave(&chan
->desc_lock
, flags
);
356 * assign cookies to all of the software descriptors
357 * that make up this transaction
359 cookie
= chan
->common
.cookie
;
360 list_for_each_entry(child
, &desc
->tx_list
, node
) {
365 child
->async_tx
.cookie
= cookie
;
368 chan
->common
.cookie
= cookie
;
370 /* put this transaction onto the tail of the pending queue */
371 append_ld_queue(chan
, desc
);
373 spin_unlock_irqrestore(&chan
->desc_lock
, flags
);
379 * fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool.
380 * @chan : Freescale DMA channel
382 * Return - The descriptor allocated. NULL for failed.
384 static struct fsl_desc_sw
*fsl_dma_alloc_descriptor(
385 struct fsldma_chan
*chan
)
387 struct fsl_desc_sw
*desc
;
390 desc
= dma_pool_alloc(chan
->desc_pool
, GFP_ATOMIC
, &pdesc
);
392 dev_dbg(chan
->dev
, "out of memory for link desc\n");
396 memset(desc
, 0, sizeof(*desc
));
397 INIT_LIST_HEAD(&desc
->tx_list
);
398 dma_async_tx_descriptor_init(&desc
->async_tx
, &chan
->common
);
399 desc
->async_tx
.tx_submit
= fsl_dma_tx_submit
;
400 desc
->async_tx
.phys
= pdesc
;
407 * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel.
408 * @chan : Freescale DMA channel
410 * This function will create a dma pool for descriptor allocation.
412 * Return - The number of descriptors allocated.
414 static int fsl_dma_alloc_chan_resources(struct dma_chan
*dchan
)
416 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
418 /* Has this channel already been allocated? */
423 * We need the descriptor to be aligned to 32bytes
424 * for meeting FSL DMA specification requirement.
426 chan
->desc_pool
= dma_pool_create("fsl_dma_engine_desc_pool",
428 sizeof(struct fsl_desc_sw
),
429 __alignof__(struct fsl_desc_sw
), 0);
430 if (!chan
->desc_pool
) {
431 dev_err(chan
->dev
, "unable to allocate channel %d "
432 "descriptor pool\n", chan
->id
);
436 /* there is at least one descriptor free to be allocated */
441 * fsldma_free_desc_list - Free all descriptors in a queue
442 * @chan: Freescae DMA channel
443 * @list: the list to free
445 * LOCKING: must hold chan->desc_lock
447 static void fsldma_free_desc_list(struct fsldma_chan
*chan
,
448 struct list_head
*list
)
450 struct fsl_desc_sw
*desc
, *_desc
;
452 list_for_each_entry_safe(desc
, _desc
, list
, node
) {
453 list_del(&desc
->node
);
454 dma_pool_free(chan
->desc_pool
, desc
, desc
->async_tx
.phys
);
458 static void fsldma_free_desc_list_reverse(struct fsldma_chan
*chan
,
459 struct list_head
*list
)
461 struct fsl_desc_sw
*desc
, *_desc
;
463 list_for_each_entry_safe_reverse(desc
, _desc
, list
, node
) {
464 list_del(&desc
->node
);
465 dma_pool_free(chan
->desc_pool
, desc
, desc
->async_tx
.phys
);
470 * fsl_dma_free_chan_resources - Free all resources of the channel.
471 * @chan : Freescale DMA channel
473 static void fsl_dma_free_chan_resources(struct dma_chan
*dchan
)
475 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
478 dev_dbg(chan
->dev
, "Free all channel resources.\n");
479 spin_lock_irqsave(&chan
->desc_lock
, flags
);
480 fsldma_free_desc_list(chan
, &chan
->ld_pending
);
481 fsldma_free_desc_list(chan
, &chan
->ld_running
);
482 spin_unlock_irqrestore(&chan
->desc_lock
, flags
);
484 dma_pool_destroy(chan
->desc_pool
);
485 chan
->desc_pool
= NULL
;
488 static struct dma_async_tx_descriptor
*
489 fsl_dma_prep_interrupt(struct dma_chan
*dchan
, unsigned long flags
)
491 struct fsldma_chan
*chan
;
492 struct fsl_desc_sw
*new;
497 chan
= to_fsl_chan(dchan
);
499 new = fsl_dma_alloc_descriptor(chan
);
501 dev_err(chan
->dev
, "No free memory for link descriptor\n");
505 new->async_tx
.cookie
= -EBUSY
;
506 new->async_tx
.flags
= flags
;
508 /* Insert the link descriptor to the LD ring */
509 list_add_tail(&new->node
, &new->tx_list
);
511 /* Set End-of-link to the last link descriptor of new list*/
512 set_ld_eol(chan
, new);
514 return &new->async_tx
;
517 static struct dma_async_tx_descriptor
*fsl_dma_prep_memcpy(
518 struct dma_chan
*dchan
, dma_addr_t dma_dst
, dma_addr_t dma_src
,
519 size_t len
, unsigned long flags
)
521 struct fsldma_chan
*chan
;
522 struct fsl_desc_sw
*first
= NULL
, *prev
= NULL
, *new;
531 chan
= to_fsl_chan(dchan
);
535 /* Allocate the link descriptor from DMA pool */
536 new = fsl_dma_alloc_descriptor(chan
);
539 "No free memory for link descriptor\n");
542 #ifdef FSL_DMA_LD_DEBUG
543 dev_dbg(chan
->dev
, "new link desc alloc %p\n", new);
546 copy
= min(len
, (size_t)FSL_DMA_BCR_MAX_CNT
);
548 set_desc_cnt(chan
, &new->hw
, copy
);
549 set_desc_src(chan
, &new->hw
, dma_src
);
550 set_desc_dst(chan
, &new->hw
, dma_dst
);
555 set_desc_next(chan
, &prev
->hw
, new->async_tx
.phys
);
557 new->async_tx
.cookie
= 0;
558 async_tx_ack(&new->async_tx
);
565 /* Insert the link descriptor to the LD ring */
566 list_add_tail(&new->node
, &first
->tx_list
);
569 new->async_tx
.flags
= flags
; /* client is in control of this ack */
570 new->async_tx
.cookie
= -EBUSY
;
572 /* Set End-of-link to the last link descriptor of new list*/
573 set_ld_eol(chan
, new);
575 return &first
->async_tx
;
581 fsldma_free_desc_list_reverse(chan
, &first
->tx_list
);
586 * fsl_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
588 * @sgl: scatterlist to transfer to/from
589 * @sg_len: number of entries in @scatterlist
590 * @direction: DMA direction
591 * @flags: DMAEngine flags
593 * Prepare a set of descriptors for a DMA_SLAVE transaction. Following the
594 * DMA_SLAVE API, this gets the device-specific information from the
595 * chan->private variable.
597 static struct dma_async_tx_descriptor
*fsl_dma_prep_slave_sg(
598 struct dma_chan
*dchan
, struct scatterlist
*sgl
, unsigned int sg_len
,
599 enum dma_data_direction direction
, unsigned long flags
)
601 struct fsldma_chan
*chan
;
602 struct fsl_desc_sw
*first
= NULL
, *prev
= NULL
, *new = NULL
;
603 struct fsl_dma_slave
*slave
;
607 struct scatterlist
*sg
;
610 struct fsl_dma_hw_addr
*hw
;
611 dma_addr_t dma_dst
, dma_src
;
619 chan
= to_fsl_chan(dchan
);
620 slave
= dchan
->private;
622 if (list_empty(&slave
->addresses
))
625 hw
= list_first_entry(&slave
->addresses
, struct fsl_dma_hw_addr
, entry
);
629 * Build the hardware transaction to copy from the scatterlist to
630 * the hardware, or from the hardware to the scatterlist
632 * If you are copying from the hardware to the scatterlist and it
633 * takes two hardware entries to fill an entire page, then both
634 * hardware entries will be coalesced into the same page
636 * If you are copying from the scatterlist to the hardware and a
637 * single page can fill two hardware entries, then the data will
638 * be read out of the page into the first hardware entry, and so on
640 for_each_sg(sgl
, sg
, sg_len
, i
) {
643 /* Loop until the entire scatterlist entry is used */
644 while (sg_used
< sg_dma_len(sg
)) {
647 * If we've used up the current hardware address/length
648 * pair, we need to load a new one
650 * This is done in a while loop so that descriptors with
651 * length == 0 will be skipped
653 while (hw_used
>= hw
->length
) {
656 * If the current hardware entry is the last
657 * entry in the list, we're finished
659 if (list_is_last(&hw
->entry
, &slave
->addresses
))
662 /* Get the next hardware address/length pair */
663 hw
= list_entry(hw
->entry
.next
,
664 struct fsl_dma_hw_addr
, entry
);
668 /* Allocate the link descriptor from DMA pool */
669 new = fsl_dma_alloc_descriptor(chan
);
671 dev_err(chan
->dev
, "No free memory for "
672 "link descriptor\n");
675 #ifdef FSL_DMA_LD_DEBUG
676 dev_dbg(chan
->dev
, "new link desc alloc %p\n", new);
680 * Calculate the maximum number of bytes to transfer,
681 * making sure it is less than the DMA controller limit
683 copy
= min_t(size_t, sg_dma_len(sg
) - sg_used
,
684 hw
->length
- hw_used
);
685 copy
= min_t(size_t, copy
, FSL_DMA_BCR_MAX_CNT
);
689 * from the hardware to the scatterlist
692 * from the scatterlist to the hardware
694 if (direction
== DMA_FROM_DEVICE
) {
695 dma_src
= hw
->address
+ hw_used
;
696 dma_dst
= sg_dma_address(sg
) + sg_used
;
698 dma_src
= sg_dma_address(sg
) + sg_used
;
699 dma_dst
= hw
->address
+ hw_used
;
702 /* Fill in the descriptor */
703 set_desc_cnt(chan
, &new->hw
, copy
);
704 set_desc_src(chan
, &new->hw
, dma_src
);
705 set_desc_dst(chan
, &new->hw
, dma_dst
);
708 * If this is not the first descriptor, chain the
709 * current descriptor after the previous descriptor
714 set_desc_next(chan
, &prev
->hw
,
718 new->async_tx
.cookie
= 0;
719 async_tx_ack(&new->async_tx
);
725 /* Insert the link descriptor into the LD ring */
726 list_add_tail(&new->node
, &first
->tx_list
);
732 /* All of the hardware address/length pairs had length == 0 */
736 new->async_tx
.flags
= flags
;
737 new->async_tx
.cookie
= -EBUSY
;
739 /* Set End-of-link to the last link descriptor of new list */
740 set_ld_eol(chan
, new);
742 /* Enable extra controller features */
743 if (chan
->set_src_loop_size
)
744 chan
->set_src_loop_size(chan
, slave
->src_loop_size
);
746 if (chan
->set_dst_loop_size
)
747 chan
->set_dst_loop_size(chan
, slave
->dst_loop_size
);
749 if (chan
->toggle_ext_start
)
750 chan
->toggle_ext_start(chan
, slave
->external_start
);
752 if (chan
->toggle_ext_pause
)
753 chan
->toggle_ext_pause(chan
, slave
->external_pause
);
755 if (chan
->set_request_count
)
756 chan
->set_request_count(chan
, slave
->request_count
);
758 return &first
->async_tx
;
761 /* If first was not set, then we failed to allocate the very first
762 * descriptor, and we're done */
767 * First is set, so all of the descriptors we allocated have been added
768 * to first->tx_list, INCLUDING "first" itself. Therefore we
769 * must traverse the list backwards freeing each descriptor in turn
771 * We're re-using variables for the loop, oh well
773 fsldma_free_desc_list_reverse(chan
, &first
->tx_list
);
777 static void fsl_dma_device_terminate_all(struct dma_chan
*dchan
)
779 struct fsldma_chan
*chan
;
785 chan
= to_fsl_chan(dchan
);
787 /* Halt the DMA engine */
790 spin_lock_irqsave(&chan
->desc_lock
, flags
);
792 /* Remove and free all of the descriptors in the LD queue */
793 fsldma_free_desc_list(chan
, &chan
->ld_pending
);
794 fsldma_free_desc_list(chan
, &chan
->ld_running
);
796 spin_unlock_irqrestore(&chan
->desc_lock
, flags
);
800 * fsl_dma_update_completed_cookie - Update the completed cookie.
801 * @chan : Freescale DMA channel
805 static void fsl_dma_update_completed_cookie(struct fsldma_chan
*chan
)
807 struct fsl_desc_sw
*desc
;
811 spin_lock_irqsave(&chan
->desc_lock
, flags
);
813 if (list_empty(&chan
->ld_running
)) {
814 dev_dbg(chan
->dev
, "no running descriptors\n");
818 /* Get the last descriptor, update the cookie to that */
819 desc
= to_fsl_desc(chan
->ld_running
.prev
);
820 if (dma_is_idle(chan
))
821 cookie
= desc
->async_tx
.cookie
;
823 cookie
= desc
->async_tx
.cookie
- 1;
825 chan
->completed_cookie
= cookie
;
828 spin_unlock_irqrestore(&chan
->desc_lock
, flags
);
832 * fsldma_desc_status - Check the status of a descriptor
833 * @chan: Freescale DMA channel
834 * @desc: DMA SW descriptor
836 * This function will return the status of the given descriptor
838 static enum dma_status
fsldma_desc_status(struct fsldma_chan
*chan
,
839 struct fsl_desc_sw
*desc
)
841 return dma_async_is_complete(desc
->async_tx
.cookie
,
842 chan
->completed_cookie
,
843 chan
->common
.cookie
);
847 * fsl_chan_ld_cleanup - Clean up link descriptors
848 * @chan : Freescale DMA channel
850 * This function clean up the ld_queue of DMA channel.
852 static void fsl_chan_ld_cleanup(struct fsldma_chan
*chan
)
854 struct fsl_desc_sw
*desc
, *_desc
;
857 spin_lock_irqsave(&chan
->desc_lock
, flags
);
859 dev_dbg(chan
->dev
, "chan completed_cookie = %d\n", chan
->completed_cookie
);
860 list_for_each_entry_safe(desc
, _desc
, &chan
->ld_running
, node
) {
861 dma_async_tx_callback callback
;
862 void *callback_param
;
864 if (fsldma_desc_status(chan
, desc
) == DMA_IN_PROGRESS
)
867 /* Remove from the list of running transactions */
868 list_del(&desc
->node
);
870 /* Run the link descriptor callback function */
871 callback
= desc
->async_tx
.callback
;
872 callback_param
= desc
->async_tx
.callback_param
;
874 spin_unlock_irqrestore(&chan
->desc_lock
, flags
);
875 dev_dbg(chan
->dev
, "LD %p callback\n", desc
);
876 callback(callback_param
);
877 spin_lock_irqsave(&chan
->desc_lock
, flags
);
880 /* Run any dependencies, then free the descriptor */
881 dma_run_dependencies(&desc
->async_tx
);
882 dma_pool_free(chan
->desc_pool
, desc
, desc
->async_tx
.phys
);
885 spin_unlock_irqrestore(&chan
->desc_lock
, flags
);
889 * fsl_chan_xfer_ld_queue - transfer any pending transactions
890 * @chan : Freescale DMA channel
892 * This will make sure that any pending transactions will be run.
893 * If the DMA controller is idle, it will be started. Otherwise,
894 * the DMA controller's interrupt handler will start any pending
895 * transactions when it becomes idle.
897 static void fsl_chan_xfer_ld_queue(struct fsldma_chan
*chan
)
899 struct fsl_desc_sw
*desc
;
902 spin_lock_irqsave(&chan
->desc_lock
, flags
);
905 * If the list of pending descriptors is empty, then we
906 * don't need to do any work at all
908 if (list_empty(&chan
->ld_pending
)) {
909 dev_dbg(chan
->dev
, "no pending LDs\n");
914 * The DMA controller is not idle, which means the interrupt
915 * handler will start any queued transactions when it runs
916 * at the end of the current transaction
918 if (!dma_is_idle(chan
)) {
919 dev_dbg(chan
->dev
, "DMA controller still busy\n");
925 * make sure the dma_halt() function really un-wedges the
926 * controller as much as possible
931 * If there are some link descriptors which have not been
932 * transferred, we need to start the controller
936 * Move all elements from the queue of pending transactions
937 * onto the list of running transactions
939 desc
= list_first_entry(&chan
->ld_pending
, struct fsl_desc_sw
, node
);
940 list_splice_tail_init(&chan
->ld_pending
, &chan
->ld_running
);
943 * Program the descriptor's address into the DMA controller,
944 * then start the DMA transaction
946 set_cdar(chan
, desc
->async_tx
.phys
);
950 spin_unlock_irqrestore(&chan
->desc_lock
, flags
);
954 * fsl_dma_memcpy_issue_pending - Issue the DMA start command
955 * @chan : Freescale DMA channel
957 static void fsl_dma_memcpy_issue_pending(struct dma_chan
*dchan
)
959 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
960 fsl_chan_xfer_ld_queue(chan
);
964 * fsl_dma_is_complete - Determine the DMA status
965 * @chan : Freescale DMA channel
967 static enum dma_status
fsl_dma_is_complete(struct dma_chan
*dchan
,
972 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
973 dma_cookie_t last_used
;
974 dma_cookie_t last_complete
;
976 fsl_chan_ld_cleanup(chan
);
978 last_used
= dchan
->cookie
;
979 last_complete
= chan
->completed_cookie
;
982 *done
= last_complete
;
987 return dma_async_is_complete(cookie
, last_complete
, last_used
);
990 /*----------------------------------------------------------------------------*/
991 /* Interrupt Handling */
992 /*----------------------------------------------------------------------------*/
994 static irqreturn_t
fsldma_chan_irq(int irq
, void *data
)
996 struct fsldma_chan
*chan
= data
;
997 int update_cookie
= 0;
1001 /* save and clear the status register */
1002 stat
= get_sr(chan
);
1004 dev_dbg(chan
->dev
, "irq: channel %d, stat = 0x%x\n", chan
->id
, stat
);
1006 stat
&= ~(FSL_DMA_SR_CB
| FSL_DMA_SR_CH
);
1010 if (stat
& FSL_DMA_SR_TE
)
1011 dev_err(chan
->dev
, "Transfer Error!\n");
1015 * The DMA_INTERRUPT async_tx is a NULL transfer, which will
1016 * triger a PE interrupt.
1018 if (stat
& FSL_DMA_SR_PE
) {
1019 dev_dbg(chan
->dev
, "irq: Programming Error INT\n");
1020 if (get_bcr(chan
) == 0) {
1021 /* BCR register is 0, this is a DMA_INTERRUPT async_tx.
1022 * Now, update the completed cookie, and continue the
1023 * next uncompleted transfer.
1028 stat
&= ~FSL_DMA_SR_PE
;
1032 * If the link descriptor segment transfer finishes,
1033 * we will recycle the used descriptor.
1035 if (stat
& FSL_DMA_SR_EOSI
) {
1036 dev_dbg(chan
->dev
, "irq: End-of-segments INT\n");
1037 dev_dbg(chan
->dev
, "irq: clndar 0x%llx, nlndar 0x%llx\n",
1038 (unsigned long long)get_cdar(chan
),
1039 (unsigned long long)get_ndar(chan
));
1040 stat
&= ~FSL_DMA_SR_EOSI
;
1045 * For MPC8349, EOCDI event need to update cookie
1046 * and start the next transfer if it exist.
1048 if (stat
& FSL_DMA_SR_EOCDI
) {
1049 dev_dbg(chan
->dev
, "irq: End-of-Chain link INT\n");
1050 stat
&= ~FSL_DMA_SR_EOCDI
;
1056 * If it current transfer is the end-of-transfer,
1057 * we should clear the Channel Start bit for
1058 * prepare next transfer.
1060 if (stat
& FSL_DMA_SR_EOLNI
) {
1061 dev_dbg(chan
->dev
, "irq: End-of-link INT\n");
1062 stat
&= ~FSL_DMA_SR_EOLNI
;
1067 fsl_dma_update_completed_cookie(chan
);
1069 fsl_chan_xfer_ld_queue(chan
);
1071 dev_dbg(chan
->dev
, "irq: unhandled sr 0x%02x\n", stat
);
1073 dev_dbg(chan
->dev
, "irq: Exit\n");
1074 tasklet_schedule(&chan
->tasklet
);
1078 static void dma_do_tasklet(unsigned long data
)
1080 struct fsldma_chan
*chan
= (struct fsldma_chan
*)data
;
1081 fsl_chan_ld_cleanup(chan
);
1084 static irqreturn_t
fsldma_ctrl_irq(int irq
, void *data
)
1086 struct fsldma_device
*fdev
= data
;
1087 struct fsldma_chan
*chan
;
1088 unsigned int handled
= 0;
1092 gsr
= (fdev
->feature
& FSL_DMA_BIG_ENDIAN
) ? in_be32(fdev
->regs
)
1093 : in_le32(fdev
->regs
);
1095 dev_dbg(fdev
->dev
, "IRQ: gsr 0x%.8x\n", gsr
);
1097 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1098 chan
= fdev
->chan
[i
];
1103 dev_dbg(fdev
->dev
, "IRQ: chan %d\n", chan
->id
);
1104 fsldma_chan_irq(irq
, chan
);
1112 return IRQ_RETVAL(handled
);
1115 static void fsldma_free_irqs(struct fsldma_device
*fdev
)
1117 struct fsldma_chan
*chan
;
1120 if (fdev
->irq
!= NO_IRQ
) {
1121 dev_dbg(fdev
->dev
, "free per-controller IRQ\n");
1122 free_irq(fdev
->irq
, fdev
);
1126 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1127 chan
= fdev
->chan
[i
];
1128 if (chan
&& chan
->irq
!= NO_IRQ
) {
1129 dev_dbg(fdev
->dev
, "free channel %d IRQ\n", chan
->id
);
1130 free_irq(chan
->irq
, chan
);
1135 static int fsldma_request_irqs(struct fsldma_device
*fdev
)
1137 struct fsldma_chan
*chan
;
1141 /* if we have a per-controller IRQ, use that */
1142 if (fdev
->irq
!= NO_IRQ
) {
1143 dev_dbg(fdev
->dev
, "request per-controller IRQ\n");
1144 ret
= request_irq(fdev
->irq
, fsldma_ctrl_irq
, IRQF_SHARED
,
1145 "fsldma-controller", fdev
);
1149 /* no per-controller IRQ, use the per-channel IRQs */
1150 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1151 chan
= fdev
->chan
[i
];
1155 if (chan
->irq
== NO_IRQ
) {
1156 dev_err(fdev
->dev
, "no interrupts property defined for "
1157 "DMA channel %d. Please fix your "
1158 "device tree\n", chan
->id
);
1163 dev_dbg(fdev
->dev
, "request channel %d IRQ\n", chan
->id
);
1164 ret
= request_irq(chan
->irq
, fsldma_chan_irq
, IRQF_SHARED
,
1165 "fsldma-chan", chan
);
1167 dev_err(fdev
->dev
, "unable to request IRQ for DMA "
1168 "channel %d\n", chan
->id
);
1176 for (/* none */; i
>= 0; i
--) {
1177 chan
= fdev
->chan
[i
];
1181 if (chan
->irq
== NO_IRQ
)
1184 free_irq(chan
->irq
, chan
);
1190 /*----------------------------------------------------------------------------*/
1191 /* OpenFirmware Subsystem */
1192 /*----------------------------------------------------------------------------*/
1194 static int __devinit
fsl_dma_chan_probe(struct fsldma_device
*fdev
,
1195 struct device_node
*node
, u32 feature
, const char *compatible
)
1197 struct fsldma_chan
*chan
;
1198 struct resource res
;
1202 chan
= kzalloc(sizeof(*chan
), GFP_KERNEL
);
1204 dev_err(fdev
->dev
, "no free memory for DMA channels!\n");
1209 /* ioremap registers for use */
1210 chan
->regs
= of_iomap(node
, 0);
1212 dev_err(fdev
->dev
, "unable to ioremap registers\n");
1217 err
= of_address_to_resource(node
, 0, &res
);
1219 dev_err(fdev
->dev
, "unable to find 'reg' property\n");
1220 goto out_iounmap_regs
;
1223 chan
->feature
= feature
;
1225 fdev
->feature
= chan
->feature
;
1228 * If the DMA device's feature is different than the feature
1229 * of its channels, report the bug
1231 WARN_ON(fdev
->feature
!= chan
->feature
);
1233 chan
->dev
= fdev
->dev
;
1234 chan
->id
= ((res
.start
- 0x100) & 0xfff) >> 7;
1235 if (chan
->id
>= FSL_DMA_MAX_CHANS_PER_DEVICE
) {
1236 dev_err(fdev
->dev
, "too many channels for device\n");
1238 goto out_iounmap_regs
;
1241 fdev
->chan
[chan
->id
] = chan
;
1242 tasklet_init(&chan
->tasklet
, dma_do_tasklet
, (unsigned long)chan
);
1244 /* Initialize the channel */
1247 /* Clear cdar registers */
1250 switch (chan
->feature
& FSL_DMA_IP_MASK
) {
1251 case FSL_DMA_IP_85XX
:
1252 chan
->toggle_ext_pause
= fsl_chan_toggle_ext_pause
;
1253 case FSL_DMA_IP_83XX
:
1254 chan
->toggle_ext_start
= fsl_chan_toggle_ext_start
;
1255 chan
->set_src_loop_size
= fsl_chan_set_src_loop_size
;
1256 chan
->set_dst_loop_size
= fsl_chan_set_dst_loop_size
;
1257 chan
->set_request_count
= fsl_chan_set_request_count
;
1260 spin_lock_init(&chan
->desc_lock
);
1261 INIT_LIST_HEAD(&chan
->ld_pending
);
1262 INIT_LIST_HEAD(&chan
->ld_running
);
1264 chan
->common
.device
= &fdev
->common
;
1266 /* find the IRQ line, if it exists in the device tree */
1267 chan
->irq
= irq_of_parse_and_map(node
, 0);
1269 /* Add the channel to DMA device channel list */
1270 list_add_tail(&chan
->common
.device_node
, &fdev
->common
.channels
);
1271 fdev
->common
.chancnt
++;
1273 dev_info(fdev
->dev
, "#%d (%s), irq %d\n", chan
->id
, compatible
,
1274 chan
->irq
!= NO_IRQ
? chan
->irq
: fdev
->irq
);
1279 iounmap(chan
->regs
);
1286 static void fsl_dma_chan_remove(struct fsldma_chan
*chan
)
1288 irq_dispose_mapping(chan
->irq
);
1289 list_del(&chan
->common
.device_node
);
1290 iounmap(chan
->regs
);
1294 static int __devinit
fsldma_of_probe(struct of_device
*op
,
1295 const struct of_device_id
*match
)
1297 struct fsldma_device
*fdev
;
1298 struct device_node
*child
;
1301 fdev
= kzalloc(sizeof(*fdev
), GFP_KERNEL
);
1303 dev_err(&op
->dev
, "No enough memory for 'priv'\n");
1308 fdev
->dev
= &op
->dev
;
1309 INIT_LIST_HEAD(&fdev
->common
.channels
);
1311 /* ioremap the registers for use */
1312 fdev
->regs
= of_iomap(op
->node
, 0);
1314 dev_err(&op
->dev
, "unable to ioremap registers\n");
1319 /* map the channel IRQ if it exists, but don't hookup the handler yet */
1320 fdev
->irq
= irq_of_parse_and_map(op
->node
, 0);
1322 dma_cap_set(DMA_MEMCPY
, fdev
->common
.cap_mask
);
1323 dma_cap_set(DMA_INTERRUPT
, fdev
->common
.cap_mask
);
1324 dma_cap_set(DMA_SLAVE
, fdev
->common
.cap_mask
);
1325 fdev
->common
.device_alloc_chan_resources
= fsl_dma_alloc_chan_resources
;
1326 fdev
->common
.device_free_chan_resources
= fsl_dma_free_chan_resources
;
1327 fdev
->common
.device_prep_dma_interrupt
= fsl_dma_prep_interrupt
;
1328 fdev
->common
.device_prep_dma_memcpy
= fsl_dma_prep_memcpy
;
1329 fdev
->common
.device_is_tx_complete
= fsl_dma_is_complete
;
1330 fdev
->common
.device_issue_pending
= fsl_dma_memcpy_issue_pending
;
1331 fdev
->common
.device_prep_slave_sg
= fsl_dma_prep_slave_sg
;
1332 fdev
->common
.device_terminate_all
= fsl_dma_device_terminate_all
;
1333 fdev
->common
.dev
= &op
->dev
;
1335 dev_set_drvdata(&op
->dev
, fdev
);
1338 * We cannot use of_platform_bus_probe() because there is no
1339 * of_platform_bus_remove(). Instead, we manually instantiate every DMA
1342 for_each_child_of_node(op
->node
, child
) {
1343 if (of_device_is_compatible(child
, "fsl,eloplus-dma-channel")) {
1344 fsl_dma_chan_probe(fdev
, child
,
1345 FSL_DMA_IP_85XX
| FSL_DMA_BIG_ENDIAN
,
1346 "fsl,eloplus-dma-channel");
1349 if (of_device_is_compatible(child
, "fsl,elo-dma-channel")) {
1350 fsl_dma_chan_probe(fdev
, child
,
1351 FSL_DMA_IP_83XX
| FSL_DMA_LITTLE_ENDIAN
,
1352 "fsl,elo-dma-channel");
1357 * Hookup the IRQ handler(s)
1359 * If we have a per-controller interrupt, we prefer that to the
1360 * per-channel interrupts to reduce the number of shared interrupt
1361 * handlers on the same IRQ line
1363 err
= fsldma_request_irqs(fdev
);
1365 dev_err(fdev
->dev
, "unable to request IRQs\n");
1369 dma_async_device_register(&fdev
->common
);
1373 irq_dispose_mapping(fdev
->irq
);
1379 static int fsldma_of_remove(struct of_device
*op
)
1381 struct fsldma_device
*fdev
;
1384 fdev
= dev_get_drvdata(&op
->dev
);
1385 dma_async_device_unregister(&fdev
->common
);
1387 fsldma_free_irqs(fdev
);
1389 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1391 fsl_dma_chan_remove(fdev
->chan
[i
]);
1394 iounmap(fdev
->regs
);
1395 dev_set_drvdata(&op
->dev
, NULL
);
1401 static const struct of_device_id fsldma_of_ids
[] = {
1402 { .compatible
= "fsl,eloplus-dma", },
1403 { .compatible
= "fsl,elo-dma", },
1407 static struct of_platform_driver fsldma_of_driver
= {
1408 .name
= "fsl-elo-dma",
1409 .match_table
= fsldma_of_ids
,
1410 .probe
= fsldma_of_probe
,
1411 .remove
= fsldma_of_remove
,
1414 /*----------------------------------------------------------------------------*/
1415 /* Module Init / Exit */
1416 /*----------------------------------------------------------------------------*/
1418 static __init
int fsldma_init(void)
1422 pr_info("Freescale Elo / Elo Plus DMA driver\n");
1424 ret
= of_register_platform_driver(&fsldma_of_driver
);
1426 pr_err("fsldma: failed to register platform driver\n");
1431 static void __exit
fsldma_exit(void)
1433 of_unregister_platform_driver(&fsldma_of_driver
);
1436 subsys_initcall(fsldma_init
);
1437 module_exit(fsldma_exit
);
1439 MODULE_DESCRIPTION("Freescale Elo / Elo Plus DMA driver");
1440 MODULE_LICENSE("GPL");