2 * Freescale MPC85xx, MPC83xx DMA Engine support
4 * Copyright (C) 2007-2010 Freescale Semiconductor, Inc. All rights reserved.
7 * Zhang Wei <wei.zhang@freescale.com>, Jul 2007
8 * Ebony Zhu <ebony.zhu@freescale.com>, May 2007
11 * DMA engine driver for Freescale MPC8540 DMA controller, which is
12 * also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc.
13 * The support for MPC8349 DMA controller is also added.
15 * This driver instructs the DMA controller to issue the PCI Read Multiple
16 * command for PCI read operations, instead of using the default PCI Read Line
17 * command. Please be aware that this setting may result in read pre-fetching
20 * This is free software; you can redistribute it and/or modify
21 * it under the terms of the GNU General Public License as published by
22 * the Free Software Foundation; either version 2 of the License, or
23 * (at your option) any later version.
27 #include <linux/init.h>
28 #include <linux/module.h>
29 #include <linux/pci.h>
30 #include <linux/slab.h>
31 #include <linux/interrupt.h>
32 #include <linux/dmaengine.h>
33 #include <linux/delay.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/dmapool.h>
36 #include <linux/of_address.h>
37 #include <linux/of_irq.h>
38 #include <linux/of_platform.h>
39 #include <linux/fsldma.h>
40 #include "dmaengine.h"
43 #define chan_dbg(chan, fmt, arg...) \
44 dev_dbg(chan->dev, "%s: " fmt, chan->name, ##arg)
45 #define chan_err(chan, fmt, arg...) \
46 dev_err(chan->dev, "%s: " fmt, chan->name, ##arg)
48 static const char msg_ld_oom
[] = "No free memory for link descriptor";
54 static void set_sr(struct fsldma_chan
*chan
, u32 val
)
56 DMA_OUT(chan
, &chan
->regs
->sr
, val
, 32);
59 static u32
get_sr(struct fsldma_chan
*chan
)
61 return DMA_IN(chan
, &chan
->regs
->sr
, 32);
64 static void set_mr(struct fsldma_chan
*chan
, u32 val
)
66 DMA_OUT(chan
, &chan
->regs
->mr
, val
, 32);
69 static u32
get_mr(struct fsldma_chan
*chan
)
71 return DMA_IN(chan
, &chan
->regs
->mr
, 32);
74 static void set_cdar(struct fsldma_chan
*chan
, dma_addr_t addr
)
76 DMA_OUT(chan
, &chan
->regs
->cdar
, addr
| FSL_DMA_SNEN
, 64);
79 static dma_addr_t
get_cdar(struct fsldma_chan
*chan
)
81 return DMA_IN(chan
, &chan
->regs
->cdar
, 64) & ~FSL_DMA_SNEN
;
84 static void set_bcr(struct fsldma_chan
*chan
, u32 val
)
86 DMA_OUT(chan
, &chan
->regs
->bcr
, val
, 32);
89 static u32
get_bcr(struct fsldma_chan
*chan
)
91 return DMA_IN(chan
, &chan
->regs
->bcr
, 32);
98 static void set_desc_cnt(struct fsldma_chan
*chan
,
99 struct fsl_dma_ld_hw
*hw
, u32 count
)
101 hw
->count
= CPU_TO_DMA(chan
, count
, 32);
104 static void set_desc_src(struct fsldma_chan
*chan
,
105 struct fsl_dma_ld_hw
*hw
, dma_addr_t src
)
109 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_85XX
)
110 ? ((u64
)FSL_DMA_SATR_SREADTYPE_SNOOP_READ
<< 32) : 0;
111 hw
->src_addr
= CPU_TO_DMA(chan
, snoop_bits
| src
, 64);
114 static void set_desc_dst(struct fsldma_chan
*chan
,
115 struct fsl_dma_ld_hw
*hw
, dma_addr_t dst
)
119 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_85XX
)
120 ? ((u64
)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE
<< 32) : 0;
121 hw
->dst_addr
= CPU_TO_DMA(chan
, snoop_bits
| dst
, 64);
124 static void set_desc_next(struct fsldma_chan
*chan
,
125 struct fsl_dma_ld_hw
*hw
, dma_addr_t next
)
129 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_83XX
)
131 hw
->next_ln_addr
= CPU_TO_DMA(chan
, snoop_bits
| next
, 64);
134 static void set_ld_eol(struct fsldma_chan
*chan
, struct fsl_desc_sw
*desc
)
138 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_83XX
)
141 desc
->hw
.next_ln_addr
= CPU_TO_DMA(chan
,
142 DMA_TO_CPU(chan
, desc
->hw
.next_ln_addr
, 64) | FSL_DMA_EOL
147 * DMA Engine Hardware Control Helpers
150 static void dma_init(struct fsldma_chan
*chan
)
152 /* Reset the channel */
155 switch (chan
->feature
& FSL_DMA_IP_MASK
) {
156 case FSL_DMA_IP_85XX
:
157 /* Set the channel to below modes:
158 * EIE - Error interrupt enable
159 * EOLNIE - End of links interrupt enable
160 * BWC - Bandwidth sharing among channels
162 set_mr(chan
, FSL_DMA_MR_BWC
| FSL_DMA_MR_EIE
163 | FSL_DMA_MR_EOLNIE
);
165 case FSL_DMA_IP_83XX
:
166 /* Set the channel to below modes:
167 * EOTIE - End-of-transfer interrupt enable
168 * PRC_RM - PCI read multiple
170 set_mr(chan
, FSL_DMA_MR_EOTIE
| FSL_DMA_MR_PRC_RM
);
175 static int dma_is_idle(struct fsldma_chan
*chan
)
177 u32 sr
= get_sr(chan
);
178 return (!(sr
& FSL_DMA_SR_CB
)) || (sr
& FSL_DMA_SR_CH
);
182 * Start the DMA controller
185 * - the CDAR register must point to the start descriptor
186 * - the MRn[CS] bit must be cleared
188 static void dma_start(struct fsldma_chan
*chan
)
194 if (chan
->feature
& FSL_DMA_CHAN_PAUSE_EXT
) {
196 mode
|= FSL_DMA_MR_EMP_EN
;
198 mode
&= ~FSL_DMA_MR_EMP_EN
;
201 if (chan
->feature
& FSL_DMA_CHAN_START_EXT
) {
202 mode
|= FSL_DMA_MR_EMS_EN
;
204 mode
&= ~FSL_DMA_MR_EMS_EN
;
205 mode
|= FSL_DMA_MR_CS
;
211 static void dma_halt(struct fsldma_chan
*chan
)
216 /* read the mode register */
220 * The 85xx controller supports channel abort, which will stop
221 * the current transfer. On 83xx, this bit is the transfer error
222 * mask bit, which should not be changed.
224 if ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_85XX
) {
225 mode
|= FSL_DMA_MR_CA
;
228 mode
&= ~FSL_DMA_MR_CA
;
231 /* stop the DMA controller */
232 mode
&= ~(FSL_DMA_MR_CS
| FSL_DMA_MR_EMS_EN
);
235 /* wait for the DMA controller to become idle */
236 for (i
= 0; i
< 100; i
++) {
237 if (dma_is_idle(chan
))
243 if (!dma_is_idle(chan
))
244 chan_err(chan
, "DMA halt timeout!\n");
248 * fsl_chan_set_src_loop_size - Set source address hold transfer size
249 * @chan : Freescale DMA channel
250 * @size : Address loop size, 0 for disable loop
252 * The set source address hold transfer size. The source
253 * address hold or loop transfer size is when the DMA transfer
254 * data from source address (SA), if the loop size is 4, the DMA will
255 * read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA,
256 * SA + 1 ... and so on.
258 static void fsl_chan_set_src_loop_size(struct fsldma_chan
*chan
, int size
)
266 mode
&= ~FSL_DMA_MR_SAHE
;
272 mode
|= FSL_DMA_MR_SAHE
| (__ilog2(size
) << 14);
280 * fsl_chan_set_dst_loop_size - Set destination address hold transfer size
281 * @chan : Freescale DMA channel
282 * @size : Address loop size, 0 for disable loop
284 * The set destination address hold transfer size. The destination
285 * address hold or loop transfer size is when the DMA transfer
286 * data to destination address (TA), if the loop size is 4, the DMA will
287 * write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA,
288 * TA + 1 ... and so on.
290 static void fsl_chan_set_dst_loop_size(struct fsldma_chan
*chan
, int size
)
298 mode
&= ~FSL_DMA_MR_DAHE
;
304 mode
|= FSL_DMA_MR_DAHE
| (__ilog2(size
) << 16);
312 * fsl_chan_set_request_count - Set DMA Request Count for external control
313 * @chan : Freescale DMA channel
314 * @size : Number of bytes to transfer in a single request
316 * The Freescale DMA channel can be controlled by the external signal DREQ#.
317 * The DMA request count is how many bytes are allowed to transfer before
318 * pausing the channel, after which a new assertion of DREQ# resumes channel
321 * A size of 0 disables external pause control. The maximum size is 1024.
323 static void fsl_chan_set_request_count(struct fsldma_chan
*chan
, int size
)
330 mode
|= (__ilog2(size
) << 24) & 0x0f000000;
336 * fsl_chan_toggle_ext_pause - Toggle channel external pause status
337 * @chan : Freescale DMA channel
338 * @enable : 0 is disabled, 1 is enabled.
340 * The Freescale DMA channel can be controlled by the external signal DREQ#.
341 * The DMA Request Count feature should be used in addition to this feature
342 * to set the number of bytes to transfer before pausing the channel.
344 static void fsl_chan_toggle_ext_pause(struct fsldma_chan
*chan
, int enable
)
347 chan
->feature
|= FSL_DMA_CHAN_PAUSE_EXT
;
349 chan
->feature
&= ~FSL_DMA_CHAN_PAUSE_EXT
;
353 * fsl_chan_toggle_ext_start - Toggle channel external start status
354 * @chan : Freescale DMA channel
355 * @enable : 0 is disabled, 1 is enabled.
357 * If enable the external start, the channel can be started by an
358 * external DMA start pin. So the dma_start() does not start the
359 * transfer immediately. The DMA channel will wait for the
360 * control pin asserted.
362 static void fsl_chan_toggle_ext_start(struct fsldma_chan
*chan
, int enable
)
365 chan
->feature
|= FSL_DMA_CHAN_START_EXT
;
367 chan
->feature
&= ~FSL_DMA_CHAN_START_EXT
;
370 int fsl_dma_external_start(struct dma_chan
*dchan
, int enable
)
372 struct fsldma_chan
*chan
;
377 chan
= to_fsl_chan(dchan
);
379 fsl_chan_toggle_ext_start(chan
, enable
);
382 EXPORT_SYMBOL_GPL(fsl_dma_external_start
);
384 static void append_ld_queue(struct fsldma_chan
*chan
, struct fsl_desc_sw
*desc
)
386 struct fsl_desc_sw
*tail
= to_fsl_desc(chan
->ld_pending
.prev
);
388 if (list_empty(&chan
->ld_pending
))
392 * Add the hardware descriptor to the chain of hardware descriptors
393 * that already exists in memory.
395 * This will un-set the EOL bit of the existing transaction, and the
396 * last link in this transaction will become the EOL descriptor.
398 set_desc_next(chan
, &tail
->hw
, desc
->async_tx
.phys
);
401 * Add the software descriptor and all children to the list
402 * of pending transactions
405 list_splice_tail_init(&desc
->tx_list
, &chan
->ld_pending
);
408 static dma_cookie_t
fsl_dma_tx_submit(struct dma_async_tx_descriptor
*tx
)
410 struct fsldma_chan
*chan
= to_fsl_chan(tx
->chan
);
411 struct fsl_desc_sw
*desc
= tx_to_fsl_desc(tx
);
412 struct fsl_desc_sw
*child
;
413 dma_cookie_t cookie
= -EINVAL
;
415 spin_lock_bh(&chan
->desc_lock
);
418 if (unlikely(chan
->pm_state
!= RUNNING
)) {
419 chan_dbg(chan
, "cannot submit due to suspend\n");
420 spin_unlock_bh(&chan
->desc_lock
);
426 * assign cookies to all of the software descriptors
427 * that make up this transaction
429 list_for_each_entry(child
, &desc
->tx_list
, node
) {
430 cookie
= dma_cookie_assign(&child
->async_tx
);
433 /* put this transaction onto the tail of the pending queue */
434 append_ld_queue(chan
, desc
);
436 spin_unlock_bh(&chan
->desc_lock
);
442 * fsl_dma_free_descriptor - Free descriptor from channel's DMA pool.
443 * @chan : Freescale DMA channel
444 * @desc: descriptor to be freed
446 static void fsl_dma_free_descriptor(struct fsldma_chan
*chan
,
447 struct fsl_desc_sw
*desc
)
449 list_del(&desc
->node
);
450 chan_dbg(chan
, "LD %p free\n", desc
);
451 dma_pool_free(chan
->desc_pool
, desc
, desc
->async_tx
.phys
);
455 * fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool.
456 * @chan : Freescale DMA channel
458 * Return - The descriptor allocated. NULL for failed.
460 static struct fsl_desc_sw
*fsl_dma_alloc_descriptor(struct fsldma_chan
*chan
)
462 struct fsl_desc_sw
*desc
;
465 desc
= dma_pool_alloc(chan
->desc_pool
, GFP_ATOMIC
, &pdesc
);
467 chan_dbg(chan
, "out of memory for link descriptor\n");
471 memset(desc
, 0, sizeof(*desc
));
472 INIT_LIST_HEAD(&desc
->tx_list
);
473 dma_async_tx_descriptor_init(&desc
->async_tx
, &chan
->common
);
474 desc
->async_tx
.tx_submit
= fsl_dma_tx_submit
;
475 desc
->async_tx
.phys
= pdesc
;
477 chan_dbg(chan
, "LD %p allocated\n", desc
);
483 * fsldma_clean_completed_descriptor - free all descriptors which
484 * has been completed and acked
485 * @chan: Freescale DMA channel
487 * This function is used on all completed and acked descriptors.
488 * All descriptors should only be freed in this function.
490 static void fsldma_clean_completed_descriptor(struct fsldma_chan
*chan
)
492 struct fsl_desc_sw
*desc
, *_desc
;
494 /* Run the callback for each descriptor, in order */
495 list_for_each_entry_safe(desc
, _desc
, &chan
->ld_completed
, node
)
496 if (async_tx_test_ack(&desc
->async_tx
))
497 fsl_dma_free_descriptor(chan
, desc
);
501 * fsldma_run_tx_complete_actions - cleanup a single link descriptor
502 * @chan: Freescale DMA channel
503 * @desc: descriptor to cleanup and free
504 * @cookie: Freescale DMA transaction identifier
506 * This function is used on a descriptor which has been executed by the DMA
507 * controller. It will run any callbacks, submit any dependencies.
509 static dma_cookie_t
fsldma_run_tx_complete_actions(struct fsldma_chan
*chan
,
510 struct fsl_desc_sw
*desc
, dma_cookie_t cookie
)
512 struct dma_async_tx_descriptor
*txd
= &desc
->async_tx
;
513 dma_cookie_t ret
= cookie
;
515 BUG_ON(txd
->cookie
< 0);
517 if (txd
->cookie
> 0) {
520 /* Run the link descriptor callback function */
522 chan_dbg(chan
, "LD %p callback\n", desc
);
523 txd
->callback(txd
->callback_param
);
527 /* Run any dependencies */
528 dma_run_dependencies(txd
);
534 * fsldma_clean_running_descriptor - move the completed descriptor from
535 * ld_running to ld_completed
536 * @chan: Freescale DMA channel
537 * @desc: the descriptor which is completed
539 * Free the descriptor directly if acked by async_tx api, or move it to
540 * queue ld_completed.
542 static void fsldma_clean_running_descriptor(struct fsldma_chan
*chan
,
543 struct fsl_desc_sw
*desc
)
545 /* Remove from the list of transactions */
546 list_del(&desc
->node
);
549 * the client is allowed to attach dependent operations
552 if (!async_tx_test_ack(&desc
->async_tx
)) {
554 * Move this descriptor to the list of descriptors which is
555 * completed, but still awaiting the 'ack' bit to be set.
557 list_add_tail(&desc
->node
, &chan
->ld_completed
);
561 dma_pool_free(chan
->desc_pool
, desc
, desc
->async_tx
.phys
);
565 * fsl_chan_xfer_ld_queue - transfer any pending transactions
566 * @chan : Freescale DMA channel
568 * HARDWARE STATE: idle
569 * LOCKING: must hold chan->desc_lock
571 static void fsl_chan_xfer_ld_queue(struct fsldma_chan
*chan
)
573 struct fsl_desc_sw
*desc
;
576 * If the list of pending descriptors is empty, then we
577 * don't need to do any work at all
579 if (list_empty(&chan
->ld_pending
)) {
580 chan_dbg(chan
, "no pending LDs\n");
585 * The DMA controller is not idle, which means that the interrupt
586 * handler will start any queued transactions when it runs after
587 * this transaction finishes
590 chan_dbg(chan
, "DMA controller still busy\n");
595 * If there are some link descriptors which have not been
596 * transferred, we need to start the controller
600 * Move all elements from the queue of pending transactions
601 * onto the list of running transactions
603 chan_dbg(chan
, "idle, starting controller\n");
604 desc
= list_first_entry(&chan
->ld_pending
, struct fsl_desc_sw
, node
);
605 list_splice_tail_init(&chan
->ld_pending
, &chan
->ld_running
);
608 * The 85xx DMA controller doesn't clear the channel start bit
609 * automatically at the end of a transfer. Therefore we must clear
610 * it in software before starting the transfer.
612 if ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_85XX
) {
616 mode
&= ~FSL_DMA_MR_CS
;
621 * Program the descriptor's address into the DMA controller,
622 * then start the DMA transaction
624 set_cdar(chan
, desc
->async_tx
.phys
);
632 * fsldma_cleanup_descriptors - cleanup link descriptors which are completed
633 * and move them to ld_completed to free until flag 'ack' is set
634 * @chan: Freescale DMA channel
636 * This function is used on descriptors which have been executed by the DMA
637 * controller. It will run any callbacks, submit any dependencies, then
638 * free these descriptors if flag 'ack' is set.
640 static void fsldma_cleanup_descriptors(struct fsldma_chan
*chan
)
642 struct fsl_desc_sw
*desc
, *_desc
;
643 dma_cookie_t cookie
= 0;
644 dma_addr_t curr_phys
= get_cdar(chan
);
645 int seen_current
= 0;
647 fsldma_clean_completed_descriptor(chan
);
649 /* Run the callback for each descriptor, in order */
650 list_for_each_entry_safe(desc
, _desc
, &chan
->ld_running
, node
) {
652 * do not advance past the current descriptor loaded into the
653 * hardware channel, subsequent descriptors are either in
654 * process or have not been submitted
660 * stop the search if we reach the current descriptor and the
663 if (desc
->async_tx
.phys
== curr_phys
) {
665 if (!dma_is_idle(chan
))
669 cookie
= fsldma_run_tx_complete_actions(chan
, desc
, cookie
);
671 fsldma_clean_running_descriptor(chan
, desc
);
675 * Start any pending transactions automatically
677 * In the ideal case, we keep the DMA controller busy while we go
678 * ahead and free the descriptors below.
680 fsl_chan_xfer_ld_queue(chan
);
683 chan
->common
.completed_cookie
= cookie
;
687 * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel.
688 * @chan : Freescale DMA channel
690 * This function will create a dma pool for descriptor allocation.
692 * Return - The number of descriptors allocated.
694 static int fsl_dma_alloc_chan_resources(struct dma_chan
*dchan
)
696 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
698 /* Has this channel already been allocated? */
703 * We need the descriptor to be aligned to 32bytes
704 * for meeting FSL DMA specification requirement.
706 chan
->desc_pool
= dma_pool_create(chan
->name
, chan
->dev
,
707 sizeof(struct fsl_desc_sw
),
708 __alignof__(struct fsl_desc_sw
), 0);
709 if (!chan
->desc_pool
) {
710 chan_err(chan
, "unable to allocate descriptor pool\n");
714 /* there is at least one descriptor free to be allocated */
719 * fsldma_free_desc_list - Free all descriptors in a queue
720 * @chan: Freescae DMA channel
721 * @list: the list to free
723 * LOCKING: must hold chan->desc_lock
725 static void fsldma_free_desc_list(struct fsldma_chan
*chan
,
726 struct list_head
*list
)
728 struct fsl_desc_sw
*desc
, *_desc
;
730 list_for_each_entry_safe(desc
, _desc
, list
, node
)
731 fsl_dma_free_descriptor(chan
, desc
);
734 static void fsldma_free_desc_list_reverse(struct fsldma_chan
*chan
,
735 struct list_head
*list
)
737 struct fsl_desc_sw
*desc
, *_desc
;
739 list_for_each_entry_safe_reverse(desc
, _desc
, list
, node
)
740 fsl_dma_free_descriptor(chan
, desc
);
744 * fsl_dma_free_chan_resources - Free all resources of the channel.
745 * @chan : Freescale DMA channel
747 static void fsl_dma_free_chan_resources(struct dma_chan
*dchan
)
749 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
751 chan_dbg(chan
, "free all channel resources\n");
752 spin_lock_bh(&chan
->desc_lock
);
753 fsldma_cleanup_descriptors(chan
);
754 fsldma_free_desc_list(chan
, &chan
->ld_pending
);
755 fsldma_free_desc_list(chan
, &chan
->ld_running
);
756 fsldma_free_desc_list(chan
, &chan
->ld_completed
);
757 spin_unlock_bh(&chan
->desc_lock
);
759 dma_pool_destroy(chan
->desc_pool
);
760 chan
->desc_pool
= NULL
;
763 static struct dma_async_tx_descriptor
*
764 fsl_dma_prep_memcpy(struct dma_chan
*dchan
,
765 dma_addr_t dma_dst
, dma_addr_t dma_src
,
766 size_t len
, unsigned long flags
)
768 struct fsldma_chan
*chan
;
769 struct fsl_desc_sw
*first
= NULL
, *prev
= NULL
, *new;
778 chan
= to_fsl_chan(dchan
);
782 /* Allocate the link descriptor from DMA pool */
783 new = fsl_dma_alloc_descriptor(chan
);
785 chan_err(chan
, "%s\n", msg_ld_oom
);
789 copy
= min(len
, (size_t)FSL_DMA_BCR_MAX_CNT
);
791 set_desc_cnt(chan
, &new->hw
, copy
);
792 set_desc_src(chan
, &new->hw
, dma_src
);
793 set_desc_dst(chan
, &new->hw
, dma_dst
);
798 set_desc_next(chan
, &prev
->hw
, new->async_tx
.phys
);
800 new->async_tx
.cookie
= 0;
801 async_tx_ack(&new->async_tx
);
808 /* Insert the link descriptor to the LD ring */
809 list_add_tail(&new->node
, &first
->tx_list
);
812 new->async_tx
.flags
= flags
; /* client is in control of this ack */
813 new->async_tx
.cookie
= -EBUSY
;
815 /* Set End-of-link to the last link descriptor of new list */
816 set_ld_eol(chan
, new);
818 return &first
->async_tx
;
824 fsldma_free_desc_list_reverse(chan
, &first
->tx_list
);
828 static struct dma_async_tx_descriptor
*fsl_dma_prep_sg(struct dma_chan
*dchan
,
829 struct scatterlist
*dst_sg
, unsigned int dst_nents
,
830 struct scatterlist
*src_sg
, unsigned int src_nents
,
833 struct fsl_desc_sw
*first
= NULL
, *prev
= NULL
, *new = NULL
;
834 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
835 size_t dst_avail
, src_avail
;
839 /* basic sanity checks */
840 if (dst_nents
== 0 || src_nents
== 0)
843 if (dst_sg
== NULL
|| src_sg
== NULL
)
847 * TODO: should we check that both scatterlists have the same
848 * TODO: number of bytes in total? Is that really an error?
851 /* get prepared for the loop */
852 dst_avail
= sg_dma_len(dst_sg
);
853 src_avail
= sg_dma_len(src_sg
);
855 /* run until we are out of scatterlist entries */
858 /* create the largest transaction possible */
859 len
= min_t(size_t, src_avail
, dst_avail
);
860 len
= min_t(size_t, len
, FSL_DMA_BCR_MAX_CNT
);
864 dst
= sg_dma_address(dst_sg
) + sg_dma_len(dst_sg
) - dst_avail
;
865 src
= sg_dma_address(src_sg
) + sg_dma_len(src_sg
) - src_avail
;
867 /* allocate and populate the descriptor */
868 new = fsl_dma_alloc_descriptor(chan
);
870 chan_err(chan
, "%s\n", msg_ld_oom
);
874 set_desc_cnt(chan
, &new->hw
, len
);
875 set_desc_src(chan
, &new->hw
, src
);
876 set_desc_dst(chan
, &new->hw
, dst
);
881 set_desc_next(chan
, &prev
->hw
, new->async_tx
.phys
);
883 new->async_tx
.cookie
= 0;
884 async_tx_ack(&new->async_tx
);
887 /* Insert the link descriptor to the LD ring */
888 list_add_tail(&new->node
, &first
->tx_list
);
890 /* update metadata */
895 /* fetch the next dst scatterlist entry */
896 if (dst_avail
== 0) {
898 /* no more entries: we're done */
902 /* fetch the next entry: if there are no more: done */
903 dst_sg
= sg_next(dst_sg
);
908 dst_avail
= sg_dma_len(dst_sg
);
911 /* fetch the next src scatterlist entry */
912 if (src_avail
== 0) {
914 /* no more entries: we're done */
918 /* fetch the next entry: if there are no more: done */
919 src_sg
= sg_next(src_sg
);
924 src_avail
= sg_dma_len(src_sg
);
928 new->async_tx
.flags
= flags
; /* client is in control of this ack */
929 new->async_tx
.cookie
= -EBUSY
;
931 /* Set End-of-link to the last link descriptor of new list */
932 set_ld_eol(chan
, new);
934 return &first
->async_tx
;
940 fsldma_free_desc_list_reverse(chan
, &first
->tx_list
);
945 * fsl_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
947 * @sgl: scatterlist to transfer to/from
948 * @sg_len: number of entries in @scatterlist
949 * @direction: DMA direction
950 * @flags: DMAEngine flags
951 * @context: transaction context (ignored)
953 * Prepare a set of descriptors for a DMA_SLAVE transaction. Following the
954 * DMA_SLAVE API, this gets the device-specific information from the
955 * chan->private variable.
957 static struct dma_async_tx_descriptor
*fsl_dma_prep_slave_sg(
958 struct dma_chan
*dchan
, struct scatterlist
*sgl
, unsigned int sg_len
,
959 enum dma_transfer_direction direction
, unsigned long flags
,
963 * This operation is not supported on the Freescale DMA controller
965 * However, we need to provide the function pointer to allow the
966 * device_control() method to work.
971 static int fsl_dma_device_control(struct dma_chan
*dchan
,
972 enum dma_ctrl_cmd cmd
, unsigned long arg
)
974 struct dma_slave_config
*config
;
975 struct fsldma_chan
*chan
;
981 chan
= to_fsl_chan(dchan
);
984 case DMA_TERMINATE_ALL
:
985 spin_lock_bh(&chan
->desc_lock
);
987 /* Halt the DMA engine */
990 /* Remove and free all of the descriptors in the LD queue */
991 fsldma_free_desc_list(chan
, &chan
->ld_pending
);
992 fsldma_free_desc_list(chan
, &chan
->ld_running
);
993 fsldma_free_desc_list(chan
, &chan
->ld_completed
);
996 spin_unlock_bh(&chan
->desc_lock
);
999 case DMA_SLAVE_CONFIG
:
1000 config
= (struct dma_slave_config
*)arg
;
1002 /* make sure the channel supports setting burst size */
1003 if (!chan
->set_request_count
)
1006 /* we set the controller burst size depending on direction */
1007 if (config
->direction
== DMA_MEM_TO_DEV
)
1008 size
= config
->dst_addr_width
* config
->dst_maxburst
;
1010 size
= config
->src_addr_width
* config
->src_maxburst
;
1012 chan
->set_request_count(chan
, size
);
1023 * fsl_dma_memcpy_issue_pending - Issue the DMA start command
1024 * @chan : Freescale DMA channel
1026 static void fsl_dma_memcpy_issue_pending(struct dma_chan
*dchan
)
1028 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
1030 spin_lock_bh(&chan
->desc_lock
);
1031 fsl_chan_xfer_ld_queue(chan
);
1032 spin_unlock_bh(&chan
->desc_lock
);
1036 * fsl_tx_status - Determine the DMA status
1037 * @chan : Freescale DMA channel
1039 static enum dma_status
fsl_tx_status(struct dma_chan
*dchan
,
1040 dma_cookie_t cookie
,
1041 struct dma_tx_state
*txstate
)
1043 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
1044 enum dma_status ret
;
1046 ret
= dma_cookie_status(dchan
, cookie
, txstate
);
1047 if (ret
== DMA_COMPLETE
)
1050 spin_lock_bh(&chan
->desc_lock
);
1051 fsldma_cleanup_descriptors(chan
);
1052 spin_unlock_bh(&chan
->desc_lock
);
1054 return dma_cookie_status(dchan
, cookie
, txstate
);
1057 /*----------------------------------------------------------------------------*/
1058 /* Interrupt Handling */
1059 /*----------------------------------------------------------------------------*/
1061 static irqreturn_t
fsldma_chan_irq(int irq
, void *data
)
1063 struct fsldma_chan
*chan
= data
;
1066 /* save and clear the status register */
1067 stat
= get_sr(chan
);
1069 chan_dbg(chan
, "irq: stat = 0x%x\n", stat
);
1071 /* check that this was really our device */
1072 stat
&= ~(FSL_DMA_SR_CB
| FSL_DMA_SR_CH
);
1076 if (stat
& FSL_DMA_SR_TE
)
1077 chan_err(chan
, "Transfer Error!\n");
1081 * The DMA_INTERRUPT async_tx is a NULL transfer, which will
1082 * trigger a PE interrupt.
1084 if (stat
& FSL_DMA_SR_PE
) {
1085 chan_dbg(chan
, "irq: Programming Error INT\n");
1086 stat
&= ~FSL_DMA_SR_PE
;
1087 if (get_bcr(chan
) != 0)
1088 chan_err(chan
, "Programming Error!\n");
1092 * For MPC8349, EOCDI event need to update cookie
1093 * and start the next transfer if it exist.
1095 if (stat
& FSL_DMA_SR_EOCDI
) {
1096 chan_dbg(chan
, "irq: End-of-Chain link INT\n");
1097 stat
&= ~FSL_DMA_SR_EOCDI
;
1101 * If it current transfer is the end-of-transfer,
1102 * we should clear the Channel Start bit for
1103 * prepare next transfer.
1105 if (stat
& FSL_DMA_SR_EOLNI
) {
1106 chan_dbg(chan
, "irq: End-of-link INT\n");
1107 stat
&= ~FSL_DMA_SR_EOLNI
;
1110 /* check that the DMA controller is really idle */
1111 if (!dma_is_idle(chan
))
1112 chan_err(chan
, "irq: controller not idle!\n");
1114 /* check that we handled all of the bits */
1116 chan_err(chan
, "irq: unhandled sr 0x%08x\n", stat
);
1119 * Schedule the tasklet to handle all cleanup of the current
1120 * transaction. It will start a new transaction if there is
1123 tasklet_schedule(&chan
->tasklet
);
1124 chan_dbg(chan
, "irq: Exit\n");
1128 static void dma_do_tasklet(unsigned long data
)
1130 struct fsldma_chan
*chan
= (struct fsldma_chan
*)data
;
1132 chan_dbg(chan
, "tasklet entry\n");
1134 spin_lock_bh(&chan
->desc_lock
);
1136 /* the hardware is now idle and ready for more */
1139 /* Run all cleanup for descriptors which have been completed */
1140 fsldma_cleanup_descriptors(chan
);
1142 spin_unlock_bh(&chan
->desc_lock
);
1144 chan_dbg(chan
, "tasklet exit\n");
1147 static irqreturn_t
fsldma_ctrl_irq(int irq
, void *data
)
1149 struct fsldma_device
*fdev
= data
;
1150 struct fsldma_chan
*chan
;
1151 unsigned int handled
= 0;
1155 gsr
= (fdev
->feature
& FSL_DMA_BIG_ENDIAN
) ? in_be32(fdev
->regs
)
1156 : in_le32(fdev
->regs
);
1158 dev_dbg(fdev
->dev
, "IRQ: gsr 0x%.8x\n", gsr
);
1160 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1161 chan
= fdev
->chan
[i
];
1166 dev_dbg(fdev
->dev
, "IRQ: chan %d\n", chan
->id
);
1167 fsldma_chan_irq(irq
, chan
);
1175 return IRQ_RETVAL(handled
);
1178 static void fsldma_free_irqs(struct fsldma_device
*fdev
)
1180 struct fsldma_chan
*chan
;
1183 if (fdev
->irq
!= NO_IRQ
) {
1184 dev_dbg(fdev
->dev
, "free per-controller IRQ\n");
1185 free_irq(fdev
->irq
, fdev
);
1189 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1190 chan
= fdev
->chan
[i
];
1191 if (chan
&& chan
->irq
!= NO_IRQ
) {
1192 chan_dbg(chan
, "free per-channel IRQ\n");
1193 free_irq(chan
->irq
, chan
);
1198 static int fsldma_request_irqs(struct fsldma_device
*fdev
)
1200 struct fsldma_chan
*chan
;
1204 /* if we have a per-controller IRQ, use that */
1205 if (fdev
->irq
!= NO_IRQ
) {
1206 dev_dbg(fdev
->dev
, "request per-controller IRQ\n");
1207 ret
= request_irq(fdev
->irq
, fsldma_ctrl_irq
, IRQF_SHARED
,
1208 "fsldma-controller", fdev
);
1212 /* no per-controller IRQ, use the per-channel IRQs */
1213 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1214 chan
= fdev
->chan
[i
];
1218 if (chan
->irq
== NO_IRQ
) {
1219 chan_err(chan
, "interrupts property missing in device tree\n");
1224 chan_dbg(chan
, "request per-channel IRQ\n");
1225 ret
= request_irq(chan
->irq
, fsldma_chan_irq
, IRQF_SHARED
,
1226 "fsldma-chan", chan
);
1228 chan_err(chan
, "unable to request per-channel IRQ\n");
1236 for (/* none */; i
>= 0; i
--) {
1237 chan
= fdev
->chan
[i
];
1241 if (chan
->irq
== NO_IRQ
)
1244 free_irq(chan
->irq
, chan
);
1250 /*----------------------------------------------------------------------------*/
1251 /* OpenFirmware Subsystem */
1252 /*----------------------------------------------------------------------------*/
1254 static int fsl_dma_chan_probe(struct fsldma_device
*fdev
,
1255 struct device_node
*node
, u32 feature
, const char *compatible
)
1257 struct fsldma_chan
*chan
;
1258 struct resource res
;
1262 chan
= kzalloc(sizeof(*chan
), GFP_KERNEL
);
1264 dev_err(fdev
->dev
, "no free memory for DMA channels!\n");
1269 /* ioremap registers for use */
1270 chan
->regs
= of_iomap(node
, 0);
1272 dev_err(fdev
->dev
, "unable to ioremap registers\n");
1277 err
= of_address_to_resource(node
, 0, &res
);
1279 dev_err(fdev
->dev
, "unable to find 'reg' property\n");
1280 goto out_iounmap_regs
;
1283 chan
->feature
= feature
;
1285 fdev
->feature
= chan
->feature
;
1288 * If the DMA device's feature is different than the feature
1289 * of its channels, report the bug
1291 WARN_ON(fdev
->feature
!= chan
->feature
);
1293 chan
->dev
= fdev
->dev
;
1294 chan
->id
= (res
.start
& 0xfff) < 0x300 ?
1295 ((res
.start
- 0x100) & 0xfff) >> 7 :
1296 ((res
.start
- 0x200) & 0xfff) >> 7;
1297 if (chan
->id
>= FSL_DMA_MAX_CHANS_PER_DEVICE
) {
1298 dev_err(fdev
->dev
, "too many channels for device\n");
1300 goto out_iounmap_regs
;
1303 fdev
->chan
[chan
->id
] = chan
;
1304 tasklet_init(&chan
->tasklet
, dma_do_tasklet
, (unsigned long)chan
);
1305 snprintf(chan
->name
, sizeof(chan
->name
), "chan%d", chan
->id
);
1307 /* Initialize the channel */
1310 /* Clear cdar registers */
1313 switch (chan
->feature
& FSL_DMA_IP_MASK
) {
1314 case FSL_DMA_IP_85XX
:
1315 chan
->toggle_ext_pause
= fsl_chan_toggle_ext_pause
;
1316 case FSL_DMA_IP_83XX
:
1317 chan
->toggle_ext_start
= fsl_chan_toggle_ext_start
;
1318 chan
->set_src_loop_size
= fsl_chan_set_src_loop_size
;
1319 chan
->set_dst_loop_size
= fsl_chan_set_dst_loop_size
;
1320 chan
->set_request_count
= fsl_chan_set_request_count
;
1323 spin_lock_init(&chan
->desc_lock
);
1324 INIT_LIST_HEAD(&chan
->ld_pending
);
1325 INIT_LIST_HEAD(&chan
->ld_running
);
1326 INIT_LIST_HEAD(&chan
->ld_completed
);
1329 chan
->pm_state
= RUNNING
;
1332 chan
->common
.device
= &fdev
->common
;
1333 dma_cookie_init(&chan
->common
);
1335 /* find the IRQ line, if it exists in the device tree */
1336 chan
->irq
= irq_of_parse_and_map(node
, 0);
1338 /* Add the channel to DMA device channel list */
1339 list_add_tail(&chan
->common
.device_node
, &fdev
->common
.channels
);
1340 fdev
->common
.chancnt
++;
1342 dev_info(fdev
->dev
, "#%d (%s), irq %d\n", chan
->id
, compatible
,
1343 chan
->irq
!= NO_IRQ
? chan
->irq
: fdev
->irq
);
1348 iounmap(chan
->regs
);
1355 static void fsl_dma_chan_remove(struct fsldma_chan
*chan
)
1357 irq_dispose_mapping(chan
->irq
);
1358 list_del(&chan
->common
.device_node
);
1359 iounmap(chan
->regs
);
1363 static int fsldma_of_probe(struct platform_device
*op
)
1365 struct fsldma_device
*fdev
;
1366 struct device_node
*child
;
1369 fdev
= kzalloc(sizeof(*fdev
), GFP_KERNEL
);
1371 dev_err(&op
->dev
, "No enough memory for 'priv'\n");
1376 fdev
->dev
= &op
->dev
;
1377 INIT_LIST_HEAD(&fdev
->common
.channels
);
1379 /* ioremap the registers for use */
1380 fdev
->regs
= of_iomap(op
->dev
.of_node
, 0);
1382 dev_err(&op
->dev
, "unable to ioremap registers\n");
1387 /* map the channel IRQ if it exists, but don't hookup the handler yet */
1388 fdev
->irq
= irq_of_parse_and_map(op
->dev
.of_node
, 0);
1390 dma_cap_set(DMA_MEMCPY
, fdev
->common
.cap_mask
);
1391 dma_cap_set(DMA_SG
, fdev
->common
.cap_mask
);
1392 dma_cap_set(DMA_SLAVE
, fdev
->common
.cap_mask
);
1393 fdev
->common
.device_alloc_chan_resources
= fsl_dma_alloc_chan_resources
;
1394 fdev
->common
.device_free_chan_resources
= fsl_dma_free_chan_resources
;
1395 fdev
->common
.device_prep_dma_memcpy
= fsl_dma_prep_memcpy
;
1396 fdev
->common
.device_prep_dma_sg
= fsl_dma_prep_sg
;
1397 fdev
->common
.device_tx_status
= fsl_tx_status
;
1398 fdev
->common
.device_issue_pending
= fsl_dma_memcpy_issue_pending
;
1399 fdev
->common
.device_prep_slave_sg
= fsl_dma_prep_slave_sg
;
1400 fdev
->common
.device_control
= fsl_dma_device_control
;
1401 fdev
->common
.dev
= &op
->dev
;
1403 dma_set_mask(&(op
->dev
), DMA_BIT_MASK(36));
1405 platform_set_drvdata(op
, fdev
);
1408 * We cannot use of_platform_bus_probe() because there is no
1409 * of_platform_bus_remove(). Instead, we manually instantiate every DMA
1412 for_each_child_of_node(op
->dev
.of_node
, child
) {
1413 if (of_device_is_compatible(child
, "fsl,eloplus-dma-channel")) {
1414 fsl_dma_chan_probe(fdev
, child
,
1415 FSL_DMA_IP_85XX
| FSL_DMA_BIG_ENDIAN
,
1416 "fsl,eloplus-dma-channel");
1419 if (of_device_is_compatible(child
, "fsl,elo-dma-channel")) {
1420 fsl_dma_chan_probe(fdev
, child
,
1421 FSL_DMA_IP_83XX
| FSL_DMA_LITTLE_ENDIAN
,
1422 "fsl,elo-dma-channel");
1427 * Hookup the IRQ handler(s)
1429 * If we have a per-controller interrupt, we prefer that to the
1430 * per-channel interrupts to reduce the number of shared interrupt
1431 * handlers on the same IRQ line
1433 err
= fsldma_request_irqs(fdev
);
1435 dev_err(fdev
->dev
, "unable to request IRQs\n");
1439 dma_async_device_register(&fdev
->common
);
1443 irq_dispose_mapping(fdev
->irq
);
1449 static int fsldma_of_remove(struct platform_device
*op
)
1451 struct fsldma_device
*fdev
;
1454 fdev
= platform_get_drvdata(op
);
1455 dma_async_device_unregister(&fdev
->common
);
1457 fsldma_free_irqs(fdev
);
1459 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1461 fsl_dma_chan_remove(fdev
->chan
[i
]);
1464 iounmap(fdev
->regs
);
1471 static int fsldma_suspend_late(struct device
*dev
)
1473 struct platform_device
*pdev
= to_platform_device(dev
);
1474 struct fsldma_device
*fdev
= platform_get_drvdata(pdev
);
1475 struct fsldma_chan
*chan
;
1478 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1479 chan
= fdev
->chan
[i
];
1483 spin_lock_bh(&chan
->desc_lock
);
1484 if (unlikely(!chan
->idle
))
1486 chan
->regs_save
.mr
= get_mr(chan
);
1487 chan
->pm_state
= SUSPENDED
;
1488 spin_unlock_bh(&chan
->desc_lock
);
1493 for (; i
>= 0; i
--) {
1494 chan
= fdev
->chan
[i
];
1497 chan
->pm_state
= RUNNING
;
1498 spin_unlock_bh(&chan
->desc_lock
);
1503 static int fsldma_resume_early(struct device
*dev
)
1505 struct platform_device
*pdev
= to_platform_device(dev
);
1506 struct fsldma_device
*fdev
= platform_get_drvdata(pdev
);
1507 struct fsldma_chan
*chan
;
1511 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1512 chan
= fdev
->chan
[i
];
1516 spin_lock_bh(&chan
->desc_lock
);
1517 mode
= chan
->regs_save
.mr
1518 & ~FSL_DMA_MR_CS
& ~FSL_DMA_MR_CC
& ~FSL_DMA_MR_CA
;
1520 chan
->pm_state
= RUNNING
;
1521 spin_unlock_bh(&chan
->desc_lock
);
1527 static const struct dev_pm_ops fsldma_pm_ops
= {
1528 .suspend_late
= fsldma_suspend_late
,
1529 .resume_early
= fsldma_resume_early
,
1533 static const struct of_device_id fsldma_of_ids
[] = {
1534 { .compatible
= "fsl,elo3-dma", },
1535 { .compatible
= "fsl,eloplus-dma", },
1536 { .compatible
= "fsl,elo-dma", },
1540 static struct platform_driver fsldma_of_driver
= {
1542 .name
= "fsl-elo-dma",
1543 .owner
= THIS_MODULE
,
1544 .of_match_table
= fsldma_of_ids
,
1546 .pm
= &fsldma_pm_ops
,
1549 .probe
= fsldma_of_probe
,
1550 .remove
= fsldma_of_remove
,
1553 /*----------------------------------------------------------------------------*/
1554 /* Module Init / Exit */
1555 /*----------------------------------------------------------------------------*/
1557 static __init
int fsldma_init(void)
1559 pr_info("Freescale Elo series DMA driver\n");
1560 return platform_driver_register(&fsldma_of_driver
);
1563 static void __exit
fsldma_exit(void)
1565 platform_driver_unregister(&fsldma_of_driver
);
1568 subsys_initcall(fsldma_init
);
1569 module_exit(fsldma_exit
);
1571 MODULE_DESCRIPTION("Freescale Elo series DMA driver");
1572 MODULE_LICENSE("GPL");