2 * TI EDMA DMA engine driver
4 * Copyright 2012 Texas Instruments
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation version 2.
10 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
11 * kind, whether express or implied; without even the implied warranty
12 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
16 #include <linux/dmaengine.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/err.h>
19 #include <linux/init.h>
20 #include <linux/interrupt.h>
21 #include <linux/list.h>
22 #include <linux/module.h>
23 #include <linux/platform_device.h>
24 #include <linux/slab.h>
25 #include <linux/spinlock.h>
27 #include <linux/platform_data/edma.h>
29 #include "dmaengine.h"
33 * This will go away when the private EDMA API is folded
34 * into this driver and the platform device(s) are
35 * instantiated in the arch code. We can only get away
36 * with this simplification because DA8XX may not be built
37 * in the same kernel image with other DaVinci parts. This
38 * avoids having to sprinkle dmaengine driver platform devices
39 * and data throughout all the existing board files.
41 #ifdef CONFIG_ARCH_DAVINCI_DA8XX
47 #endif /* CONFIG_ARCH_DAVINCI_DA8XX */
49 /* Max of 16 segments per channel to conserve PaRAM slots */
51 #define EDMA_MAX_SLOTS MAX_NR_SG
52 #define EDMA_DESCRIPTORS 16
55 struct virt_dma_desc vdesc
;
56 struct list_head node
;
60 struct edmacc_param pset
[0];
66 struct virt_dma_chan vchan
;
67 struct list_head node
;
68 struct edma_desc
*edesc
;
72 int slot
[EDMA_MAX_SLOTS
];
73 struct dma_slave_config cfg
;
78 struct dma_device dma_slave
;
79 struct edma_chan slave_chans
[EDMA_CHANS
];
84 static inline struct edma_cc
*to_edma_cc(struct dma_device
*d
)
86 return container_of(d
, struct edma_cc
, dma_slave
);
89 static inline struct edma_chan
*to_edma_chan(struct dma_chan
*c
)
91 return container_of(c
, struct edma_chan
, vchan
.chan
);
94 static inline struct edma_desc
95 *to_edma_desc(struct dma_async_tx_descriptor
*tx
)
97 return container_of(tx
, struct edma_desc
, vdesc
.tx
);
100 static void edma_desc_free(struct virt_dma_desc
*vdesc
)
102 kfree(container_of(vdesc
, struct edma_desc
, vdesc
));
105 /* Dispatch a queued descriptor to the controller (caller holds lock) */
106 static void edma_execute(struct edma_chan
*echan
)
108 struct virt_dma_desc
*vdesc
;
109 struct edma_desc
*edesc
;
110 struct device
*dev
= echan
->vchan
.chan
.device
->dev
;
111 int i
, j
, left
, nslots
;
113 /* If either we processed all psets or we're still not started */
115 echan
->edesc
->pset_nr
== echan
->edesc
->processed
) {
117 vdesc
= vchan_next_desc(&echan
->vchan
);
122 list_del(&vdesc
->node
);
123 echan
->edesc
= to_edma_desc(&vdesc
->tx
);
126 edesc
= echan
->edesc
;
128 /* Find out how many left */
129 left
= edesc
->pset_nr
- edesc
->processed
;
130 nslots
= min(MAX_NR_SG
, left
);
132 /* Write descriptor PaRAM set(s) */
133 for (i
= 0; i
< nslots
; i
++) {
134 j
= i
+ edesc
->processed
;
135 edma_write_slot(echan
->slot
[i
], &edesc
->pset
[j
]);
136 dev_dbg(echan
->vchan
.chan
.device
->dev
,
148 j
, echan
->ch_num
, echan
->slot
[i
],
152 edesc
->pset
[j
].a_b_cnt
,
154 edesc
->pset
[j
].src_dst_bidx
,
155 edesc
->pset
[j
].src_dst_cidx
,
156 edesc
->pset
[j
].link_bcntrld
);
157 /* Link to the previous slot if not the last set */
158 if (i
!= (nslots
- 1))
159 edma_link(echan
->slot
[i
], echan
->slot
[i
+1]);
160 /* Final pset links to the dummy pset */
162 edma_link(echan
->slot
[i
], echan
->ecc
->dummy_slot
);
165 edesc
->processed
+= nslots
;
167 edma_resume(echan
->ch_num
);
169 if (edesc
->processed
<= MAX_NR_SG
) {
170 dev_dbg(dev
, "first transfer starting %d\n", echan
->ch_num
);
171 edma_start(echan
->ch_num
);
175 static int edma_terminate_all(struct edma_chan
*echan
)
180 spin_lock_irqsave(&echan
->vchan
.lock
, flags
);
183 * Stop DMA activity: we assume the callback will not be called
184 * after edma_dma() returns (even if it does, it will see
185 * echan->edesc is NULL and exit.)
189 edma_stop(echan
->ch_num
);
192 vchan_get_all_descriptors(&echan
->vchan
, &head
);
193 spin_unlock_irqrestore(&echan
->vchan
.lock
, flags
);
194 vchan_dma_desc_free_list(&echan
->vchan
, &head
);
199 static int edma_slave_config(struct edma_chan
*echan
,
200 struct dma_slave_config
*cfg
)
202 if (cfg
->src_addr_width
== DMA_SLAVE_BUSWIDTH_8_BYTES
||
203 cfg
->dst_addr_width
== DMA_SLAVE_BUSWIDTH_8_BYTES
)
206 memcpy(&echan
->cfg
, cfg
, sizeof(echan
->cfg
));
211 static int edma_control(struct dma_chan
*chan
, enum dma_ctrl_cmd cmd
,
215 struct dma_slave_config
*config
;
216 struct edma_chan
*echan
= to_edma_chan(chan
);
219 case DMA_TERMINATE_ALL
:
220 edma_terminate_all(echan
);
222 case DMA_SLAVE_CONFIG
:
223 config
= (struct dma_slave_config
*)arg
;
224 ret
= edma_slave_config(echan
, config
);
233 static struct dma_async_tx_descriptor
*edma_prep_slave_sg(
234 struct dma_chan
*chan
, struct scatterlist
*sgl
,
235 unsigned int sg_len
, enum dma_transfer_direction direction
,
236 unsigned long tx_flags
, void *context
)
238 struct edma_chan
*echan
= to_edma_chan(chan
);
239 struct device
*dev
= chan
->device
->dev
;
240 struct edma_desc
*edesc
;
242 enum dma_slave_buswidth dev_width
;
244 struct scatterlist
*sg
;
245 int acnt
, bcnt
, ccnt
, src
, dst
, cidx
;
246 int src_bidx
, dst_bidx
, src_cidx
, dst_cidx
;
249 if (unlikely(!echan
|| !sgl
|| !sg_len
))
252 if (direction
== DMA_DEV_TO_MEM
) {
253 dev_addr
= echan
->cfg
.src_addr
;
254 dev_width
= echan
->cfg
.src_addr_width
;
255 burst
= echan
->cfg
.src_maxburst
;
256 } else if (direction
== DMA_MEM_TO_DEV
) {
257 dev_addr
= echan
->cfg
.dst_addr
;
258 dev_width
= echan
->cfg
.dst_addr_width
;
259 burst
= echan
->cfg
.dst_maxburst
;
261 dev_err(dev
, "%s: bad direction?\n", __func__
);
265 if (dev_width
== DMA_SLAVE_BUSWIDTH_UNDEFINED
) {
266 dev_err(dev
, "Undefined slave buswidth\n");
270 if (sg_len
> MAX_NR_SG
) {
271 dev_err(dev
, "Exceeded max SG segments %d > %d\n",
276 edesc
= kzalloc(sizeof(*edesc
) + sg_len
*
277 sizeof(edesc
->pset
[0]), GFP_ATOMIC
);
279 dev_dbg(dev
, "Failed to allocate a descriptor\n");
283 edesc
->pset_nr
= sg_len
;
285 /* Allocate a PaRAM slot, if needed */
286 nslots
= min_t(unsigned, MAX_NR_SG
, sg_len
);
288 for (i
= 0; i
< nslots
; i
++) {
289 if (echan
->slot
[i
] < 0) {
291 edma_alloc_slot(EDMA_CTLR(echan
->ch_num
),
293 if (echan
->slot
[i
] < 0) {
294 dev_err(dev
, "Failed to allocate slot\n");
300 /* Configure PaRAM sets for each SG */
301 for_each_sg(sgl
, sg
, sg_len
, i
) {
306 * If the maxburst is equal to the fifo width, use
307 * A-synced transfers. This allows for large contiguous
308 * buffer transfers using only one PaRAM set.
311 edesc
->absync
= false;
312 ccnt
= sg_dma_len(sg
) / acnt
/ (SZ_64K
- 1);
313 bcnt
= sg_dma_len(sg
) / acnt
- ccnt
* (SZ_64K
- 1);
320 * If maxburst is greater than the fifo address_width,
321 * use AB-synced transfers where A count is the fifo
322 * address_width and B count is the maxburst. In this
323 * case, we are limited to transfers of C count frames
324 * of (address_width * maxburst) where C count is limited
325 * to SZ_64K-1. This places an upper bound on the length
326 * of an SG segment that can be handled.
329 edesc
->absync
= true;
331 ccnt
= sg_dma_len(sg
) / (acnt
* bcnt
);
332 if (ccnt
> (SZ_64K
- 1)) {
333 dev_err(dev
, "Exceeded max SG segment size\n");
339 if (direction
== DMA_MEM_TO_DEV
) {
340 src
= sg_dma_address(sg
);
348 dst
= sg_dma_address(sg
);
355 edesc
->pset
[i
].opt
= EDMA_TCC(EDMA_CHAN_SLOT(echan
->ch_num
));
356 /* Configure A or AB synchronized transfers */
358 edesc
->pset
[i
].opt
|= SYNCDIM
;
360 /* If this is the last in a current SG set of transactions,
361 enable interrupts so that next set is processed */
362 if (!((i
+1) % MAX_NR_SG
))
363 edesc
->pset
[i
].opt
|= TCINTEN
;
365 /* If this is the last set, enable completion interrupt flag */
367 edesc
->pset
[i
].opt
|= TCINTEN
;
369 edesc
->pset
[i
].src
= src
;
370 edesc
->pset
[i
].dst
= dst
;
372 edesc
->pset
[i
].src_dst_bidx
= (dst_bidx
<< 16) | src_bidx
;
373 edesc
->pset
[i
].src_dst_cidx
= (dst_cidx
<< 16) | src_cidx
;
375 edesc
->pset
[i
].a_b_cnt
= bcnt
<< 16 | acnt
;
376 edesc
->pset
[i
].ccnt
= ccnt
;
377 edesc
->pset
[i
].link_bcntrld
= 0xffffffff;
381 return vchan_tx_prep(&echan
->vchan
, &edesc
->vdesc
, tx_flags
);
384 static void edma_callback(unsigned ch_num
, u16 ch_status
, void *data
)
386 struct edma_chan
*echan
= data
;
387 struct device
*dev
= echan
->vchan
.chan
.device
->dev
;
388 struct edma_desc
*edesc
;
391 /* Pause the channel */
392 edma_pause(echan
->ch_num
);
396 spin_lock_irqsave(&echan
->vchan
.lock
, flags
);
398 edesc
= echan
->edesc
;
400 if (edesc
->processed
== edesc
->pset_nr
) {
401 dev_dbg(dev
, "Transfer complete, stopping channel %d\n", ch_num
);
402 edma_stop(echan
->ch_num
);
403 vchan_cookie_complete(&edesc
->vdesc
);
405 dev_dbg(dev
, "Intermediate transfer complete on channel %d\n", ch_num
);
411 spin_unlock_irqrestore(&echan
->vchan
.lock
, flags
);
415 dev_dbg(dev
, "transfer error on channel %d\n", ch_num
);
422 /* Alloc channel resources */
423 static int edma_alloc_chan_resources(struct dma_chan
*chan
)
425 struct edma_chan
*echan
= to_edma_chan(chan
);
426 struct device
*dev
= chan
->device
->dev
;
431 a_ch_num
= edma_alloc_channel(echan
->ch_num
, edma_callback
,
432 chan
, EVENTQ_DEFAULT
);
439 if (a_ch_num
!= echan
->ch_num
) {
440 dev_err(dev
, "failed to allocate requested channel %u:%u\n",
441 EDMA_CTLR(echan
->ch_num
),
442 EDMA_CHAN_SLOT(echan
->ch_num
));
447 echan
->alloced
= true;
448 echan
->slot
[0] = echan
->ch_num
;
450 dev_info(dev
, "allocated channel for %u:%u\n",
451 EDMA_CTLR(echan
->ch_num
), EDMA_CHAN_SLOT(echan
->ch_num
));
456 edma_free_channel(a_ch_num
);
461 /* Free channel resources */
462 static void edma_free_chan_resources(struct dma_chan
*chan
)
464 struct edma_chan
*echan
= to_edma_chan(chan
);
465 struct device
*dev
= chan
->device
->dev
;
468 /* Terminate transfers */
469 edma_stop(echan
->ch_num
);
471 vchan_free_chan_resources(&echan
->vchan
);
473 /* Free EDMA PaRAM slots */
474 for (i
= 1; i
< EDMA_MAX_SLOTS
; i
++) {
475 if (echan
->slot
[i
] >= 0) {
476 edma_free_slot(echan
->slot
[i
]);
481 /* Free EDMA channel */
482 if (echan
->alloced
) {
483 edma_free_channel(echan
->ch_num
);
484 echan
->alloced
= false;
487 dev_info(dev
, "freeing channel for %u\n", echan
->ch_num
);
490 /* Send pending descriptor to hardware */
491 static void edma_issue_pending(struct dma_chan
*chan
)
493 struct edma_chan
*echan
= to_edma_chan(chan
);
496 spin_lock_irqsave(&echan
->vchan
.lock
, flags
);
497 if (vchan_issue_pending(&echan
->vchan
) && !echan
->edesc
)
499 spin_unlock_irqrestore(&echan
->vchan
.lock
, flags
);
502 static size_t edma_desc_size(struct edma_desc
*edesc
)
508 for (size
= i
= 0; i
< edesc
->pset_nr
; i
++)
509 size
+= (edesc
->pset
[i
].a_b_cnt
& 0xffff) *
510 (edesc
->pset
[i
].a_b_cnt
>> 16) *
513 size
= (edesc
->pset
[0].a_b_cnt
& 0xffff) *
514 (edesc
->pset
[0].a_b_cnt
>> 16) +
515 (edesc
->pset
[0].a_b_cnt
& 0xffff) *
516 (SZ_64K
- 1) * edesc
->pset
[0].ccnt
;
521 /* Check request completion status */
522 static enum dma_status
edma_tx_status(struct dma_chan
*chan
,
524 struct dma_tx_state
*txstate
)
526 struct edma_chan
*echan
= to_edma_chan(chan
);
527 struct virt_dma_desc
*vdesc
;
531 ret
= dma_cookie_status(chan
, cookie
, txstate
);
532 if (ret
== DMA_SUCCESS
|| !txstate
)
535 spin_lock_irqsave(&echan
->vchan
.lock
, flags
);
536 vdesc
= vchan_find_desc(&echan
->vchan
, cookie
);
538 txstate
->residue
= edma_desc_size(to_edma_desc(&vdesc
->tx
));
539 } else if (echan
->edesc
&& echan
->edesc
->vdesc
.tx
.cookie
== cookie
) {
540 struct edma_desc
*edesc
= echan
->edesc
;
541 txstate
->residue
= edma_desc_size(edesc
);
543 spin_unlock_irqrestore(&echan
->vchan
.lock
, flags
);
548 static void __init
edma_chan_init(struct edma_cc
*ecc
,
549 struct dma_device
*dma
,
550 struct edma_chan
*echans
)
554 for (i
= 0; i
< EDMA_CHANS
; i
++) {
555 struct edma_chan
*echan
= &echans
[i
];
556 echan
->ch_num
= EDMA_CTLR_CHAN(ecc
->ctlr
, i
);
558 echan
->vchan
.desc_free
= edma_desc_free
;
560 vchan_init(&echan
->vchan
, dma
);
562 INIT_LIST_HEAD(&echan
->node
);
563 for (j
= 0; j
< EDMA_MAX_SLOTS
; j
++)
568 static void edma_dma_init(struct edma_cc
*ecc
, struct dma_device
*dma
,
571 dma
->device_prep_slave_sg
= edma_prep_slave_sg
;
572 dma
->device_alloc_chan_resources
= edma_alloc_chan_resources
;
573 dma
->device_free_chan_resources
= edma_free_chan_resources
;
574 dma
->device_issue_pending
= edma_issue_pending
;
575 dma
->device_tx_status
= edma_tx_status
;
576 dma
->device_control
= edma_control
;
579 INIT_LIST_HEAD(&dma
->channels
);
582 static int edma_probe(struct platform_device
*pdev
)
587 ecc
= devm_kzalloc(&pdev
->dev
, sizeof(*ecc
), GFP_KERNEL
);
589 dev_err(&pdev
->dev
, "Can't allocate controller\n");
593 ecc
->ctlr
= pdev
->id
;
594 ecc
->dummy_slot
= edma_alloc_slot(ecc
->ctlr
, EDMA_SLOT_ANY
);
595 if (ecc
->dummy_slot
< 0) {
596 dev_err(&pdev
->dev
, "Can't allocate PaRAM dummy slot\n");
600 dma_cap_zero(ecc
->dma_slave
.cap_mask
);
601 dma_cap_set(DMA_SLAVE
, ecc
->dma_slave
.cap_mask
);
603 edma_dma_init(ecc
, &ecc
->dma_slave
, &pdev
->dev
);
605 edma_chan_init(ecc
, &ecc
->dma_slave
, ecc
->slave_chans
);
607 ret
= dma_async_device_register(&ecc
->dma_slave
);
611 platform_set_drvdata(pdev
, ecc
);
613 dev_info(&pdev
->dev
, "TI EDMA DMA engine driver\n");
618 edma_free_slot(ecc
->dummy_slot
);
622 static int edma_remove(struct platform_device
*pdev
)
624 struct device
*dev
= &pdev
->dev
;
625 struct edma_cc
*ecc
= dev_get_drvdata(dev
);
627 dma_async_device_unregister(&ecc
->dma_slave
);
628 edma_free_slot(ecc
->dummy_slot
);
633 static struct platform_driver edma_driver
= {
635 .remove
= edma_remove
,
637 .name
= "edma-dma-engine",
638 .owner
= THIS_MODULE
,
642 bool edma_filter_fn(struct dma_chan
*chan
, void *param
)
644 if (chan
->device
->dev
->driver
== &edma_driver
.driver
) {
645 struct edma_chan
*echan
= to_edma_chan(chan
);
646 unsigned ch_req
= *(unsigned *)param
;
647 return ch_req
== echan
->ch_num
;
651 EXPORT_SYMBOL(edma_filter_fn
);
653 static struct platform_device
*pdev0
, *pdev1
;
655 static const struct platform_device_info edma_dev_info0
= {
656 .name
= "edma-dma-engine",
660 static const struct platform_device_info edma_dev_info1
= {
661 .name
= "edma-dma-engine",
665 static int edma_init(void)
667 int ret
= platform_driver_register(&edma_driver
);
670 pdev0
= platform_device_register_full(&edma_dev_info0
);
672 platform_driver_unregister(&edma_driver
);
673 ret
= PTR_ERR(pdev0
);
676 pdev0
->dev
.dma_mask
= &pdev0
->dev
.coherent_dma_mask
;
677 pdev0
->dev
.coherent_dma_mask
= DMA_BIT_MASK(32);
680 if (EDMA_CTLRS
== 2) {
681 pdev1
= platform_device_register_full(&edma_dev_info1
);
683 platform_driver_unregister(&edma_driver
);
684 platform_device_unregister(pdev0
);
685 ret
= PTR_ERR(pdev1
);
687 pdev1
->dev
.dma_mask
= &pdev1
->dev
.coherent_dma_mask
;
688 pdev1
->dev
.coherent_dma_mask
= DMA_BIT_MASK(32);
694 subsys_initcall(edma_init
);
696 static void __exit
edma_exit(void)
698 platform_device_unregister(pdev0
);
700 platform_device_unregister(pdev1
);
701 platform_driver_unregister(&edma_driver
);
703 module_exit(edma_exit
);
705 MODULE_AUTHOR("Matt Porter <mporter@ti.com>");
706 MODULE_DESCRIPTION("TI EDMA DMA engine driver");
707 MODULE_LICENSE("GPL v2");