2 * OMAP DMAengine support
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 #include <linux/dmaengine.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/err.h>
11 #include <linux/init.h>
12 #include <linux/interrupt.h>
13 #include <linux/list.h>
14 #include <linux/module.h>
15 #include <linux/omap-dma.h>
16 #include <linux/platform_device.h>
17 #include <linux/slab.h>
18 #include <linux/spinlock.h>
23 struct dma_device ddev
;
25 struct tasklet_struct task
;
26 struct list_head pending
;
30 struct virt_dma_chan vc
;
31 struct list_head node
;
33 struct dma_slave_config cfg
;
39 struct omap_desc
*desc
;
45 uint32_t en
; /* number of elements (24-bit) */
46 uint32_t fn
; /* number of frames (16-bit) */
50 struct virt_dma_desc vd
;
51 enum dma_transfer_direction dir
;
54 int16_t fi
; /* for OMAP_DMA_SYNC_PACKET */
55 uint8_t es
; /* OMAP_DMA_DATA_TYPE_xxx */
56 uint8_t sync_mode
; /* OMAP_DMA_SYNC_xxx */
57 uint8_t sync_type
; /* OMAP_DMA_xxx_SYNC* */
58 uint8_t periph_port
; /* Peripheral port */
64 static const unsigned es_bytes
[] = {
65 [OMAP_DMA_DATA_TYPE_S8
] = 1,
66 [OMAP_DMA_DATA_TYPE_S16
] = 2,
67 [OMAP_DMA_DATA_TYPE_S32
] = 4,
70 static inline struct omap_dmadev
*to_omap_dma_dev(struct dma_device
*d
)
72 return container_of(d
, struct omap_dmadev
, ddev
);
75 static inline struct omap_chan
*to_omap_dma_chan(struct dma_chan
*c
)
77 return container_of(c
, struct omap_chan
, vc
.chan
);
80 static inline struct omap_desc
*to_omap_dma_desc(struct dma_async_tx_descriptor
*t
)
82 return container_of(t
, struct omap_desc
, vd
.tx
);
85 static void omap_dma_desc_free(struct virt_dma_desc
*vd
)
87 kfree(container_of(vd
, struct omap_desc
, vd
));
90 static void omap_dma_start_sg(struct omap_chan
*c
, struct omap_desc
*d
,
93 struct omap_sg
*sg
= d
->sg
+ idx
;
95 if (d
->dir
== DMA_DEV_TO_MEM
)
96 omap_set_dma_dest_params(c
->dma_ch
, OMAP_DMA_PORT_EMIFF
,
97 OMAP_DMA_AMODE_POST_INC
, sg
->addr
, 0, 0);
99 omap_set_dma_src_params(c
->dma_ch
, OMAP_DMA_PORT_EMIFF
,
100 OMAP_DMA_AMODE_POST_INC
, sg
->addr
, 0, 0);
102 omap_set_dma_transfer_params(c
->dma_ch
, d
->es
, sg
->en
, sg
->fn
,
103 d
->sync_mode
, c
->dma_sig
, d
->sync_type
);
105 omap_start_dma(c
->dma_ch
);
108 static void omap_dma_start_desc(struct omap_chan
*c
)
110 struct virt_dma_desc
*vd
= vchan_next_desc(&c
->vc
);
120 c
->desc
= d
= to_omap_dma_desc(&vd
->tx
);
123 if (d
->dir
== DMA_DEV_TO_MEM
)
124 omap_set_dma_src_params(c
->dma_ch
, d
->periph_port
,
125 OMAP_DMA_AMODE_CONSTANT
, d
->dev_addr
, 0, d
->fi
);
127 omap_set_dma_dest_params(c
->dma_ch
, d
->periph_port
,
128 OMAP_DMA_AMODE_CONSTANT
, d
->dev_addr
, 0, d
->fi
);
130 omap_dma_start_sg(c
, d
, 0);
133 static void omap_dma_callback(int ch
, u16 status
, void *data
)
135 struct omap_chan
*c
= data
;
139 spin_lock_irqsave(&c
->vc
.lock
, flags
);
143 if (++c
->sgidx
< d
->sglen
) {
144 omap_dma_start_sg(c
, d
, c
->sgidx
);
146 omap_dma_start_desc(c
);
147 vchan_cookie_complete(&d
->vd
);
150 vchan_cyclic_callback(&d
->vd
);
153 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
157 * This callback schedules all pending channels. We could be more
158 * clever here by postponing allocation of the real DMA channels to
159 * this point, and freeing them when our virtual channel becomes idle.
161 * We would then need to deal with 'all channels in-use'
163 static void omap_dma_sched(unsigned long data
)
165 struct omap_dmadev
*d
= (struct omap_dmadev
*)data
;
168 spin_lock_irq(&d
->lock
);
169 list_splice_tail_init(&d
->pending
, &head
);
170 spin_unlock_irq(&d
->lock
);
172 while (!list_empty(&head
)) {
173 struct omap_chan
*c
= list_first_entry(&head
,
174 struct omap_chan
, node
);
176 spin_lock_irq(&c
->vc
.lock
);
177 list_del_init(&c
->node
);
178 omap_dma_start_desc(c
);
179 spin_unlock_irq(&c
->vc
.lock
);
183 static int omap_dma_alloc_chan_resources(struct dma_chan
*chan
)
185 struct omap_chan
*c
= to_omap_dma_chan(chan
);
187 dev_info(c
->vc
.chan
.device
->dev
, "allocating channel for %u\n", c
->dma_sig
);
189 return omap_request_dma(c
->dma_sig
, "DMA engine",
190 omap_dma_callback
, c
, &c
->dma_ch
);
193 static void omap_dma_free_chan_resources(struct dma_chan
*chan
)
195 struct omap_chan
*c
= to_omap_dma_chan(chan
);
197 vchan_free_chan_resources(&c
->vc
);
198 omap_free_dma(c
->dma_ch
);
200 dev_info(c
->vc
.chan
.device
->dev
, "freeing channel for %u\n", c
->dma_sig
);
203 static size_t omap_dma_sg_size(struct omap_sg
*sg
)
205 return sg
->en
* sg
->fn
;
208 static size_t omap_dma_desc_size(struct omap_desc
*d
)
213 for (size
= i
= 0; i
< d
->sglen
; i
++)
214 size
+= omap_dma_sg_size(&d
->sg
[i
]);
216 return size
* es_bytes
[d
->es
];
219 static size_t omap_dma_desc_size_pos(struct omap_desc
*d
, dma_addr_t addr
)
222 size_t size
, es_size
= es_bytes
[d
->es
];
224 for (size
= i
= 0; i
< d
->sglen
; i
++) {
225 size_t this_size
= omap_dma_sg_size(&d
->sg
[i
]) * es_size
;
229 else if (addr
>= d
->sg
[i
].addr
&&
230 addr
< d
->sg
[i
].addr
+ this_size
)
231 size
+= d
->sg
[i
].addr
+ this_size
- addr
;
236 static enum dma_status
omap_dma_tx_status(struct dma_chan
*chan
,
237 dma_cookie_t cookie
, struct dma_tx_state
*txstate
)
239 struct omap_chan
*c
= to_omap_dma_chan(chan
);
240 struct virt_dma_desc
*vd
;
244 ret
= dma_cookie_status(chan
, cookie
, txstate
);
245 if (ret
== DMA_SUCCESS
|| !txstate
)
248 spin_lock_irqsave(&c
->vc
.lock
, flags
);
249 vd
= vchan_find_desc(&c
->vc
, cookie
);
251 txstate
->residue
= omap_dma_desc_size(to_omap_dma_desc(&vd
->tx
));
252 } else if (c
->desc
&& c
->desc
->vd
.tx
.cookie
== cookie
) {
253 struct omap_desc
*d
= c
->desc
;
256 if (d
->dir
== DMA_MEM_TO_DEV
)
257 pos
= omap_get_dma_src_pos(c
->dma_ch
);
258 else if (d
->dir
== DMA_DEV_TO_MEM
)
259 pos
= omap_get_dma_dst_pos(c
->dma_ch
);
263 txstate
->residue
= omap_dma_desc_size_pos(d
, pos
);
265 txstate
->residue
= 0;
267 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
272 static void omap_dma_issue_pending(struct dma_chan
*chan
)
274 struct omap_chan
*c
= to_omap_dma_chan(chan
);
277 spin_lock_irqsave(&c
->vc
.lock
, flags
);
278 if (vchan_issue_pending(&c
->vc
) && !c
->desc
) {
279 struct omap_dmadev
*d
= to_omap_dma_dev(chan
->device
);
281 if (list_empty(&c
->node
))
282 list_add_tail(&c
->node
, &d
->pending
);
283 spin_unlock(&d
->lock
);
284 tasklet_schedule(&d
->task
);
286 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
289 static struct dma_async_tx_descriptor
*omap_dma_prep_slave_sg(
290 struct dma_chan
*chan
, struct scatterlist
*sgl
, unsigned sglen
,
291 enum dma_transfer_direction dir
, unsigned long tx_flags
, void *context
)
293 struct omap_chan
*c
= to_omap_dma_chan(chan
);
294 enum dma_slave_buswidth dev_width
;
295 struct scatterlist
*sgent
;
298 unsigned i
, j
= 0, es
, en
, frame_bytes
, sync_type
;
301 if (dir
== DMA_DEV_TO_MEM
) {
302 dev_addr
= c
->cfg
.src_addr
;
303 dev_width
= c
->cfg
.src_addr_width
;
304 burst
= c
->cfg
.src_maxburst
;
305 sync_type
= OMAP_DMA_SRC_SYNC
;
306 } else if (dir
== DMA_MEM_TO_DEV
) {
307 dev_addr
= c
->cfg
.dst_addr
;
308 dev_width
= c
->cfg
.dst_addr_width
;
309 burst
= c
->cfg
.dst_maxburst
;
310 sync_type
= OMAP_DMA_DST_SYNC
;
312 dev_err(chan
->device
->dev
, "%s: bad direction?\n", __func__
);
316 /* Bus width translates to the element size (ES) */
318 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
319 es
= OMAP_DMA_DATA_TYPE_S8
;
321 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
322 es
= OMAP_DMA_DATA_TYPE_S16
;
324 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
325 es
= OMAP_DMA_DATA_TYPE_S32
;
327 default: /* not reached */
331 /* Now allocate and setup the descriptor. */
332 d
= kzalloc(sizeof(*d
) + sglen
* sizeof(d
->sg
[0]), GFP_ATOMIC
);
337 d
->dev_addr
= dev_addr
;
339 d
->sync_mode
= OMAP_DMA_SYNC_FRAME
;
340 d
->sync_type
= sync_type
;
341 d
->periph_port
= OMAP_DMA_PORT_TIPB
;
344 * Build our scatterlist entries: each contains the address,
345 * the number of elements (EN) in each frame, and the number of
346 * frames (FN). Number of bytes for this entry = ES * EN * FN.
348 * Burst size translates to number of elements with frame sync.
349 * Note: DMA engine defines burst to be the number of dev-width
353 frame_bytes
= es_bytes
[es
] * en
;
354 for_each_sg(sgl
, sgent
, sglen
, i
) {
355 d
->sg
[j
].addr
= sg_dma_address(sgent
);
357 d
->sg
[j
].fn
= sg_dma_len(sgent
) / frame_bytes
;
363 return vchan_tx_prep(&c
->vc
, &d
->vd
, tx_flags
);
366 static struct dma_async_tx_descriptor
*omap_dma_prep_dma_cyclic(
367 struct dma_chan
*chan
, dma_addr_t buf_addr
, size_t buf_len
,
368 size_t period_len
, enum dma_transfer_direction dir
, unsigned long flags
,
371 struct omap_chan
*c
= to_omap_dma_chan(chan
);
372 enum dma_slave_buswidth dev_width
;
375 unsigned es
, sync_type
;
378 if (dir
== DMA_DEV_TO_MEM
) {
379 dev_addr
= c
->cfg
.src_addr
;
380 dev_width
= c
->cfg
.src_addr_width
;
381 burst
= c
->cfg
.src_maxburst
;
382 sync_type
= OMAP_DMA_SRC_SYNC
;
383 } else if (dir
== DMA_MEM_TO_DEV
) {
384 dev_addr
= c
->cfg
.dst_addr
;
385 dev_width
= c
->cfg
.dst_addr_width
;
386 burst
= c
->cfg
.dst_maxburst
;
387 sync_type
= OMAP_DMA_DST_SYNC
;
389 dev_err(chan
->device
->dev
, "%s: bad direction?\n", __func__
);
393 /* Bus width translates to the element size (ES) */
395 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
396 es
= OMAP_DMA_DATA_TYPE_S8
;
398 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
399 es
= OMAP_DMA_DATA_TYPE_S16
;
401 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
402 es
= OMAP_DMA_DATA_TYPE_S32
;
404 default: /* not reached */
408 /* Now allocate and setup the descriptor. */
409 d
= kzalloc(sizeof(*d
) + sizeof(d
->sg
[0]), GFP_ATOMIC
);
414 d
->dev_addr
= dev_addr
;
418 d
->sync_mode
= OMAP_DMA_SYNC_PACKET
;
420 d
->sync_mode
= OMAP_DMA_SYNC_ELEMENT
;
421 d
->sync_type
= sync_type
;
422 d
->periph_port
= OMAP_DMA_PORT_MPUI
;
423 d
->sg
[0].addr
= buf_addr
;
424 d
->sg
[0].en
= period_len
/ es_bytes
[es
];
425 d
->sg
[0].fn
= buf_len
/ period_len
;
430 omap_dma_link_lch(c
->dma_ch
, c
->dma_ch
);
432 if (flags
& DMA_PREP_INTERRUPT
)
433 omap_enable_dma_irq(c
->dma_ch
, OMAP_DMA_FRAME_IRQ
);
435 omap_disable_dma_irq(c
->dma_ch
, OMAP_DMA_BLOCK_IRQ
);
438 if (dma_omap2plus()) {
439 omap_set_dma_src_burst_mode(c
->dma_ch
, OMAP_DMA_DATA_BURST_16
);
440 omap_set_dma_dest_burst_mode(c
->dma_ch
, OMAP_DMA_DATA_BURST_16
);
443 return vchan_tx_prep(&c
->vc
, &d
->vd
, flags
);
446 static int omap_dma_slave_config(struct omap_chan
*c
, struct dma_slave_config
*cfg
)
448 if (cfg
->src_addr_width
== DMA_SLAVE_BUSWIDTH_8_BYTES
||
449 cfg
->dst_addr_width
== DMA_SLAVE_BUSWIDTH_8_BYTES
)
452 memcpy(&c
->cfg
, cfg
, sizeof(c
->cfg
));
457 static int omap_dma_terminate_all(struct omap_chan
*c
)
459 struct omap_dmadev
*d
= to_omap_dma_dev(c
->vc
.chan
.device
);
463 spin_lock_irqsave(&c
->vc
.lock
, flags
);
465 /* Prevent this channel being scheduled */
467 list_del_init(&c
->node
);
468 spin_unlock(&d
->lock
);
471 * Stop DMA activity: we assume the callback will not be called
472 * after omap_stop_dma() returns (even if it does, it will see
473 * c->desc is NULL and exit.)
477 /* Avoid stopping the dma twice */
479 omap_stop_dma(c
->dma_ch
);
485 omap_dma_unlink_lch(c
->dma_ch
, c
->dma_ch
);
488 vchan_get_all_descriptors(&c
->vc
, &head
);
489 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
490 vchan_dma_desc_free_list(&c
->vc
, &head
);
495 static int omap_dma_pause(struct omap_chan
*c
)
497 /* Pause/Resume only allowed with cyclic mode */
502 omap_stop_dma(c
->dma_ch
);
509 static int omap_dma_resume(struct omap_chan
*c
)
511 /* Pause/Resume only allowed with cyclic mode */
516 omap_start_dma(c
->dma_ch
);
523 static int omap_dma_control(struct dma_chan
*chan
, enum dma_ctrl_cmd cmd
,
526 struct omap_chan
*c
= to_omap_dma_chan(chan
);
530 case DMA_SLAVE_CONFIG
:
531 ret
= omap_dma_slave_config(c
, (struct dma_slave_config
*)arg
);
534 case DMA_TERMINATE_ALL
:
535 ret
= omap_dma_terminate_all(c
);
539 ret
= omap_dma_pause(c
);
543 ret
= omap_dma_resume(c
);
554 static int omap_dma_chan_init(struct omap_dmadev
*od
, int dma_sig
)
558 c
= kzalloc(sizeof(*c
), GFP_KERNEL
);
562 c
->dma_sig
= dma_sig
;
563 c
->vc
.desc_free
= omap_dma_desc_free
;
564 vchan_init(&c
->vc
, &od
->ddev
);
565 INIT_LIST_HEAD(&c
->node
);
572 static void omap_dma_free(struct omap_dmadev
*od
)
574 tasklet_kill(&od
->task
);
575 while (!list_empty(&od
->ddev
.channels
)) {
576 struct omap_chan
*c
= list_first_entry(&od
->ddev
.channels
,
577 struct omap_chan
, vc
.chan
.device_node
);
579 list_del(&c
->vc
.chan
.device_node
);
580 tasklet_kill(&c
->vc
.task
);
586 static int omap_dma_probe(struct platform_device
*pdev
)
588 struct omap_dmadev
*od
;
591 od
= kzalloc(sizeof(*od
), GFP_KERNEL
);
595 dma_cap_set(DMA_SLAVE
, od
->ddev
.cap_mask
);
596 dma_cap_set(DMA_CYCLIC
, od
->ddev
.cap_mask
);
597 od
->ddev
.device_alloc_chan_resources
= omap_dma_alloc_chan_resources
;
598 od
->ddev
.device_free_chan_resources
= omap_dma_free_chan_resources
;
599 od
->ddev
.device_tx_status
= omap_dma_tx_status
;
600 od
->ddev
.device_issue_pending
= omap_dma_issue_pending
;
601 od
->ddev
.device_prep_slave_sg
= omap_dma_prep_slave_sg
;
602 od
->ddev
.device_prep_dma_cyclic
= omap_dma_prep_dma_cyclic
;
603 od
->ddev
.device_control
= omap_dma_control
;
604 od
->ddev
.dev
= &pdev
->dev
;
605 INIT_LIST_HEAD(&od
->ddev
.channels
);
606 INIT_LIST_HEAD(&od
->pending
);
607 spin_lock_init(&od
->lock
);
609 tasklet_init(&od
->task
, omap_dma_sched
, (unsigned long)od
);
611 for (i
= 0; i
< 127; i
++) {
612 rc
= omap_dma_chan_init(od
, i
);
619 rc
= dma_async_device_register(&od
->ddev
);
621 pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
625 platform_set_drvdata(pdev
, od
);
628 dev_info(&pdev
->dev
, "OMAP DMA engine driver\n");
633 static int omap_dma_remove(struct platform_device
*pdev
)
635 struct omap_dmadev
*od
= platform_get_drvdata(pdev
);
637 dma_async_device_unregister(&od
->ddev
);
643 static struct platform_driver omap_dma_driver
= {
644 .probe
= omap_dma_probe
,
645 .remove
= omap_dma_remove
,
647 .name
= "omap-dma-engine",
648 .owner
= THIS_MODULE
,
652 bool omap_dma_filter_fn(struct dma_chan
*chan
, void *param
)
654 if (chan
->device
->dev
->driver
== &omap_dma_driver
.driver
) {
655 struct omap_chan
*c
= to_omap_dma_chan(chan
);
656 unsigned req
= *(unsigned *)param
;
658 return req
== c
->dma_sig
;
662 EXPORT_SYMBOL_GPL(omap_dma_filter_fn
);
664 static int omap_dma_init(void)
666 return platform_driver_register(&omap_dma_driver
);
668 subsys_initcall(omap_dma_init
);
670 static void __exit
omap_dma_exit(void)
672 platform_driver_unregister(&omap_dma_driver
);
674 module_exit(omap_dma_exit
);
676 MODULE_AUTHOR("Russell King");
677 MODULE_LICENSE("GPL");