2 * driver/dma/coh901318.c
4 * Copyright (C) 2007-2009 ST-Ericsson
5 * License terms: GNU General Public License (GPL) version 2
6 * DMA driver for COH 901 318
7 * Author: Per Friden <per.friden@stericsson.com>
10 #include <linux/init.h>
11 #include <linux/module.h>
12 #include <linux/kernel.h> /* printk() */
13 #include <linux/fs.h> /* everything... */
14 #include <linux/slab.h> /* kmalloc() */
15 #include <linux/dmaengine.h>
16 #include <linux/platform_device.h>
17 #include <linux/device.h>
18 #include <linux/irqreturn.h>
19 #include <linux/interrupt.h>
21 #include <linux/uaccess.h>
22 #include <linux/debugfs.h>
23 #include <mach/coh901318.h>
25 #include "coh901318_lli.h"
27 #define COHC_2_DEV(cohc) (&cohc->chan.dev->device)
30 #define COH_DBG(x) ({ if (1) x; 0; })
32 #define COH_DBG(x) ({ if (0) x; 0; })
35 struct coh901318_desc
{
36 struct dma_async_tx_descriptor desc
;
37 struct list_head node
;
38 struct scatterlist
*sg
;
40 struct coh901318_lli
*data
;
41 enum dma_data_direction dir
;
46 struct coh901318_base
{
48 void __iomem
*virtbase
;
49 struct coh901318_pool pool
;
51 struct dma_device dma_slave
;
52 struct dma_device dma_memcpy
;
53 struct coh901318_chan
*chans
;
54 struct coh901318_platform
*platform
;
57 struct coh901318_chan
{
64 struct work_struct free_work
;
67 struct tasklet_struct tasklet
;
69 struct list_head active
;
70 struct list_head queue
;
71 struct list_head free
;
73 unsigned long nbr_active_done
;
77 struct coh901318_base
*base
;
80 static void coh901318_list_print(struct coh901318_chan
*cohc
,
81 struct coh901318_lli
*lli
)
83 struct coh901318_lli
*l
;
84 dma_addr_t addr
= virt_to_phys(lli
);
88 l
= phys_to_virt(addr
);
89 dev_vdbg(COHC_2_DEV(cohc
), "i %d, lli %p, ctrl 0x%x, src 0x%x"
90 ", dst 0x%x, link 0x%x link_virt 0x%p\n",
91 i
, l
, l
->control
, l
->src_addr
, l
->dst_addr
,
92 l
->link_addr
, phys_to_virt(l
->link_addr
));
98 #ifdef CONFIG_DEBUG_FS
100 #define COH901318_DEBUGFS_ASSIGN(x, y) (x = y)
102 static struct coh901318_base
*debugfs_dma_base
;
103 static struct dentry
*dma_dentry
;
105 static int coh901318_debugfs_open(struct inode
*inode
, struct file
*file
)
108 file
->private_data
= inode
->i_private
;
112 static int coh901318_debugfs_read(struct file
*file
, char __user
*buf
,
113 size_t count
, loff_t
*f_pos
)
115 u64 started_channels
= debugfs_dma_base
->pm
.started_channels
;
116 int pool_count
= debugfs_dma_base
->pool
.debugfs_pool_counter
;
123 dev_buf
= kmalloc(4*1024, GFP_KERNEL
);
128 tmp
+= sprintf(tmp
, "DMA -- enable dma channels\n");
130 for (i
= 0; i
< debugfs_dma_base
->platform
->max_channels
; i
++)
131 if (started_channels
& (1 << i
))
132 tmp
+= sprintf(tmp
, "channel %d\n", i
);
134 tmp
+= sprintf(tmp
, "Pool alloc nbr %d\n", pool_count
);
135 dev_size
= tmp
- dev_buf
;
137 /* No more to read if offset != 0 */
138 if (*f_pos
> dev_size
)
141 if (count
> dev_size
- *f_pos
)
142 count
= dev_size
- *f_pos
;
144 if (copy_to_user(buf
, dev_buf
+ *f_pos
, count
))
157 static const struct file_operations coh901318_debugfs_status_operations
= {
158 .owner
= THIS_MODULE
,
159 .open
= coh901318_debugfs_open
,
160 .read
= coh901318_debugfs_read
,
164 static int __init
init_coh901318_debugfs(void)
167 dma_dentry
= debugfs_create_dir("dma", NULL
);
169 (void) debugfs_create_file("status",
172 &coh901318_debugfs_status_operations
);
176 static void __exit
exit_coh901318_debugfs(void)
178 debugfs_remove_recursive(dma_dentry
);
181 module_init(init_coh901318_debugfs
);
182 module_exit(exit_coh901318_debugfs
);
185 #define COH901318_DEBUGFS_ASSIGN(x, y)
187 #endif /* CONFIG_DEBUG_FS */
189 static inline struct coh901318_chan
*to_coh901318_chan(struct dma_chan
*chan
)
191 return container_of(chan
, struct coh901318_chan
, chan
);
194 static inline dma_addr_t
195 cohc_dev_addr(struct coh901318_chan
*cohc
)
197 return cohc
->base
->platform
->chan_conf
[cohc
->id
].dev_addr
;
200 static inline const struct coh901318_params
*
201 cohc_chan_param(struct coh901318_chan
*cohc
)
203 return &cohc
->base
->platform
->chan_conf
[cohc
->id
].param
;
206 static inline const struct coh_dma_channel
*
207 cohc_chan_conf(struct coh901318_chan
*cohc
)
209 return &cohc
->base
->platform
->chan_conf
[cohc
->id
];
212 static void enable_powersave(struct coh901318_chan
*cohc
)
215 struct powersave
*pm
= &cohc
->base
->pm
;
217 spin_lock_irqsave(&pm
->lock
, flags
);
219 pm
->started_channels
&= ~(1ULL << cohc
->id
);
221 if (!pm
->started_channels
) {
222 /* DMA no longer intends to access memory */
223 cohc
->base
->platform
->access_memory_state(cohc
->base
->dev
,
227 spin_unlock_irqrestore(&pm
->lock
, flags
);
229 static void disable_powersave(struct coh901318_chan
*cohc
)
232 struct powersave
*pm
= &cohc
->base
->pm
;
234 spin_lock_irqsave(&pm
->lock
, flags
);
236 if (!pm
->started_channels
) {
237 /* DMA intends to access memory */
238 cohc
->base
->platform
->access_memory_state(cohc
->base
->dev
,
242 pm
->started_channels
|= (1ULL << cohc
->id
);
244 spin_unlock_irqrestore(&pm
->lock
, flags
);
247 static inline int coh901318_set_ctrl(struct coh901318_chan
*cohc
, u32 control
)
249 int channel
= cohc
->id
;
250 void __iomem
*virtbase
= cohc
->base
->virtbase
;
253 virtbase
+ COH901318_CX_CTRL
+
254 COH901318_CX_CTRL_SPACING
* channel
);
258 static inline int coh901318_set_conf(struct coh901318_chan
*cohc
, u32 conf
)
260 int channel
= cohc
->id
;
261 void __iomem
*virtbase
= cohc
->base
->virtbase
;
264 virtbase
+ COH901318_CX_CFG
+
265 COH901318_CX_CFG_SPACING
*channel
);
270 static int coh901318_start(struct coh901318_chan
*cohc
)
273 int channel
= cohc
->id
;
274 void __iomem
*virtbase
= cohc
->base
->virtbase
;
276 disable_powersave(cohc
);
278 val
= readl(virtbase
+ COH901318_CX_CFG
+
279 COH901318_CX_CFG_SPACING
* channel
);
282 val
|= COH901318_CX_CFG_CH_ENABLE
;
283 writel(val
, virtbase
+ COH901318_CX_CFG
+
284 COH901318_CX_CFG_SPACING
* channel
);
289 static int coh901318_prep_linked_list(struct coh901318_chan
*cohc
,
290 struct coh901318_lli
*data
)
292 int channel
= cohc
->id
;
293 void __iomem
*virtbase
= cohc
->base
->virtbase
;
295 BUG_ON(readl(virtbase
+ COH901318_CX_STAT
+
296 COH901318_CX_STAT_SPACING
*channel
) &
297 COH901318_CX_STAT_ACTIVE
);
299 writel(data
->src_addr
,
300 virtbase
+ COH901318_CX_SRC_ADDR
+
301 COH901318_CX_SRC_ADDR_SPACING
* channel
);
303 writel(data
->dst_addr
, virtbase
+
304 COH901318_CX_DST_ADDR
+
305 COH901318_CX_DST_ADDR_SPACING
* channel
);
307 writel(data
->link_addr
, virtbase
+ COH901318_CX_LNK_ADDR
+
308 COH901318_CX_LNK_ADDR_SPACING
* channel
);
310 writel(data
->control
, virtbase
+ COH901318_CX_CTRL
+
311 COH901318_CX_CTRL_SPACING
* channel
);
316 coh901318_assign_cookie(struct coh901318_chan
*cohc
,
317 struct coh901318_desc
*cohd
)
319 dma_cookie_t cookie
= cohc
->chan
.cookie
;
324 cohc
->chan
.cookie
= cookie
;
325 cohd
->desc
.cookie
= cookie
;
330 static struct coh901318_desc
*
331 coh901318_desc_get(struct coh901318_chan
*cohc
)
333 struct coh901318_desc
*desc
;
335 if (list_empty(&cohc
->free
)) {
336 /* alloc new desc because we're out of used ones
337 * TODO: alloc a pile of descs instead of just one,
338 * avoid many small allocations.
340 desc
= kmalloc(sizeof(struct coh901318_desc
), GFP_NOWAIT
);
343 INIT_LIST_HEAD(&desc
->node
);
345 /* Reuse an old desc. */
346 desc
= list_first_entry(&cohc
->free
,
347 struct coh901318_desc
,
349 list_del(&desc
->node
);
357 coh901318_desc_free(struct coh901318_chan
*cohc
, struct coh901318_desc
*cohd
)
359 list_add_tail(&cohd
->node
, &cohc
->free
);
362 /* call with irq lock held */
364 coh901318_desc_submit(struct coh901318_chan
*cohc
, struct coh901318_desc
*desc
)
366 list_add_tail(&desc
->node
, &cohc
->active
);
368 BUG_ON(cohc
->pending_irqs
!= 0);
370 cohc
->pending_irqs
= desc
->pending_irqs
;
373 static struct coh901318_desc
*
374 coh901318_first_active_get(struct coh901318_chan
*cohc
)
376 struct coh901318_desc
*d
;
378 if (list_empty(&cohc
->active
))
381 d
= list_first_entry(&cohc
->active
,
382 struct coh901318_desc
,
388 coh901318_desc_remove(struct coh901318_desc
*cohd
)
390 list_del(&cohd
->node
);
394 coh901318_desc_queue(struct coh901318_chan
*cohc
, struct coh901318_desc
*desc
)
396 list_add_tail(&desc
->node
, &cohc
->queue
);
399 static struct coh901318_desc
*
400 coh901318_first_queued(struct coh901318_chan
*cohc
)
402 struct coh901318_desc
*d
;
404 if (list_empty(&cohc
->queue
))
407 d
= list_first_entry(&cohc
->queue
,
408 struct coh901318_desc
,
414 * DMA start/stop controls
416 u32
coh901318_get_bytes_left(struct dma_chan
*chan
)
420 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
422 spin_lock_irqsave(&cohc
->lock
, flags
);
424 /* Read transfer count value */
425 ret
= readl(cohc
->base
->virtbase
+
426 COH901318_CX_CTRL
+COH901318_CX_CTRL_SPACING
*
427 cohc
->id
) & COH901318_CX_CTRL_TC_VALUE_MASK
;
429 spin_unlock_irqrestore(&cohc
->lock
, flags
);
433 EXPORT_SYMBOL(coh901318_get_bytes_left
);
436 /* Stops a transfer without losing data. Enables power save.
437 Use this function in conjunction with coh901318_continue(..)
439 void coh901318_stop(struct dma_chan
*chan
)
443 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
444 int channel
= cohc
->id
;
445 void __iomem
*virtbase
= cohc
->base
->virtbase
;
447 spin_lock_irqsave(&cohc
->lock
, flags
);
449 /* Disable channel in HW */
450 val
= readl(virtbase
+ COH901318_CX_CFG
+
451 COH901318_CX_CFG_SPACING
* channel
);
453 /* Stopping infinit transfer */
454 if ((val
& COH901318_CX_CTRL_TC_ENABLE
) == 0 &&
455 (val
& COH901318_CX_CFG_CH_ENABLE
))
459 val
&= ~COH901318_CX_CFG_CH_ENABLE
;
460 /* Enable twice, HW bug work around */
461 writel(val
, virtbase
+ COH901318_CX_CFG
+
462 COH901318_CX_CFG_SPACING
* channel
);
463 writel(val
, virtbase
+ COH901318_CX_CFG
+
464 COH901318_CX_CFG_SPACING
* channel
);
466 /* Spin-wait for it to actually go inactive */
467 while (readl(virtbase
+ COH901318_CX_STAT
+COH901318_CX_STAT_SPACING
*
468 channel
) & COH901318_CX_STAT_ACTIVE
)
471 /* Check if we stopped an active job */
472 if ((readl(virtbase
+ COH901318_CX_CTRL
+COH901318_CX_CTRL_SPACING
*
473 channel
) & COH901318_CX_CTRL_TC_VALUE_MASK
) > 0)
476 enable_powersave(cohc
);
478 spin_unlock_irqrestore(&cohc
->lock
, flags
);
480 EXPORT_SYMBOL(coh901318_stop
);
482 /* Continues a transfer that has been stopped via 300_dma_stop(..).
483 Power save is handled.
485 void coh901318_continue(struct dma_chan
*chan
)
489 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
490 int channel
= cohc
->id
;
492 spin_lock_irqsave(&cohc
->lock
, flags
);
494 disable_powersave(cohc
);
497 /* Enable channel in HW */
498 val
= readl(cohc
->base
->virtbase
+ COH901318_CX_CFG
+
499 COH901318_CX_CFG_SPACING
* channel
);
501 val
|= COH901318_CX_CFG_CH_ENABLE
;
503 writel(val
, cohc
->base
->virtbase
+ COH901318_CX_CFG
+
504 COH901318_CX_CFG_SPACING
*channel
);
509 spin_unlock_irqrestore(&cohc
->lock
, flags
);
511 EXPORT_SYMBOL(coh901318_continue
);
513 bool coh901318_filter_id(struct dma_chan
*chan
, void *chan_id
)
515 unsigned int ch_nr
= (unsigned int) chan_id
;
517 if (ch_nr
== to_coh901318_chan(chan
)->id
)
522 EXPORT_SYMBOL(coh901318_filter_id
);
525 * DMA channel allocation
527 static int coh901318_config(struct coh901318_chan
*cohc
,
528 struct coh901318_params
*param
)
531 const struct coh901318_params
*p
;
532 int channel
= cohc
->id
;
533 void __iomem
*virtbase
= cohc
->base
->virtbase
;
535 spin_lock_irqsave(&cohc
->lock
, flags
);
540 p
= &cohc
->base
->platform
->chan_conf
[channel
].param
;
542 /* Clear any pending BE or TC interrupt */
544 writel(1 << channel
, virtbase
+ COH901318_BE_INT_CLEAR1
);
545 writel(1 << channel
, virtbase
+ COH901318_TC_INT_CLEAR1
);
547 writel(1 << (channel
- 32), virtbase
+
548 COH901318_BE_INT_CLEAR2
);
549 writel(1 << (channel
- 32), virtbase
+
550 COH901318_TC_INT_CLEAR2
);
553 coh901318_set_conf(cohc
, p
->config
);
554 coh901318_set_ctrl(cohc
, p
->ctrl_lli_last
);
556 spin_unlock_irqrestore(&cohc
->lock
, flags
);
561 /* must lock when calling this function
562 * start queued jobs, if any
563 * TODO: start all queued jobs in one go
565 * Returns descriptor if queued job is started otherwise NULL.
566 * If the queue is empty NULL is returned.
568 static struct coh901318_desc
*coh901318_queue_start(struct coh901318_chan
*cohc
)
570 struct coh901318_desc
*cohd_que
;
572 /* start queued jobs, if any
573 * TODO: transmit all queued jobs in one go
575 cohd_que
= coh901318_first_queued(cohc
);
577 if (cohd_que
!= NULL
) {
578 /* Remove from queue */
579 coh901318_desc_remove(cohd_que
);
580 /* initiate DMA job */
583 coh901318_desc_submit(cohc
, cohd_que
);
585 coh901318_prep_linked_list(cohc
, cohd_que
->data
);
588 coh901318_start(cohc
);
595 static void dma_tasklet(unsigned long data
)
597 struct coh901318_chan
*cohc
= (struct coh901318_chan
*) data
;
598 struct coh901318_desc
*cohd_fin
;
600 dma_async_tx_callback callback
;
601 void *callback_param
;
603 spin_lock_irqsave(&cohc
->lock
, flags
);
605 /* get first active entry from list */
606 cohd_fin
= coh901318_first_active_get(cohc
);
608 BUG_ON(cohd_fin
->pending_irqs
== 0);
610 if (cohd_fin
== NULL
)
613 cohd_fin
->pending_irqs
--;
614 cohc
->completed
= cohd_fin
->desc
.cookie
;
616 BUG_ON(cohc
->nbr_active_done
&& cohd_fin
== NULL
);
618 if (cohc
->nbr_active_done
== 0)
621 if (!cohd_fin
->pending_irqs
) {
622 /* release the lli allocation*/
623 coh901318_lli_free(&cohc
->base
->pool
, &cohd_fin
->data
);
626 dev_vdbg(COHC_2_DEV(cohc
), "[%s] chan_id %d pending_irqs %d"
627 " nbr_active_done %ld\n", __func__
,
628 cohc
->id
, cohc
->pending_irqs
, cohc
->nbr_active_done
);
630 /* callback to client */
631 callback
= cohd_fin
->desc
.callback
;
632 callback_param
= cohd_fin
->desc
.callback_param
;
634 if (!cohd_fin
->pending_irqs
) {
635 coh901318_desc_remove(cohd_fin
);
637 /* return desc to free-list */
638 coh901318_desc_free(cohc
, cohd_fin
);
641 if (cohc
->nbr_active_done
)
642 cohc
->nbr_active_done
--;
644 if (cohc
->nbr_active_done
) {
645 if (cohc_chan_conf(cohc
)->priority_high
)
646 tasklet_hi_schedule(&cohc
->tasklet
);
648 tasklet_schedule(&cohc
->tasklet
);
650 spin_unlock_irqrestore(&cohc
->lock
, flags
);
653 callback(callback_param
);
658 spin_unlock_irqrestore(&cohc
->lock
, flags
);
659 dev_err(COHC_2_DEV(cohc
), "[%s] No active dma desc\n", __func__
);
663 /* called from interrupt context */
664 static void dma_tc_handle(struct coh901318_chan
*cohc
)
666 BUG_ON(!cohc
->allocated
&& (list_empty(&cohc
->active
) ||
667 list_empty(&cohc
->queue
)));
669 if (!cohc
->allocated
)
672 BUG_ON(cohc
->pending_irqs
== 0);
674 cohc
->pending_irqs
--;
675 cohc
->nbr_active_done
++;
677 if (cohc
->pending_irqs
== 0 && coh901318_queue_start(cohc
) == NULL
)
680 BUG_ON(list_empty(&cohc
->active
));
682 if (cohc_chan_conf(cohc
)->priority_high
)
683 tasklet_hi_schedule(&cohc
->tasklet
);
685 tasklet_schedule(&cohc
->tasklet
);
689 static irqreturn_t
dma_irq_handler(int irq
, void *dev_id
)
695 struct coh901318_base
*base
= dev_id
;
696 struct coh901318_chan
*cohc
;
697 void __iomem
*virtbase
= base
->virtbase
;
699 status1
= readl(virtbase
+ COH901318_INT_STATUS1
);
700 status2
= readl(virtbase
+ COH901318_INT_STATUS2
);
702 if (unlikely(status1
== 0 && status2
== 0)) {
703 dev_warn(base
->dev
, "spurious DMA IRQ from no channel!\n");
707 /* TODO: consider handle IRQ in tasklet here to
708 * minimize interrupt latency */
710 /* Check the first 32 DMA channels for IRQ */
712 /* Find first bit set, return as a number. */
713 i
= ffs(status1
) - 1;
716 cohc
= &base
->chans
[ch
];
717 spin_lock(&cohc
->lock
);
719 /* Mask off this bit */
720 status1
&= ~(1 << i
);
721 /* Check the individual channel bits */
722 if (test_bit(i
, virtbase
+ COH901318_BE_INT_STATUS1
)) {
723 dev_crit(COHC_2_DEV(cohc
),
724 "DMA bus error on channel %d!\n", ch
);
726 /* Clear BE interrupt */
727 __set_bit(i
, virtbase
+ COH901318_BE_INT_CLEAR1
);
729 /* Caused by TC, really? */
730 if (unlikely(!test_bit(i
, virtbase
+
731 COH901318_TC_INT_STATUS1
))) {
732 dev_warn(COHC_2_DEV(cohc
),
733 "ignoring interrupt not caused by terminal count on channel %d\n", ch
);
734 /* Clear TC interrupt */
736 __set_bit(i
, virtbase
+ COH901318_TC_INT_CLEAR1
);
738 /* Enable powersave if transfer has finished */
739 if (!(readl(virtbase
+ COH901318_CX_STAT
+
740 COH901318_CX_STAT_SPACING
*ch
) &
741 COH901318_CX_STAT_ENABLED
)) {
742 enable_powersave(cohc
);
745 /* Must clear TC interrupt before calling
747 * in case tc_handle initate a new dma job
749 __set_bit(i
, virtbase
+ COH901318_TC_INT_CLEAR1
);
754 spin_unlock(&cohc
->lock
);
757 /* Check the remaining 32 DMA channels for IRQ */
759 /* Find first bit set, return as a number. */
760 i
= ffs(status2
) - 1;
762 cohc
= &base
->chans
[ch
];
763 spin_lock(&cohc
->lock
);
765 /* Mask off this bit */
766 status2
&= ~(1 << i
);
767 /* Check the individual channel bits */
768 if (test_bit(i
, virtbase
+ COH901318_BE_INT_STATUS2
)) {
769 dev_crit(COHC_2_DEV(cohc
),
770 "DMA bus error on channel %d!\n", ch
);
771 /* Clear BE interrupt */
773 __set_bit(i
, virtbase
+ COH901318_BE_INT_CLEAR2
);
775 /* Caused by TC, really? */
776 if (unlikely(!test_bit(i
, virtbase
+
777 COH901318_TC_INT_STATUS2
))) {
778 dev_warn(COHC_2_DEV(cohc
),
779 "ignoring interrupt not caused by terminal count on channel %d\n", ch
);
780 /* Clear TC interrupt */
781 __set_bit(i
, virtbase
+ COH901318_TC_INT_CLEAR2
);
784 /* Enable powersave if transfer has finished */
785 if (!(readl(virtbase
+ COH901318_CX_STAT
+
786 COH901318_CX_STAT_SPACING
*ch
) &
787 COH901318_CX_STAT_ENABLED
)) {
788 enable_powersave(cohc
);
790 /* Must clear TC interrupt before calling
792 * in case tc_handle initate a new dma job
794 __set_bit(i
, virtbase
+ COH901318_TC_INT_CLEAR2
);
799 spin_unlock(&cohc
->lock
);
805 static int coh901318_alloc_chan_resources(struct dma_chan
*chan
)
807 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
809 dev_vdbg(COHC_2_DEV(cohc
), "[%s] DMA channel %d\n",
812 if (chan
->client_count
> 1)
815 coh901318_config(cohc
, NULL
);
818 cohc
->completed
= chan
->cookie
= 1;
824 coh901318_free_chan_resources(struct dma_chan
*chan
)
826 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
827 int channel
= cohc
->id
;
830 spin_lock_irqsave(&cohc
->lock
, flags
);
833 writel(0x00000000U
, cohc
->base
->virtbase
+ COH901318_CX_CFG
+
834 COH901318_CX_CFG_SPACING
*channel
);
835 writel(0x00000000U
, cohc
->base
->virtbase
+ COH901318_CX_CTRL
+
836 COH901318_CX_CTRL_SPACING
*channel
);
840 spin_unlock_irqrestore(&cohc
->lock
, flags
);
842 chan
->device
->device_terminate_all(chan
);
847 coh901318_tx_submit(struct dma_async_tx_descriptor
*tx
)
849 struct coh901318_desc
*cohd
= container_of(tx
, struct coh901318_desc
,
851 struct coh901318_chan
*cohc
= to_coh901318_chan(tx
->chan
);
854 spin_lock_irqsave(&cohc
->lock
, flags
);
856 tx
->cookie
= coh901318_assign_cookie(cohc
, cohd
);
858 coh901318_desc_queue(cohc
, cohd
);
860 spin_unlock_irqrestore(&cohc
->lock
, flags
);
865 static struct dma_async_tx_descriptor
*
866 coh901318_prep_memcpy(struct dma_chan
*chan
, dma_addr_t dest
, dma_addr_t src
,
867 size_t size
, unsigned long flags
)
869 struct coh901318_lli
*data
;
870 struct coh901318_desc
*cohd
;
872 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
874 u32 ctrl_last
= cohc_chan_param(cohc
)->ctrl_lli_last
;
876 spin_lock_irqsave(&cohc
->lock
, flg
);
878 dev_vdbg(COHC_2_DEV(cohc
),
879 "[%s] channel %d src 0x%x dest 0x%x size %d\n",
880 __func__
, cohc
->id
, src
, dest
, size
);
882 if (flags
& DMA_PREP_INTERRUPT
)
883 /* Trigger interrupt after last lli */
884 ctrl_last
|= COH901318_CX_CTRL_TC_IRQ_ENABLE
;
886 lli_len
= size
>> MAX_DMA_PACKET_SIZE_SHIFT
;
887 if ((lli_len
<< MAX_DMA_PACKET_SIZE_SHIFT
) < size
)
890 data
= coh901318_lli_alloc(&cohc
->base
->pool
, lli_len
);
895 cohd
= coh901318_desc_get(cohc
);
901 coh901318_lli_fill_memcpy(
902 &cohc
->base
->pool
, data
, src
, size
, dest
,
903 cohc_chan_param(cohc
)->ctrl_lli_chained
,
907 COH_DBG(coh901318_list_print(cohc
, data
));
909 dma_async_tx_descriptor_init(&cohd
->desc
, chan
);
911 cohd
->desc
.tx_submit
= coh901318_tx_submit
;
913 spin_unlock_irqrestore(&cohc
->lock
, flg
);
917 spin_unlock_irqrestore(&cohc
->lock
, flg
);
921 static struct dma_async_tx_descriptor
*
922 coh901318_prep_slave_sg(struct dma_chan
*chan
, struct scatterlist
*sgl
,
923 unsigned int sg_len
, enum dma_data_direction direction
,
926 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
927 struct coh901318_lli
*data
;
928 struct coh901318_desc
*cohd
;
929 struct scatterlist
*sg
;
933 u32 ctrl_chained
= cohc_chan_param(cohc
)->ctrl_lli_chained
;
934 u32 ctrl
= cohc_chan_param(cohc
)->ctrl_lli
;
935 u32 ctrl_last
= cohc_chan_param(cohc
)->ctrl_lli_last
;
940 if (sgl
->length
== 0)
943 spin_lock_irqsave(&cohc
->lock
, flg
);
945 dev_vdbg(COHC_2_DEV(cohc
), "[%s] sg_len %d dir %d\n",
946 __func__
, sg_len
, direction
);
948 if (flags
& DMA_PREP_INTERRUPT
)
949 /* Trigger interrupt after last lli */
950 ctrl_last
|= COH901318_CX_CTRL_TC_IRQ_ENABLE
;
952 cohd
= coh901318_desc_get(cohc
);
955 cohd
->dir
= direction
;
957 if (direction
== DMA_TO_DEVICE
) {
958 u32 tx_flags
= COH901318_CX_CTRL_PRDD_SOURCE
|
959 COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE
;
961 ctrl_chained
|= tx_flags
;
962 ctrl_last
|= tx_flags
;
964 } else if (direction
== DMA_FROM_DEVICE
) {
965 u32 rx_flags
= COH901318_CX_CTRL_PRDD_DEST
|
966 COH901318_CX_CTRL_DST_ADDR_INC_ENABLE
;
968 ctrl_chained
|= rx_flags
;
969 ctrl_last
|= rx_flags
;
974 dma_async_tx_descriptor_init(&cohd
->desc
, chan
);
976 cohd
->desc
.tx_submit
= coh901318_tx_submit
;
979 /* The dma only supports transmitting packages up to
980 * MAX_DMA_PACKET_SIZE. Calculate to total number of
981 * dma elemts required to send the entire sg list
983 for_each_sg(sgl
, sg
, sg_len
, i
) {
985 size
= sg_dma_len(sg
);
987 if (size
<= MAX_DMA_PACKET_SIZE
) {
992 factor
= size
>> MAX_DMA_PACKET_SIZE_SHIFT
;
993 if ((factor
<< MAX_DMA_PACKET_SIZE_SHIFT
) < size
)
999 data
= coh901318_lli_alloc(&cohc
->base
->pool
, len
);
1004 /* initiate allocated data list */
1005 cohd
->pending_irqs
=
1006 coh901318_lli_fill_sg(&cohc
->base
->pool
, data
, sgl
, sg_len
,
1007 cohc_dev_addr(cohc
),
1011 direction
, COH901318_CX_CTRL_TC_IRQ_ENABLE
);
1014 cohd
->flags
= flags
;
1016 COH_DBG(coh901318_list_print(cohc
, data
));
1018 spin_unlock_irqrestore(&cohc
->lock
, flg
);
1023 coh901318_desc_remove(cohd
);
1024 coh901318_desc_free(cohc
, cohd
);
1025 spin_unlock_irqrestore(&cohc
->lock
, flg
);
1030 static enum dma_status
1031 coh901318_is_tx_complete(struct dma_chan
*chan
,
1032 dma_cookie_t cookie
, dma_cookie_t
*done
,
1035 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
1036 dma_cookie_t last_used
;
1037 dma_cookie_t last_complete
;
1040 last_complete
= cohc
->completed
;
1041 last_used
= chan
->cookie
;
1043 ret
= dma_async_is_complete(cookie
, last_complete
, last_used
);
1046 *done
= last_complete
;
1054 coh901318_issue_pending(struct dma_chan
*chan
)
1056 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
1057 unsigned long flags
;
1059 spin_lock_irqsave(&cohc
->lock
, flags
);
1061 /* Busy means that pending jobs are already being processed */
1063 coh901318_queue_start(cohc
);
1065 spin_unlock_irqrestore(&cohc
->lock
, flags
);
1069 coh901318_terminate_all(struct dma_chan
*chan
)
1071 unsigned long flags
;
1072 struct coh901318_chan
*cohc
= to_coh901318_chan(chan
);
1073 struct coh901318_desc
*cohd
;
1074 void __iomem
*virtbase
= cohc
->base
->virtbase
;
1076 coh901318_stop(chan
);
1078 spin_lock_irqsave(&cohc
->lock
, flags
);
1080 /* Clear any pending BE or TC interrupt */
1081 if (cohc
->id
< 32) {
1082 writel(1 << cohc
->id
, virtbase
+ COH901318_BE_INT_CLEAR1
);
1083 writel(1 << cohc
->id
, virtbase
+ COH901318_TC_INT_CLEAR1
);
1085 writel(1 << (cohc
->id
- 32), virtbase
+
1086 COH901318_BE_INT_CLEAR2
);
1087 writel(1 << (cohc
->id
- 32), virtbase
+
1088 COH901318_TC_INT_CLEAR2
);
1091 enable_powersave(cohc
);
1093 while ((cohd
= coh901318_first_active_get(cohc
))) {
1094 /* release the lli allocation*/
1095 coh901318_lli_free(&cohc
->base
->pool
, &cohd
->data
);
1097 coh901318_desc_remove(cohd
);
1099 /* return desc to free-list */
1100 coh901318_desc_free(cohc
, cohd
);
1103 while ((cohd
= coh901318_first_queued(cohc
))) {
1104 /* release the lli allocation*/
1105 coh901318_lli_free(&cohc
->base
->pool
, &cohd
->data
);
1107 coh901318_desc_remove(cohd
);
1109 /* return desc to free-list */
1110 coh901318_desc_free(cohc
, cohd
);
1114 cohc
->nbr_active_done
= 0;
1116 cohc
->pending_irqs
= 0;
1118 spin_unlock_irqrestore(&cohc
->lock
, flags
);
1120 void coh901318_base_init(struct dma_device
*dma
, const int *pick_chans
,
1121 struct coh901318_base
*base
)
1125 struct coh901318_chan
*cohc
;
1127 INIT_LIST_HEAD(&dma
->channels
);
1129 for (chans_i
= 0; pick_chans
[chans_i
] != -1; chans_i
+= 2) {
1130 for (i
= pick_chans
[chans_i
]; i
<= pick_chans
[chans_i
+1]; i
++) {
1131 cohc
= &base
->chans
[i
];
1134 cohc
->chan
.device
= dma
;
1137 /* TODO: do we really need this lock if only one
1138 * client is connected to each channel?
1141 spin_lock_init(&cohc
->lock
);
1143 cohc
->pending_irqs
= 0;
1144 cohc
->nbr_active_done
= 0;
1146 INIT_LIST_HEAD(&cohc
->free
);
1147 INIT_LIST_HEAD(&cohc
->active
);
1148 INIT_LIST_HEAD(&cohc
->queue
);
1150 tasklet_init(&cohc
->tasklet
, dma_tasklet
,
1151 (unsigned long) cohc
);
1153 list_add_tail(&cohc
->chan
.device_node
,
1159 static int __init
coh901318_probe(struct platform_device
*pdev
)
1162 struct coh901318_platform
*pdata
;
1163 struct coh901318_base
*base
;
1165 struct resource
*io
;
1167 io
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1169 goto err_get_resource
;
1171 /* Map DMA controller registers to virtual memory */
1172 if (request_mem_region(io
->start
,
1174 pdev
->dev
.driver
->name
) == NULL
) {
1176 goto err_request_mem
;
1179 pdata
= pdev
->dev
.platform_data
;
1181 goto err_no_platformdata
;
1183 base
= kmalloc(ALIGN(sizeof(struct coh901318_base
), 4) +
1184 pdata
->max_channels
*
1185 sizeof(struct coh901318_chan
),
1188 goto err_alloc_coh_dma_channels
;
1190 base
->chans
= ((void *)base
) + ALIGN(sizeof(struct coh901318_base
), 4);
1192 base
->virtbase
= ioremap(io
->start
, resource_size(io
));
1193 if (!base
->virtbase
) {
1195 goto err_no_ioremap
;
1198 base
->dev
= &pdev
->dev
;
1199 base
->platform
= pdata
;
1200 spin_lock_init(&base
->pm
.lock
);
1201 base
->pm
.started_channels
= 0;
1203 COH901318_DEBUGFS_ASSIGN(debugfs_dma_base
, base
);
1205 platform_set_drvdata(pdev
, base
);
1207 irq
= platform_get_irq(pdev
, 0);
1211 err
= request_irq(irq
, dma_irq_handler
, IRQF_DISABLED
,
1214 dev_crit(&pdev
->dev
,
1215 "Cannot allocate IRQ for DMA controller!\n");
1216 goto err_request_irq
;
1219 err
= coh901318_pool_create(&base
->pool
, &pdev
->dev
,
1220 sizeof(struct coh901318_lli
),
1223 goto err_pool_create
;
1225 /* init channels for device transfers */
1226 coh901318_base_init(&base
->dma_slave
, base
->platform
->chans_slave
,
1229 dma_cap_zero(base
->dma_slave
.cap_mask
);
1230 dma_cap_set(DMA_SLAVE
, base
->dma_slave
.cap_mask
);
1232 base
->dma_slave
.device_alloc_chan_resources
= coh901318_alloc_chan_resources
;
1233 base
->dma_slave
.device_free_chan_resources
= coh901318_free_chan_resources
;
1234 base
->dma_slave
.device_prep_slave_sg
= coh901318_prep_slave_sg
;
1235 base
->dma_slave
.device_is_tx_complete
= coh901318_is_tx_complete
;
1236 base
->dma_slave
.device_issue_pending
= coh901318_issue_pending
;
1237 base
->dma_slave
.device_terminate_all
= coh901318_terminate_all
;
1238 base
->dma_slave
.dev
= &pdev
->dev
;
1240 err
= dma_async_device_register(&base
->dma_slave
);
1243 goto err_register_slave
;
1245 /* init channels for memcpy */
1246 coh901318_base_init(&base
->dma_memcpy
, base
->platform
->chans_memcpy
,
1249 dma_cap_zero(base
->dma_memcpy
.cap_mask
);
1250 dma_cap_set(DMA_MEMCPY
, base
->dma_memcpy
.cap_mask
);
1252 base
->dma_memcpy
.device_alloc_chan_resources
= coh901318_alloc_chan_resources
;
1253 base
->dma_memcpy
.device_free_chan_resources
= coh901318_free_chan_resources
;
1254 base
->dma_memcpy
.device_prep_dma_memcpy
= coh901318_prep_memcpy
;
1255 base
->dma_memcpy
.device_is_tx_complete
= coh901318_is_tx_complete
;
1256 base
->dma_memcpy
.device_issue_pending
= coh901318_issue_pending
;
1257 base
->dma_memcpy
.device_terminate_all
= coh901318_terminate_all
;
1258 base
->dma_memcpy
.dev
= &pdev
->dev
;
1259 err
= dma_async_device_register(&base
->dma_memcpy
);
1262 goto err_register_memcpy
;
1264 dev_dbg(&pdev
->dev
, "Initialized COH901318 DMA on virtual base 0x%08x\n",
1265 (u32
) base
->virtbase
);
1269 err_register_memcpy
:
1270 dma_async_device_unregister(&base
->dma_slave
);
1272 coh901318_pool_destroy(&base
->pool
);
1274 free_irq(platform_get_irq(pdev
, 0), base
);
1277 iounmap(base
->virtbase
);
1280 err_alloc_coh_dma_channels
:
1281 err_no_platformdata
:
1282 release_mem_region(pdev
->resource
->start
,
1283 resource_size(pdev
->resource
));
1289 static int __exit
coh901318_remove(struct platform_device
*pdev
)
1291 struct coh901318_base
*base
= platform_get_drvdata(pdev
);
1293 dma_async_device_unregister(&base
->dma_memcpy
);
1294 dma_async_device_unregister(&base
->dma_slave
);
1295 coh901318_pool_destroy(&base
->pool
);
1296 free_irq(platform_get_irq(pdev
, 0), base
);
1297 iounmap(base
->virtbase
);
1299 release_mem_region(pdev
->resource
->start
,
1300 resource_size(pdev
->resource
));
1305 static struct platform_driver coh901318_driver
= {
1306 .remove
= __exit_p(coh901318_remove
),
1308 .name
= "coh901318",
1312 int __init
coh901318_init(void)
1314 return platform_driver_probe(&coh901318_driver
, coh901318_probe
);
1316 subsys_initcall(coh901318_init
);
1318 void __exit
coh901318_exit(void)
1320 platform_driver_unregister(&coh901318_driver
);
1322 module_exit(coh901318_exit
);
1324 MODULE_LICENSE("GPL");
1325 MODULE_AUTHOR("Per Friden");