2 * Renesas SuperH DMA Engine support
4 * base is drivers/dma/flsdma.c
6 * Copyright (C) 2009 Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>
7 * Copyright (C) 2009 Renesas Solutions, Inc. All rights reserved.
8 * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved.
10 * This is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
15 * - DMA of SuperH does not have Hardware DMA chain mode.
16 * - MAX DMA size is 16MB.
20 #include <linux/init.h>
21 #include <linux/module.h>
22 #include <linux/slab.h>
23 #include <linux/interrupt.h>
24 #include <linux/dmaengine.h>
25 #include <linux/delay.h>
26 #include <linux/dma-mapping.h>
27 #include <linux/platform_device.h>
28 #include <linux/pm_runtime.h>
30 #include <asm/dmaengine.h>
34 /* DMA descriptor control */
35 enum sh_dmae_desc_status
{
39 DESC_COMPLETED
, /* completed, have to call callback */
40 DESC_WAITING
, /* callback called, waiting for ack / re-submit */
43 #define NR_DESCS_PER_CHANNEL 32
44 /* Default MEMCPY transfer size = 2^2 = 4 bytes */
45 #define LOG2_DEFAULT_XFER_SIZE 2
47 /* A bitmask with bits enough for enum sh_dmae_slave_chan_id */
48 static unsigned long sh_dmae_slave_used
[BITS_TO_LONGS(SHDMA_SLAVE_NUMBER
)];
50 static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan
*sh_chan
, bool all
);
52 static void sh_dmae_writel(struct sh_dmae_chan
*sh_dc
, u32 data
, u32 reg
)
54 __raw_writel(data
, sh_dc
->base
+ reg
/ sizeof(u32
));
57 static u32
sh_dmae_readl(struct sh_dmae_chan
*sh_dc
, u32 reg
)
59 return __raw_readl(sh_dc
->base
+ reg
/ sizeof(u32
));
62 static u16
dmaor_read(struct sh_dmae_device
*shdev
)
64 return __raw_readw(shdev
->chan_reg
+ DMAOR
/ sizeof(u32
));
67 static void dmaor_write(struct sh_dmae_device
*shdev
, u16 data
)
69 __raw_writew(data
, shdev
->chan_reg
+ DMAOR
/ sizeof(u32
));
73 * Reset DMA controller
75 * SH7780 has two DMAOR register
77 static void sh_dmae_ctl_stop(struct sh_dmae_device
*shdev
)
79 unsigned short dmaor
= dmaor_read(shdev
);
81 dmaor_write(shdev
, dmaor
& ~(DMAOR_NMIF
| DMAOR_AE
| DMAOR_DME
));
84 static int sh_dmae_rst(struct sh_dmae_device
*shdev
)
88 sh_dmae_ctl_stop(shdev
);
89 dmaor
= dmaor_read(shdev
) | shdev
->pdata
->dmaor_init
;
91 dmaor_write(shdev
, dmaor
);
92 if (dmaor_read(shdev
) & (DMAOR_AE
| DMAOR_NMIF
)) {
93 pr_warning("dma-sh: Can't initialize DMAOR.\n");
99 static bool dmae_is_busy(struct sh_dmae_chan
*sh_chan
)
101 u32 chcr
= sh_dmae_readl(sh_chan
, CHCR
);
103 if ((chcr
& (CHCR_DE
| CHCR_TE
)) == CHCR_DE
)
104 return true; /* working */
106 return false; /* waiting */
109 static unsigned int calc_xmit_shift(struct sh_dmae_chan
*sh_chan
, u32 chcr
)
111 struct sh_dmae_device
*shdev
= container_of(sh_chan
->common
.device
,
112 struct sh_dmae_device
, common
);
113 struct sh_dmae_pdata
*pdata
= shdev
->pdata
;
114 int cnt
= ((chcr
& pdata
->ts_low_mask
) >> pdata
->ts_low_shift
) |
115 ((chcr
& pdata
->ts_high_mask
) >> pdata
->ts_high_shift
);
117 if (cnt
>= pdata
->ts_shift_num
)
120 return pdata
->ts_shift
[cnt
];
123 static u32
log2size_to_chcr(struct sh_dmae_chan
*sh_chan
, int l2size
)
125 struct sh_dmae_device
*shdev
= container_of(sh_chan
->common
.device
,
126 struct sh_dmae_device
, common
);
127 struct sh_dmae_pdata
*pdata
= shdev
->pdata
;
130 for (i
= 0; i
< pdata
->ts_shift_num
; i
++)
131 if (pdata
->ts_shift
[i
] == l2size
)
134 if (i
== pdata
->ts_shift_num
)
137 return ((i
<< pdata
->ts_low_shift
) & pdata
->ts_low_mask
) |
138 ((i
<< pdata
->ts_high_shift
) & pdata
->ts_high_mask
);
141 static void dmae_set_reg(struct sh_dmae_chan
*sh_chan
, struct sh_dmae_regs
*hw
)
143 sh_dmae_writel(sh_chan
, hw
->sar
, SAR
);
144 sh_dmae_writel(sh_chan
, hw
->dar
, DAR
);
145 sh_dmae_writel(sh_chan
, hw
->tcr
>> sh_chan
->xmit_shift
, TCR
);
148 static void dmae_start(struct sh_dmae_chan
*sh_chan
)
150 u32 chcr
= sh_dmae_readl(sh_chan
, CHCR
);
152 chcr
|= CHCR_DE
| CHCR_IE
;
153 sh_dmae_writel(sh_chan
, chcr
& ~CHCR_TE
, CHCR
);
156 static void dmae_halt(struct sh_dmae_chan
*sh_chan
)
158 u32 chcr
= sh_dmae_readl(sh_chan
, CHCR
);
160 chcr
&= ~(CHCR_DE
| CHCR_TE
| CHCR_IE
);
161 sh_dmae_writel(sh_chan
, chcr
, CHCR
);
164 static void dmae_init(struct sh_dmae_chan
*sh_chan
)
167 * Default configuration for dual address memory-memory transfer.
168 * 0x400 represents auto-request.
170 u32 chcr
= DM_INC
| SM_INC
| 0x400 | log2size_to_chcr(sh_chan
,
171 LOG2_DEFAULT_XFER_SIZE
);
172 sh_chan
->xmit_shift
= calc_xmit_shift(sh_chan
, chcr
);
173 sh_dmae_writel(sh_chan
, chcr
, CHCR
);
176 static int dmae_set_chcr(struct sh_dmae_chan
*sh_chan
, u32 val
)
178 /* When DMA was working, can not set data to CHCR */
179 if (dmae_is_busy(sh_chan
))
182 sh_chan
->xmit_shift
= calc_xmit_shift(sh_chan
, val
);
183 sh_dmae_writel(sh_chan
, val
, CHCR
);
188 static int dmae_set_dmars(struct sh_dmae_chan
*sh_chan
, u16 val
)
190 struct sh_dmae_device
*shdev
= container_of(sh_chan
->common
.device
,
191 struct sh_dmae_device
, common
);
192 struct sh_dmae_pdata
*pdata
= shdev
->pdata
;
193 struct sh_dmae_channel
*chan_pdata
= &pdata
->channel
[sh_chan
->id
];
194 u16 __iomem
*addr
= shdev
->dmars
+ chan_pdata
->dmars
/ sizeof(u16
);
195 int shift
= chan_pdata
->dmars_bit
;
197 if (dmae_is_busy(sh_chan
))
200 __raw_writew((__raw_readw(addr
) & (0xff00 >> shift
)) | (val
<< shift
),
206 static dma_cookie_t
sh_dmae_tx_submit(struct dma_async_tx_descriptor
*tx
)
208 struct sh_desc
*desc
= tx_to_sh_desc(tx
), *chunk
, *last
= desc
, *c
;
209 struct sh_dmae_chan
*sh_chan
= to_sh_chan(tx
->chan
);
210 dma_async_tx_callback callback
= tx
->callback
;
213 spin_lock_bh(&sh_chan
->desc_lock
);
215 cookie
= sh_chan
->common
.cookie
;
220 sh_chan
->common
.cookie
= cookie
;
223 /* Mark all chunks of this descriptor as submitted, move to the queue */
224 list_for_each_entry_safe(chunk
, c
, desc
->node
.prev
, node
) {
226 * All chunks are on the global ld_free, so, we have to find
227 * the end of the chain ourselves
229 if (chunk
!= desc
&& (chunk
->mark
== DESC_IDLE
||
230 chunk
->async_tx
.cookie
> 0 ||
231 chunk
->async_tx
.cookie
== -EBUSY
||
232 &chunk
->node
== &sh_chan
->ld_free
))
234 chunk
->mark
= DESC_SUBMITTED
;
235 /* Callback goes to the last chunk */
236 chunk
->async_tx
.callback
= NULL
;
237 chunk
->cookie
= cookie
;
238 list_move_tail(&chunk
->node
, &sh_chan
->ld_queue
);
242 last
->async_tx
.callback
= callback
;
243 last
->async_tx
.callback_param
= tx
->callback_param
;
245 dev_dbg(sh_chan
->dev
, "submit #%d@%p on %d: %x[%d] -> %x\n",
246 tx
->cookie
, &last
->async_tx
, sh_chan
->id
,
247 desc
->hw
.sar
, desc
->hw
.tcr
, desc
->hw
.dar
);
249 spin_unlock_bh(&sh_chan
->desc_lock
);
254 /* Called with desc_lock held */
255 static struct sh_desc
*sh_dmae_get_desc(struct sh_dmae_chan
*sh_chan
)
257 struct sh_desc
*desc
;
259 list_for_each_entry(desc
, &sh_chan
->ld_free
, node
)
260 if (desc
->mark
!= DESC_PREPARED
) {
261 BUG_ON(desc
->mark
!= DESC_IDLE
);
262 list_del(&desc
->node
);
269 static struct sh_dmae_slave_config
*sh_dmae_find_slave(
270 struct sh_dmae_chan
*sh_chan
, enum sh_dmae_slave_chan_id slave_id
)
272 struct dma_device
*dma_dev
= sh_chan
->common
.device
;
273 struct sh_dmae_device
*shdev
= container_of(dma_dev
,
274 struct sh_dmae_device
, common
);
275 struct sh_dmae_pdata
*pdata
= shdev
->pdata
;
278 if ((unsigned)slave_id
>= SHDMA_SLAVE_NUMBER
)
281 for (i
= 0; i
< pdata
->slave_num
; i
++)
282 if (pdata
->slave
[i
].slave_id
== slave_id
)
283 return pdata
->slave
+ i
;
288 static int sh_dmae_alloc_chan_resources(struct dma_chan
*chan
)
290 struct sh_dmae_chan
*sh_chan
= to_sh_chan(chan
);
291 struct sh_desc
*desc
;
292 struct sh_dmae_slave
*param
= chan
->private;
295 pm_runtime_get_sync(sh_chan
->dev
);
298 * This relies on the guarantee from dmaengine that alloc_chan_resources
299 * never runs concurrently with itself or free_chan_resources.
302 struct sh_dmae_slave_config
*cfg
;
304 cfg
= sh_dmae_find_slave(sh_chan
, param
->slave_id
);
310 if (test_and_set_bit(param
->slave_id
, sh_dmae_slave_used
)) {
317 dmae_set_dmars(sh_chan
, cfg
->mid_rid
);
318 dmae_set_chcr(sh_chan
, cfg
->chcr
);
319 } else if ((sh_dmae_readl(sh_chan
, CHCR
) & 0xf00) != 0x400) {
323 spin_lock_bh(&sh_chan
->desc_lock
);
324 while (sh_chan
->descs_allocated
< NR_DESCS_PER_CHANNEL
) {
325 spin_unlock_bh(&sh_chan
->desc_lock
);
326 desc
= kzalloc(sizeof(struct sh_desc
), GFP_KERNEL
);
328 spin_lock_bh(&sh_chan
->desc_lock
);
331 dma_async_tx_descriptor_init(&desc
->async_tx
,
333 desc
->async_tx
.tx_submit
= sh_dmae_tx_submit
;
334 desc
->mark
= DESC_IDLE
;
336 spin_lock_bh(&sh_chan
->desc_lock
);
337 list_add(&desc
->node
, &sh_chan
->ld_free
);
338 sh_chan
->descs_allocated
++;
340 spin_unlock_bh(&sh_chan
->desc_lock
);
342 if (!sh_chan
->descs_allocated
) {
347 return sh_chan
->descs_allocated
;
351 clear_bit(param
->slave_id
, sh_dmae_slave_used
);
354 pm_runtime_put(sh_chan
->dev
);
359 * sh_dma_free_chan_resources - Free all resources of the channel.
361 static void sh_dmae_free_chan_resources(struct dma_chan
*chan
)
363 struct sh_dmae_chan
*sh_chan
= to_sh_chan(chan
);
364 struct sh_desc
*desc
, *_desc
;
366 int descs
= sh_chan
->descs_allocated
;
370 /* Prepared and not submitted descriptors can still be on the queue */
371 if (!list_empty(&sh_chan
->ld_queue
))
372 sh_dmae_chan_ld_cleanup(sh_chan
, true);
375 /* The caller is holding dma_list_mutex */
376 struct sh_dmae_slave
*param
= chan
->private;
377 clear_bit(param
->slave_id
, sh_dmae_slave_used
);
380 spin_lock_bh(&sh_chan
->desc_lock
);
382 list_splice_init(&sh_chan
->ld_free
, &list
);
383 sh_chan
->descs_allocated
= 0;
385 spin_unlock_bh(&sh_chan
->desc_lock
);
388 pm_runtime_put(sh_chan
->dev
);
390 list_for_each_entry_safe(desc
, _desc
, &list
, node
)
395 * sh_dmae_add_desc - get, set up and return one transfer descriptor
396 * @sh_chan: DMA channel
397 * @flags: DMA transfer flags
398 * @dest: destination DMA address, incremented when direction equals
399 * DMA_FROM_DEVICE or DMA_BIDIRECTIONAL
400 * @src: source DMA address, incremented when direction equals
401 * DMA_TO_DEVICE or DMA_BIDIRECTIONAL
402 * @len: DMA transfer length
403 * @first: if NULL, set to the current descriptor and cookie set to -EBUSY
404 * @direction: needed for slave DMA to decide which address to keep constant,
405 * equals DMA_BIDIRECTIONAL for MEMCPY
406 * Returns 0 or an error
407 * Locks: called with desc_lock held
409 static struct sh_desc
*sh_dmae_add_desc(struct sh_dmae_chan
*sh_chan
,
410 unsigned long flags
, dma_addr_t
*dest
, dma_addr_t
*src
, size_t *len
,
411 struct sh_desc
**first
, enum dma_data_direction direction
)
419 /* Allocate the link descriptor from the free list */
420 new = sh_dmae_get_desc(sh_chan
);
422 dev_err(sh_chan
->dev
, "No free link descriptor available\n");
426 copy_size
= min(*len
, (size_t)SH_DMA_TCR_MAX
+ 1);
430 new->hw
.tcr
= copy_size
;
434 new->async_tx
.cookie
= -EBUSY
;
437 /* Other desc - invisible to the user */
438 new->async_tx
.cookie
= -EINVAL
;
441 dev_dbg(sh_chan
->dev
,
442 "chaining (%u/%u)@%x -> %x with %p, cookie %d, shift %d\n",
443 copy_size
, *len
, *src
, *dest
, &new->async_tx
,
444 new->async_tx
.cookie
, sh_chan
->xmit_shift
);
446 new->mark
= DESC_PREPARED
;
447 new->async_tx
.flags
= flags
;
448 new->direction
= direction
;
451 if (direction
== DMA_BIDIRECTIONAL
|| direction
== DMA_TO_DEVICE
)
453 if (direction
== DMA_BIDIRECTIONAL
|| direction
== DMA_FROM_DEVICE
)
460 * sh_dmae_prep_sg - prepare transfer descriptors from an SG list
462 * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also
463 * converted to scatter-gather to guarantee consistent locking and a correct
464 * list manipulation. For slave DMA direction carries the usual meaning, and,
465 * logically, the SG list is RAM and the addr variable contains slave address,
466 * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_BIDIRECTIONAL
467 * and the SG list contains only one element and points at the source buffer.
469 static struct dma_async_tx_descriptor
*sh_dmae_prep_sg(struct sh_dmae_chan
*sh_chan
,
470 struct scatterlist
*sgl
, unsigned int sg_len
, dma_addr_t
*addr
,
471 enum dma_data_direction direction
, unsigned long flags
)
473 struct scatterlist
*sg
;
474 struct sh_desc
*first
= NULL
, *new = NULL
/* compiler... */;
482 for_each_sg(sgl
, sg
, sg_len
, i
)
483 chunks
+= (sg_dma_len(sg
) + SH_DMA_TCR_MAX
) /
484 (SH_DMA_TCR_MAX
+ 1);
486 /* Have to lock the whole loop to protect against concurrent release */
487 spin_lock_bh(&sh_chan
->desc_lock
);
491 * first descriptor is what user is dealing with in all API calls, its
492 * cookie is at first set to -EBUSY, at tx-submit to a positive
494 * if more than one chunk is needed further chunks have cookie = -EINVAL
495 * the last chunk, if not equal to the first, has cookie = -ENOSPC
496 * all chunks are linked onto the tx_list head with their .node heads
497 * only during this function, then they are immediately spliced
498 * back onto the free list in form of a chain
500 for_each_sg(sgl
, sg
, sg_len
, i
) {
501 dma_addr_t sg_addr
= sg_dma_address(sg
);
502 size_t len
= sg_dma_len(sg
);
508 dev_dbg(sh_chan
->dev
, "Add SG #%d@%p[%d], dma %llx\n",
509 i
, sg
, len
, (unsigned long long)sg_addr
);
511 if (direction
== DMA_FROM_DEVICE
)
512 new = sh_dmae_add_desc(sh_chan
, flags
,
513 &sg_addr
, addr
, &len
, &first
,
516 new = sh_dmae_add_desc(sh_chan
, flags
,
517 addr
, &sg_addr
, &len
, &first
,
522 new->chunks
= chunks
--;
523 list_add_tail(&new->node
, &tx_list
);
528 new->async_tx
.cookie
= -ENOSPC
;
530 /* Put them back on the free list, so, they don't get lost */
531 list_splice_tail(&tx_list
, &sh_chan
->ld_free
);
533 spin_unlock_bh(&sh_chan
->desc_lock
);
535 return &first
->async_tx
;
538 list_for_each_entry(new, &tx_list
, node
)
539 new->mark
= DESC_IDLE
;
540 list_splice(&tx_list
, &sh_chan
->ld_free
);
542 spin_unlock_bh(&sh_chan
->desc_lock
);
547 static struct dma_async_tx_descriptor
*sh_dmae_prep_memcpy(
548 struct dma_chan
*chan
, dma_addr_t dma_dest
, dma_addr_t dma_src
,
549 size_t len
, unsigned long flags
)
551 struct sh_dmae_chan
*sh_chan
;
552 struct scatterlist sg
;
557 chan
->private = NULL
;
559 sh_chan
= to_sh_chan(chan
);
561 sg_init_table(&sg
, 1);
562 sg_set_page(&sg
, pfn_to_page(PFN_DOWN(dma_src
)), len
,
563 offset_in_page(dma_src
));
564 sg_dma_address(&sg
) = dma_src
;
565 sg_dma_len(&sg
) = len
;
567 return sh_dmae_prep_sg(sh_chan
, &sg
, 1, &dma_dest
, DMA_BIDIRECTIONAL
,
571 static struct dma_async_tx_descriptor
*sh_dmae_prep_slave_sg(
572 struct dma_chan
*chan
, struct scatterlist
*sgl
, unsigned int sg_len
,
573 enum dma_data_direction direction
, unsigned long flags
)
575 struct sh_dmae_slave
*param
;
576 struct sh_dmae_chan
*sh_chan
;
581 sh_chan
= to_sh_chan(chan
);
582 param
= chan
->private;
584 /* Someone calling slave DMA on a public channel? */
585 if (!param
|| !sg_len
) {
586 dev_warn(sh_chan
->dev
, "%s: bad parameter: %p, %d, %d\n",
587 __func__
, param
, sg_len
, param
? param
->slave_id
: -1);
592 * if (param != NULL), this is a successfully requested slave channel,
593 * therefore param->config != NULL too.
595 return sh_dmae_prep_sg(sh_chan
, sgl
, sg_len
, ¶m
->config
->addr
,
599 static void sh_dmae_terminate_all(struct dma_chan
*chan
)
601 struct sh_dmae_chan
*sh_chan
= to_sh_chan(chan
);
608 spin_lock_bh(&sh_chan
->desc_lock
);
609 if (!list_empty(&sh_chan
->ld_queue
)) {
610 /* Record partial transfer */
611 struct sh_desc
*desc
= list_entry(sh_chan
->ld_queue
.next
,
612 struct sh_desc
, node
);
613 desc
->partial
= (desc
->hw
.tcr
- sh_dmae_readl(sh_chan
, TCR
)) <<
617 spin_unlock_bh(&sh_chan
->desc_lock
);
619 sh_dmae_chan_ld_cleanup(sh_chan
, true);
622 static dma_async_tx_callback
__ld_cleanup(struct sh_dmae_chan
*sh_chan
, bool all
)
624 struct sh_desc
*desc
, *_desc
;
625 /* Is the "exposed" head of a chain acked? */
626 bool head_acked
= false;
627 dma_cookie_t cookie
= 0;
628 dma_async_tx_callback callback
= NULL
;
631 spin_lock_bh(&sh_chan
->desc_lock
);
632 list_for_each_entry_safe(desc
, _desc
, &sh_chan
->ld_queue
, node
) {
633 struct dma_async_tx_descriptor
*tx
= &desc
->async_tx
;
635 BUG_ON(tx
->cookie
> 0 && tx
->cookie
!= desc
->cookie
);
636 BUG_ON(desc
->mark
!= DESC_SUBMITTED
&&
637 desc
->mark
!= DESC_COMPLETED
&&
638 desc
->mark
!= DESC_WAITING
);
641 * queue is ordered, and we use this loop to (1) clean up all
642 * completed descriptors, and to (2) update descriptor flags of
643 * any chunks in a (partially) completed chain
645 if (!all
&& desc
->mark
== DESC_SUBMITTED
&&
646 desc
->cookie
!= cookie
)
652 if (desc
->mark
== DESC_COMPLETED
&& desc
->chunks
== 1) {
653 if (sh_chan
->completed_cookie
!= desc
->cookie
- 1)
654 dev_dbg(sh_chan
->dev
,
655 "Completing cookie %d, expected %d\n",
657 sh_chan
->completed_cookie
+ 1);
658 sh_chan
->completed_cookie
= desc
->cookie
;
661 /* Call callback on the last chunk */
662 if (desc
->mark
== DESC_COMPLETED
&& tx
->callback
) {
663 desc
->mark
= DESC_WAITING
;
664 callback
= tx
->callback
;
665 param
= tx
->callback_param
;
666 dev_dbg(sh_chan
->dev
, "descriptor #%d@%p on %d callback\n",
667 tx
->cookie
, tx
, sh_chan
->id
);
668 BUG_ON(desc
->chunks
!= 1);
672 if (tx
->cookie
> 0 || tx
->cookie
== -EBUSY
) {
673 if (desc
->mark
== DESC_COMPLETED
) {
674 BUG_ON(tx
->cookie
< 0);
675 desc
->mark
= DESC_WAITING
;
677 head_acked
= async_tx_test_ack(tx
);
679 switch (desc
->mark
) {
681 desc
->mark
= DESC_WAITING
;
685 async_tx_ack(&desc
->async_tx
);
689 dev_dbg(sh_chan
->dev
, "descriptor %p #%d completed.\n",
692 if (((desc
->mark
== DESC_COMPLETED
||
693 desc
->mark
== DESC_WAITING
) &&
694 async_tx_test_ack(&desc
->async_tx
)) || all
) {
695 /* Remove from ld_queue list */
696 desc
->mark
= DESC_IDLE
;
697 list_move(&desc
->node
, &sh_chan
->ld_free
);
700 spin_unlock_bh(&sh_chan
->desc_lock
);
709 * sh_chan_ld_cleanup - Clean up link descriptors
711 * This function cleans up the ld_queue of DMA channel.
713 static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan
*sh_chan
, bool all
)
715 while (__ld_cleanup(sh_chan
, all
))
719 static void sh_chan_xfer_ld_queue(struct sh_dmae_chan
*sh_chan
)
721 struct sh_desc
*desc
;
723 spin_lock_bh(&sh_chan
->desc_lock
);
725 if (dmae_is_busy(sh_chan
)) {
726 spin_unlock_bh(&sh_chan
->desc_lock
);
730 /* Find the first not transferred desciptor */
731 list_for_each_entry(desc
, &sh_chan
->ld_queue
, node
)
732 if (desc
->mark
== DESC_SUBMITTED
) {
733 dev_dbg(sh_chan
->dev
, "Queue #%d to %d: %u@%x -> %x\n",
734 desc
->async_tx
.cookie
, sh_chan
->id
,
735 desc
->hw
.tcr
, desc
->hw
.sar
, desc
->hw
.dar
);
736 /* Get the ld start address from ld_queue */
737 dmae_set_reg(sh_chan
, &desc
->hw
);
742 spin_unlock_bh(&sh_chan
->desc_lock
);
745 static void sh_dmae_memcpy_issue_pending(struct dma_chan
*chan
)
747 struct sh_dmae_chan
*sh_chan
= to_sh_chan(chan
);
748 sh_chan_xfer_ld_queue(sh_chan
);
751 static enum dma_status
sh_dmae_is_complete(struct dma_chan
*chan
,
756 struct sh_dmae_chan
*sh_chan
= to_sh_chan(chan
);
757 dma_cookie_t last_used
;
758 dma_cookie_t last_complete
;
759 enum dma_status status
;
761 sh_dmae_chan_ld_cleanup(sh_chan
, false);
763 last_used
= chan
->cookie
;
764 last_complete
= sh_chan
->completed_cookie
;
765 BUG_ON(last_complete
< 0);
768 *done
= last_complete
;
773 spin_lock_bh(&sh_chan
->desc_lock
);
775 status
= dma_async_is_complete(cookie
, last_complete
, last_used
);
778 * If we don't find cookie on the queue, it has been aborted and we have
781 if (status
!= DMA_SUCCESS
) {
782 struct sh_desc
*desc
;
784 list_for_each_entry(desc
, &sh_chan
->ld_queue
, node
)
785 if (desc
->cookie
== cookie
) {
786 status
= DMA_IN_PROGRESS
;
791 spin_unlock_bh(&sh_chan
->desc_lock
);
796 static irqreturn_t
sh_dmae_interrupt(int irq
, void *data
)
798 irqreturn_t ret
= IRQ_NONE
;
799 struct sh_dmae_chan
*sh_chan
= (struct sh_dmae_chan
*)data
;
800 u32 chcr
= sh_dmae_readl(sh_chan
, CHCR
);
802 if (chcr
& CHCR_TE
) {
807 tasklet_schedule(&sh_chan
->tasklet
);
813 #if defined(CONFIG_CPU_SH4)
814 static irqreturn_t
sh_dmae_err(int irq
, void *data
)
816 struct sh_dmae_device
*shdev
= (struct sh_dmae_device
*)data
;
819 /* halt the dma controller */
820 sh_dmae_ctl_stop(shdev
);
822 /* We cannot detect, which channel caused the error, have to reset all */
823 for (i
= 0; i
< SH_DMAC_MAX_CHANNELS
; i
++) {
824 struct sh_dmae_chan
*sh_chan
= shdev
->chan
[i
];
826 struct sh_desc
*desc
;
827 /* Stop the channel */
830 list_for_each_entry(desc
, &sh_chan
->ld_queue
, node
) {
831 struct dma_async_tx_descriptor
*tx
= &desc
->async_tx
;
832 desc
->mark
= DESC_IDLE
;
834 tx
->callback(tx
->callback_param
);
836 list_splice_init(&sh_chan
->ld_queue
, &sh_chan
->ld_free
);
845 static void dmae_do_tasklet(unsigned long data
)
847 struct sh_dmae_chan
*sh_chan
= (struct sh_dmae_chan
*)data
;
848 struct sh_desc
*desc
;
849 u32 sar_buf
= sh_dmae_readl(sh_chan
, SAR
);
850 u32 dar_buf
= sh_dmae_readl(sh_chan
, DAR
);
852 spin_lock(&sh_chan
->desc_lock
);
853 list_for_each_entry(desc
, &sh_chan
->ld_queue
, node
) {
854 if (desc
->mark
== DESC_SUBMITTED
&&
855 ((desc
->direction
== DMA_FROM_DEVICE
&&
856 (desc
->hw
.dar
+ desc
->hw
.tcr
) == dar_buf
) ||
857 (desc
->hw
.sar
+ desc
->hw
.tcr
) == sar_buf
)) {
858 dev_dbg(sh_chan
->dev
, "done #%d@%p dst %u\n",
859 desc
->async_tx
.cookie
, &desc
->async_tx
,
861 desc
->mark
= DESC_COMPLETED
;
865 spin_unlock(&sh_chan
->desc_lock
);
868 sh_chan_xfer_ld_queue(sh_chan
);
869 sh_dmae_chan_ld_cleanup(sh_chan
, false);
872 static int __devinit
sh_dmae_chan_probe(struct sh_dmae_device
*shdev
, int id
,
873 int irq
, unsigned long flags
)
876 struct sh_dmae_channel
*chan_pdata
= &shdev
->pdata
->channel
[id
];
877 struct platform_device
*pdev
= to_platform_device(shdev
->common
.dev
);
878 struct sh_dmae_chan
*new_sh_chan
;
881 new_sh_chan
= kzalloc(sizeof(struct sh_dmae_chan
), GFP_KERNEL
);
883 dev_err(shdev
->common
.dev
,
884 "No free memory for allocating dma channels!\n");
888 /* copy struct dma_device */
889 new_sh_chan
->common
.device
= &shdev
->common
;
891 new_sh_chan
->dev
= shdev
->common
.dev
;
892 new_sh_chan
->id
= id
;
893 new_sh_chan
->irq
= irq
;
894 new_sh_chan
->base
= shdev
->chan_reg
+ chan_pdata
->offset
/ sizeof(u32
);
896 /* Init DMA tasklet */
897 tasklet_init(&new_sh_chan
->tasklet
, dmae_do_tasklet
,
898 (unsigned long)new_sh_chan
);
900 /* Init the channel */
901 dmae_init(new_sh_chan
);
903 spin_lock_init(&new_sh_chan
->desc_lock
);
905 /* Init descripter manage list */
906 INIT_LIST_HEAD(&new_sh_chan
->ld_queue
);
907 INIT_LIST_HEAD(&new_sh_chan
->ld_free
);
909 /* Add the channel to DMA device channel list */
910 list_add_tail(&new_sh_chan
->common
.device_node
,
911 &shdev
->common
.channels
);
912 shdev
->common
.chancnt
++;
915 snprintf(new_sh_chan
->dev_id
, sizeof(new_sh_chan
->dev_id
),
916 "sh-dmae%d.%d", pdev
->id
, new_sh_chan
->id
);
918 snprintf(new_sh_chan
->dev_id
, sizeof(new_sh_chan
->dev_id
),
919 "sh-dma%d", new_sh_chan
->id
);
921 /* set up channel irq */
922 err
= request_irq(irq
, &sh_dmae_interrupt
, flags
,
923 new_sh_chan
->dev_id
, new_sh_chan
);
925 dev_err(shdev
->common
.dev
, "DMA channel %d request_irq error "
926 "with return %d\n", id
, err
);
930 shdev
->chan
[id
] = new_sh_chan
;
934 /* remove from dmaengine device node */
935 list_del(&new_sh_chan
->common
.device_node
);
940 static void sh_dmae_chan_remove(struct sh_dmae_device
*shdev
)
944 for (i
= shdev
->common
.chancnt
- 1 ; i
>= 0 ; i
--) {
945 if (shdev
->chan
[i
]) {
946 struct sh_dmae_chan
*sh_chan
= shdev
->chan
[i
];
948 free_irq(sh_chan
->irq
, sh_chan
);
950 list_del(&sh_chan
->common
.device_node
);
952 shdev
->chan
[i
] = NULL
;
955 shdev
->common
.chancnt
= 0;
958 static int __init
sh_dmae_probe(struct platform_device
*pdev
)
960 struct sh_dmae_pdata
*pdata
= pdev
->dev
.platform_data
;
961 unsigned long irqflags
= IRQF_DISABLED
,
962 chan_flag
[SH_DMAC_MAX_CHANNELS
] = {};
963 int errirq
, chan_irq
[SH_DMAC_MAX_CHANNELS
];
964 int err
, i
, irq_cnt
= 0, irqres
= 0;
965 struct sh_dmae_device
*shdev
;
966 struct resource
*chan
, *dmars
, *errirq_res
, *chanirq_res
;
968 /* get platform data */
969 if (!pdata
|| !pdata
->channel_num
)
972 chan
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
973 /* DMARS area is optional, if absent, this controller cannot do slave DMA */
974 dmars
= platform_get_resource(pdev
, IORESOURCE_MEM
, 1);
977 * 1. there always must be at least one IRQ IO-resource. On SH4 it is
978 * the error IRQ, in which case it is the only IRQ in this resource:
979 * start == end. If it is the only IRQ resource, all channels also
981 * 2. DMA channel IRQ resources can be specified one per resource or in
982 * ranges (start != end)
983 * 3. iff all events (channels and, optionally, error) on this
984 * controller use the same IRQ, only one IRQ resource can be
985 * specified, otherwise there must be one IRQ per channel, even if
986 * some of them are equal
987 * 4. if all IRQs on this controller are equal or if some specific IRQs
988 * specify IORESOURCE_IRQ_SHAREABLE in their resources, they will be
989 * requested with the IRQF_SHARED flag
991 errirq_res
= platform_get_resource(pdev
, IORESOURCE_IRQ
, 0);
992 if (!chan
|| !errirq_res
)
995 if (!request_mem_region(chan
->start
, resource_size(chan
), pdev
->name
)) {
996 dev_err(&pdev
->dev
, "DMAC register region already claimed\n");
1000 if (dmars
&& !request_mem_region(dmars
->start
, resource_size(dmars
), pdev
->name
)) {
1001 dev_err(&pdev
->dev
, "DMAC DMARS region already claimed\n");
1007 shdev
= kzalloc(sizeof(struct sh_dmae_device
), GFP_KERNEL
);
1009 dev_err(&pdev
->dev
, "Not enough memory\n");
1013 shdev
->chan_reg
= ioremap(chan
->start
, resource_size(chan
));
1014 if (!shdev
->chan_reg
)
1017 shdev
->dmars
= ioremap(dmars
->start
, resource_size(dmars
));
1023 shdev
->pdata
= pdata
;
1025 pm_runtime_enable(&pdev
->dev
);
1026 pm_runtime_get_sync(&pdev
->dev
);
1028 /* reset dma controller */
1029 err
= sh_dmae_rst(shdev
);
1033 INIT_LIST_HEAD(&shdev
->common
.channels
);
1035 dma_cap_set(DMA_MEMCPY
, shdev
->common
.cap_mask
);
1037 dma_cap_set(DMA_SLAVE
, shdev
->common
.cap_mask
);
1039 shdev
->common
.device_alloc_chan_resources
1040 = sh_dmae_alloc_chan_resources
;
1041 shdev
->common
.device_free_chan_resources
= sh_dmae_free_chan_resources
;
1042 shdev
->common
.device_prep_dma_memcpy
= sh_dmae_prep_memcpy
;
1043 shdev
->common
.device_is_tx_complete
= sh_dmae_is_complete
;
1044 shdev
->common
.device_issue_pending
= sh_dmae_memcpy_issue_pending
;
1046 /* Compulsory for DMA_SLAVE fields */
1047 shdev
->common
.device_prep_slave_sg
= sh_dmae_prep_slave_sg
;
1048 shdev
->common
.device_terminate_all
= sh_dmae_terminate_all
;
1050 shdev
->common
.dev
= &pdev
->dev
;
1051 /* Default transfer size of 32 bytes requires 32-byte alignment */
1052 shdev
->common
.copy_align
= LOG2_DEFAULT_XFER_SIZE
;
1054 #if defined(CONFIG_CPU_SH4)
1055 chanirq_res
= platform_get_resource(pdev
, IORESOURCE_IRQ
, 1);
1058 chanirq_res
= errirq_res
;
1062 if (chanirq_res
== errirq_res
||
1063 (errirq_res
->flags
& IORESOURCE_BITS
) == IORESOURCE_IRQ_SHAREABLE
)
1064 irqflags
= IRQF_SHARED
;
1066 errirq
= errirq_res
->start
;
1068 err
= request_irq(errirq
, sh_dmae_err
, irqflags
,
1069 "DMAC Address Error", shdev
);
1072 "DMA failed requesting irq #%d, error %d\n",
1078 chanirq_res
= errirq_res
;
1079 #endif /* CONFIG_CPU_SH4 */
1081 if (chanirq_res
->start
== chanirq_res
->end
&&
1082 !platform_get_resource(pdev
, IORESOURCE_IRQ
, 1)) {
1083 /* Special case - all multiplexed */
1084 for (; irq_cnt
< pdata
->channel_num
; irq_cnt
++) {
1085 chan_irq
[irq_cnt
] = chanirq_res
->start
;
1086 chan_flag
[irq_cnt
] = IRQF_SHARED
;
1090 for (i
= chanirq_res
->start
; i
<= chanirq_res
->end
; i
++) {
1091 if ((errirq_res
->flags
& IORESOURCE_BITS
) ==
1092 IORESOURCE_IRQ_SHAREABLE
)
1093 chan_flag
[irq_cnt
] = IRQF_SHARED
;
1095 chan_flag
[irq_cnt
] = IRQF_DISABLED
;
1097 "Found IRQ %d for channel %d\n",
1099 chan_irq
[irq_cnt
++] = i
;
1101 chanirq_res
= platform_get_resource(pdev
,
1102 IORESOURCE_IRQ
, ++irqres
);
1103 } while (irq_cnt
< pdata
->channel_num
&& chanirq_res
);
1106 if (irq_cnt
< pdata
->channel_num
)
1109 /* Create DMA Channel */
1110 for (i
= 0; i
< pdata
->channel_num
; i
++) {
1111 err
= sh_dmae_chan_probe(shdev
, i
, chan_irq
[i
], chan_flag
[i
]);
1113 goto chan_probe_err
;
1116 pm_runtime_put(&pdev
->dev
);
1118 platform_set_drvdata(pdev
, shdev
);
1119 dma_async_device_register(&shdev
->common
);
1124 sh_dmae_chan_remove(shdev
);
1126 #if defined(CONFIG_CPU_SH4)
1127 free_irq(errirq
, shdev
);
1131 pm_runtime_put(&pdev
->dev
);
1133 iounmap(shdev
->dmars
);
1135 iounmap(shdev
->chan_reg
);
1140 release_mem_region(dmars
->start
, resource_size(dmars
));
1142 release_mem_region(chan
->start
, resource_size(chan
));
1147 static int __exit
sh_dmae_remove(struct platform_device
*pdev
)
1149 struct sh_dmae_device
*shdev
= platform_get_drvdata(pdev
);
1150 struct resource
*res
;
1151 int errirq
= platform_get_irq(pdev
, 0);
1153 dma_async_device_unregister(&shdev
->common
);
1156 free_irq(errirq
, shdev
);
1158 /* channel data remove */
1159 sh_dmae_chan_remove(shdev
);
1161 pm_runtime_disable(&pdev
->dev
);
1164 iounmap(shdev
->dmars
);
1165 iounmap(shdev
->chan_reg
);
1169 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1171 release_mem_region(res
->start
, resource_size(res
));
1172 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 1);
1174 release_mem_region(res
->start
, resource_size(res
));
1179 static void sh_dmae_shutdown(struct platform_device
*pdev
)
1181 struct sh_dmae_device
*shdev
= platform_get_drvdata(pdev
);
1182 sh_dmae_ctl_stop(shdev
);
1185 static struct platform_driver sh_dmae_driver
= {
1186 .remove
= __exit_p(sh_dmae_remove
),
1187 .shutdown
= sh_dmae_shutdown
,
1189 .name
= "sh-dma-engine",
1193 static int __init
sh_dmae_init(void)
1195 return platform_driver_probe(&sh_dmae_driver
, sh_dmae_probe
);
1197 module_init(sh_dmae_init
);
1199 static void __exit
sh_dmae_exit(void)
1201 platform_driver_unregister(&sh_dmae_driver
);
1203 module_exit(sh_dmae_exit
);
1205 MODULE_AUTHOR("Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>");
1206 MODULE_DESCRIPTION("Renesas SH DMA Engine driver");
1207 MODULE_LICENSE("GPL");