2 * DMA driver for Nvidia's Tegra20 APB DMA controller.
4 * Copyright (c) 2012, NVIDIA CORPORATION. All rights reserved.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 #include <linux/bitops.h>
20 #include <linux/clk.h>
21 #include <linux/delay.h>
22 #include <linux/dmaengine.h>
23 #include <linux/dma-mapping.h>
24 #include <linux/err.h>
25 #include <linux/init.h>
26 #include <linux/interrupt.h>
29 #include <linux/module.h>
31 #include <linux/of_device.h>
32 #include <linux/platform_device.h>
33 #include <linux/pm_runtime.h>
34 #include <linux/slab.h>
37 #include "dmaengine.h"
39 #define TEGRA_APBDMA_GENERAL 0x0
40 #define TEGRA_APBDMA_GENERAL_ENABLE BIT(31)
42 #define TEGRA_APBDMA_CONTROL 0x010
43 #define TEGRA_APBDMA_IRQ_MASK 0x01c
44 #define TEGRA_APBDMA_IRQ_MASK_SET 0x020
47 #define TEGRA_APBDMA_CHAN_CSR 0x00
48 #define TEGRA_APBDMA_CSR_ENB BIT(31)
49 #define TEGRA_APBDMA_CSR_IE_EOC BIT(30)
50 #define TEGRA_APBDMA_CSR_HOLD BIT(29)
51 #define TEGRA_APBDMA_CSR_DIR BIT(28)
52 #define TEGRA_APBDMA_CSR_ONCE BIT(27)
53 #define TEGRA_APBDMA_CSR_FLOW BIT(21)
54 #define TEGRA_APBDMA_CSR_REQ_SEL_SHIFT 16
55 #define TEGRA_APBDMA_CSR_WCOUNT_MASK 0xFFFC
58 #define TEGRA_APBDMA_CHAN_STATUS 0x004
59 #define TEGRA_APBDMA_STATUS_BUSY BIT(31)
60 #define TEGRA_APBDMA_STATUS_ISE_EOC BIT(30)
61 #define TEGRA_APBDMA_STATUS_HALT BIT(29)
62 #define TEGRA_APBDMA_STATUS_PING_PONG BIT(28)
63 #define TEGRA_APBDMA_STATUS_COUNT_SHIFT 2
64 #define TEGRA_APBDMA_STATUS_COUNT_MASK 0xFFFC
66 /* AHB memory address */
67 #define TEGRA_APBDMA_CHAN_AHBPTR 0x010
69 /* AHB sequence register */
70 #define TEGRA_APBDMA_CHAN_AHBSEQ 0x14
71 #define TEGRA_APBDMA_AHBSEQ_INTR_ENB BIT(31)
72 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_8 (0 << 28)
73 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_16 (1 << 28)
74 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32 (2 << 28)
75 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_64 (3 << 28)
76 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_128 (4 << 28)
77 #define TEGRA_APBDMA_AHBSEQ_DATA_SWAP BIT(27)
78 #define TEGRA_APBDMA_AHBSEQ_BURST_1 (4 << 24)
79 #define TEGRA_APBDMA_AHBSEQ_BURST_4 (5 << 24)
80 #define TEGRA_APBDMA_AHBSEQ_BURST_8 (6 << 24)
81 #define TEGRA_APBDMA_AHBSEQ_DBL_BUF BIT(19)
82 #define TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT 16
83 #define TEGRA_APBDMA_AHBSEQ_WRAP_NONE 0
86 #define TEGRA_APBDMA_CHAN_APBPTR 0x018
88 /* APB sequence register */
89 #define TEGRA_APBDMA_CHAN_APBSEQ 0x01c
90 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_8 (0 << 28)
91 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_16 (1 << 28)
92 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32 (2 << 28)
93 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64 (3 << 28)
94 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_128 (4 << 28)
95 #define TEGRA_APBDMA_APBSEQ_DATA_SWAP BIT(27)
96 #define TEGRA_APBDMA_APBSEQ_WRAP_WORD_1 (1 << 16)
99 * If any burst is in flight and DMA paused then this is the time to complete
100 * on-flight burst and update DMA status register.
102 #define TEGRA_APBDMA_BURST_COMPLETE_TIME 20
104 /* Channel base address offset from APBDMA base address */
105 #define TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET 0x1000
107 /* DMA channel register space size */
108 #define TEGRA_APBDMA_CHANNEL_REGISTER_SIZE 0x20
113 * tegra_dma_chip_data Tegra chip specific DMA data
114 * @nr_channels: Number of channels available in the controller.
115 * @max_dma_count: Maximum DMA transfer count supported by DMA controller.
117 struct tegra_dma_chip_data
{
122 /* DMA channel registers */
123 struct tegra_dma_channel_regs
{
125 unsigned long ahb_ptr
;
126 unsigned long apb_ptr
;
127 unsigned long ahb_seq
;
128 unsigned long apb_seq
;
132 * tegra_dma_sg_req: Dma request details to configure hardware. This
133 * contains the details for one transfer to configure DMA hw.
134 * The client's request for data transfer can be broken into multiple
135 * sub-transfer as per requester details and hw support.
136 * This sub transfer get added in the list of transfer and point to Tegra
137 * DMA descriptor which manages the transfer details.
139 struct tegra_dma_sg_req
{
140 struct tegra_dma_channel_regs ch_regs
;
145 struct list_head node
;
146 struct tegra_dma_desc
*dma_desc
;
150 * tegra_dma_desc: Tegra DMA descriptors which manages the client requests.
151 * This descriptor keep track of transfer status, callbacks and request
154 struct tegra_dma_desc
{
155 struct dma_async_tx_descriptor txd
;
157 int bytes_transferred
;
158 enum dma_status dma_status
;
159 struct list_head node
;
160 struct list_head tx_list
;
161 struct list_head cb_node
;
165 struct tegra_dma_channel
;
167 typedef void (*dma_isr_handler
)(struct tegra_dma_channel
*tdc
,
170 /* tegra_dma_channel: Channel specific information */
171 struct tegra_dma_channel
{
172 struct dma_chan dma_chan
;
177 unsigned long chan_base_offset
;
180 struct tegra_dma
*tdma
;
183 /* Different lists for managing the requests */
184 struct list_head free_sg_req
;
185 struct list_head pending_sg_req
;
186 struct list_head free_dma_desc
;
187 struct list_head cb_desc
;
189 /* ISR handler and tasklet for bottom half of isr handling */
190 dma_isr_handler isr_handler
;
191 struct tasklet_struct tasklet
;
192 dma_async_tx_callback callback
;
193 void *callback_param
;
195 /* Channel-slave specific configuration */
196 struct dma_slave_config dma_sconfig
;
199 /* tegra_dma: Tegra DMA specific information */
201 struct dma_device dma_dev
;
204 spinlock_t global_lock
;
205 void __iomem
*base_addr
;
206 const struct tegra_dma_chip_data
*chip_data
;
208 /* Some register need to be cache before suspend */
211 /* Last member of the structure */
212 struct tegra_dma_channel channels
[0];
215 static inline void tdma_write(struct tegra_dma
*tdma
, u32 reg
, u32 val
)
217 writel(val
, tdma
->base_addr
+ reg
);
220 static inline u32
tdma_read(struct tegra_dma
*tdma
, u32 reg
)
222 return readl(tdma
->base_addr
+ reg
);
225 static inline void tdc_write(struct tegra_dma_channel
*tdc
,
228 writel(val
, tdc
->tdma
->base_addr
+ tdc
->chan_base_offset
+ reg
);
231 static inline u32
tdc_read(struct tegra_dma_channel
*tdc
, u32 reg
)
233 return readl(tdc
->tdma
->base_addr
+ tdc
->chan_base_offset
+ reg
);
236 static inline struct tegra_dma_channel
*to_tegra_dma_chan(struct dma_chan
*dc
)
238 return container_of(dc
, struct tegra_dma_channel
, dma_chan
);
241 static inline struct tegra_dma_desc
*txd_to_tegra_dma_desc(
242 struct dma_async_tx_descriptor
*td
)
244 return container_of(td
, struct tegra_dma_desc
, txd
);
247 static inline struct device
*tdc2dev(struct tegra_dma_channel
*tdc
)
249 return &tdc
->dma_chan
.dev
->device
;
252 static dma_cookie_t
tegra_dma_tx_submit(struct dma_async_tx_descriptor
*tx
);
253 static int tegra_dma_runtime_suspend(struct device
*dev
);
254 static int tegra_dma_runtime_resume(struct device
*dev
);
256 /* Get DMA desc from free list, if not there then allocate it. */
257 static struct tegra_dma_desc
*tegra_dma_desc_get(
258 struct tegra_dma_channel
*tdc
)
260 struct tegra_dma_desc
*dma_desc
;
263 spin_lock_irqsave(&tdc
->lock
, flags
);
265 /* Do not allocate if desc are waiting for ack */
266 list_for_each_entry(dma_desc
, &tdc
->free_dma_desc
, node
) {
267 if (async_tx_test_ack(&dma_desc
->txd
)) {
268 list_del(&dma_desc
->node
);
269 spin_unlock_irqrestore(&tdc
->lock
, flags
);
274 spin_unlock_irqrestore(&tdc
->lock
, flags
);
276 /* Allocate DMA desc */
277 dma_desc
= kzalloc(sizeof(*dma_desc
), GFP_ATOMIC
);
279 dev_err(tdc2dev(tdc
), "dma_desc alloc failed\n");
283 dma_async_tx_descriptor_init(&dma_desc
->txd
, &tdc
->dma_chan
);
284 dma_desc
->txd
.tx_submit
= tegra_dma_tx_submit
;
285 dma_desc
->txd
.flags
= 0;
289 static void tegra_dma_desc_put(struct tegra_dma_channel
*tdc
,
290 struct tegra_dma_desc
*dma_desc
)
294 spin_lock_irqsave(&tdc
->lock
, flags
);
295 if (!list_empty(&dma_desc
->tx_list
))
296 list_splice_init(&dma_desc
->tx_list
, &tdc
->free_sg_req
);
297 list_add_tail(&dma_desc
->node
, &tdc
->free_dma_desc
);
298 spin_unlock_irqrestore(&tdc
->lock
, flags
);
301 static struct tegra_dma_sg_req
*tegra_dma_sg_req_get(
302 struct tegra_dma_channel
*tdc
)
304 struct tegra_dma_sg_req
*sg_req
= NULL
;
307 spin_lock_irqsave(&tdc
->lock
, flags
);
308 if (!list_empty(&tdc
->free_sg_req
)) {
309 sg_req
= list_first_entry(&tdc
->free_sg_req
,
310 typeof(*sg_req
), node
);
311 list_del(&sg_req
->node
);
312 spin_unlock_irqrestore(&tdc
->lock
, flags
);
315 spin_unlock_irqrestore(&tdc
->lock
, flags
);
317 sg_req
= kzalloc(sizeof(struct tegra_dma_sg_req
), GFP_ATOMIC
);
319 dev_err(tdc2dev(tdc
), "sg_req alloc failed\n");
323 static int tegra_dma_slave_config(struct dma_chan
*dc
,
324 struct dma_slave_config
*sconfig
)
326 struct tegra_dma_channel
*tdc
= to_tegra_dma_chan(dc
);
328 if (!list_empty(&tdc
->pending_sg_req
)) {
329 dev_err(tdc2dev(tdc
), "Configuration not allowed\n");
333 memcpy(&tdc
->dma_sconfig
, sconfig
, sizeof(*sconfig
));
334 tdc
->config_init
= true;
338 static void tegra_dma_global_pause(struct tegra_dma_channel
*tdc
,
339 bool wait_for_burst_complete
)
341 struct tegra_dma
*tdma
= tdc
->tdma
;
343 spin_lock(&tdma
->global_lock
);
344 tdma_write(tdma
, TEGRA_APBDMA_GENERAL
, 0);
345 if (wait_for_burst_complete
)
346 udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME
);
349 static void tegra_dma_global_resume(struct tegra_dma_channel
*tdc
)
351 struct tegra_dma
*tdma
= tdc
->tdma
;
353 tdma_write(tdma
, TEGRA_APBDMA_GENERAL
, TEGRA_APBDMA_GENERAL_ENABLE
);
354 spin_unlock(&tdma
->global_lock
);
357 static void tegra_dma_stop(struct tegra_dma_channel
*tdc
)
362 /* Disable interrupts */
363 csr
= tdc_read(tdc
, TEGRA_APBDMA_CHAN_CSR
);
364 csr
&= ~TEGRA_APBDMA_CSR_IE_EOC
;
365 tdc_write(tdc
, TEGRA_APBDMA_CHAN_CSR
, csr
);
368 csr
&= ~TEGRA_APBDMA_CSR_ENB
;
369 tdc_write(tdc
, TEGRA_APBDMA_CHAN_CSR
, csr
);
371 /* Clear interrupt status if it is there */
372 status
= tdc_read(tdc
, TEGRA_APBDMA_CHAN_STATUS
);
373 if (status
& TEGRA_APBDMA_STATUS_ISE_EOC
) {
374 dev_dbg(tdc2dev(tdc
), "%s():clearing interrupt\n", __func__
);
375 tdc_write(tdc
, TEGRA_APBDMA_CHAN_STATUS
, status
);
380 static void tegra_dma_start(struct tegra_dma_channel
*tdc
,
381 struct tegra_dma_sg_req
*sg_req
)
383 struct tegra_dma_channel_regs
*ch_regs
= &sg_req
->ch_regs
;
385 tdc_write(tdc
, TEGRA_APBDMA_CHAN_CSR
, ch_regs
->csr
);
386 tdc_write(tdc
, TEGRA_APBDMA_CHAN_APBSEQ
, ch_regs
->apb_seq
);
387 tdc_write(tdc
, TEGRA_APBDMA_CHAN_APBPTR
, ch_regs
->apb_ptr
);
388 tdc_write(tdc
, TEGRA_APBDMA_CHAN_AHBSEQ
, ch_regs
->ahb_seq
);
389 tdc_write(tdc
, TEGRA_APBDMA_CHAN_AHBPTR
, ch_regs
->ahb_ptr
);
392 tdc_write(tdc
, TEGRA_APBDMA_CHAN_CSR
,
393 ch_regs
->csr
| TEGRA_APBDMA_CSR_ENB
);
396 static void tegra_dma_configure_for_next(struct tegra_dma_channel
*tdc
,
397 struct tegra_dma_sg_req
*nsg_req
)
399 unsigned long status
;
402 * The DMA controller reloads the new configuration for next transfer
403 * after last burst of current transfer completes.
404 * If there is no IEC status then this makes sure that last burst
405 * has not be completed. There may be case that last burst is on
406 * flight and so it can complete but because DMA is paused, it
407 * will not generates interrupt as well as not reload the new
409 * If there is already IEC status then interrupt handler need to
410 * load new configuration.
412 tegra_dma_global_pause(tdc
, false);
413 status
= tdc_read(tdc
, TEGRA_APBDMA_CHAN_STATUS
);
416 * If interrupt is pending then do nothing as the ISR will handle
417 * the programing for new request.
419 if (status
& TEGRA_APBDMA_STATUS_ISE_EOC
) {
420 dev_err(tdc2dev(tdc
),
421 "Skipping new configuration as interrupt is pending\n");
422 tegra_dma_global_resume(tdc
);
426 /* Safe to program new configuration */
427 tdc_write(tdc
, TEGRA_APBDMA_CHAN_APBPTR
, nsg_req
->ch_regs
.apb_ptr
);
428 tdc_write(tdc
, TEGRA_APBDMA_CHAN_AHBPTR
, nsg_req
->ch_regs
.ahb_ptr
);
429 tdc_write(tdc
, TEGRA_APBDMA_CHAN_CSR
,
430 nsg_req
->ch_regs
.csr
| TEGRA_APBDMA_CSR_ENB
);
431 nsg_req
->configured
= true;
433 tegra_dma_global_resume(tdc
);
436 static void tdc_start_head_req(struct tegra_dma_channel
*tdc
)
438 struct tegra_dma_sg_req
*sg_req
;
440 if (list_empty(&tdc
->pending_sg_req
))
443 sg_req
= list_first_entry(&tdc
->pending_sg_req
,
444 typeof(*sg_req
), node
);
445 tegra_dma_start(tdc
, sg_req
);
446 sg_req
->configured
= true;
450 static void tdc_configure_next_head_desc(struct tegra_dma_channel
*tdc
)
452 struct tegra_dma_sg_req
*hsgreq
;
453 struct tegra_dma_sg_req
*hnsgreq
;
455 if (list_empty(&tdc
->pending_sg_req
))
458 hsgreq
= list_first_entry(&tdc
->pending_sg_req
, typeof(*hsgreq
), node
);
459 if (!list_is_last(&hsgreq
->node
, &tdc
->pending_sg_req
)) {
460 hnsgreq
= list_first_entry(&hsgreq
->node
,
461 typeof(*hnsgreq
), node
);
462 tegra_dma_configure_for_next(tdc
, hnsgreq
);
466 static inline int get_current_xferred_count(struct tegra_dma_channel
*tdc
,
467 struct tegra_dma_sg_req
*sg_req
, unsigned long status
)
469 return sg_req
->req_len
- (status
& TEGRA_APBDMA_STATUS_COUNT_MASK
) - 4;
472 static void tegra_dma_abort_all(struct tegra_dma_channel
*tdc
)
474 struct tegra_dma_sg_req
*sgreq
;
475 struct tegra_dma_desc
*dma_desc
;
477 while (!list_empty(&tdc
->pending_sg_req
)) {
478 sgreq
= list_first_entry(&tdc
->pending_sg_req
,
479 typeof(*sgreq
), node
);
480 list_move_tail(&sgreq
->node
, &tdc
->free_sg_req
);
481 if (sgreq
->last_sg
) {
482 dma_desc
= sgreq
->dma_desc
;
483 dma_desc
->dma_status
= DMA_ERROR
;
484 list_add_tail(&dma_desc
->node
, &tdc
->free_dma_desc
);
486 /* Add in cb list if it is not there. */
487 if (!dma_desc
->cb_count
)
488 list_add_tail(&dma_desc
->cb_node
,
490 dma_desc
->cb_count
++;
493 tdc
->isr_handler
= NULL
;
496 static bool handle_continuous_head_request(struct tegra_dma_channel
*tdc
,
497 struct tegra_dma_sg_req
*last_sg_req
, bool to_terminate
)
499 struct tegra_dma_sg_req
*hsgreq
= NULL
;
501 if (list_empty(&tdc
->pending_sg_req
)) {
502 dev_err(tdc2dev(tdc
), "Dma is running without req\n");
508 * Check that head req on list should be in flight.
509 * If it is not in flight then abort transfer as
510 * looping of transfer can not continue.
512 hsgreq
= list_first_entry(&tdc
->pending_sg_req
, typeof(*hsgreq
), node
);
513 if (!hsgreq
->configured
) {
515 dev_err(tdc2dev(tdc
), "Error in dma transfer, aborting dma\n");
516 tegra_dma_abort_all(tdc
);
520 /* Configure next request */
522 tdc_configure_next_head_desc(tdc
);
526 static void handle_once_dma_done(struct tegra_dma_channel
*tdc
,
529 struct tegra_dma_sg_req
*sgreq
;
530 struct tegra_dma_desc
*dma_desc
;
533 sgreq
= list_first_entry(&tdc
->pending_sg_req
, typeof(*sgreq
), node
);
534 dma_desc
= sgreq
->dma_desc
;
535 dma_desc
->bytes_transferred
+= sgreq
->req_len
;
537 list_del(&sgreq
->node
);
538 if (sgreq
->last_sg
) {
539 dma_desc
->dma_status
= DMA_SUCCESS
;
540 dma_cookie_complete(&dma_desc
->txd
);
541 if (!dma_desc
->cb_count
)
542 list_add_tail(&dma_desc
->cb_node
, &tdc
->cb_desc
);
543 dma_desc
->cb_count
++;
544 list_add_tail(&dma_desc
->node
, &tdc
->free_dma_desc
);
546 list_add_tail(&sgreq
->node
, &tdc
->free_sg_req
);
548 /* Do not start DMA if it is going to be terminate */
549 if (to_terminate
|| list_empty(&tdc
->pending_sg_req
))
552 tdc_start_head_req(tdc
);
556 static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel
*tdc
,
559 struct tegra_dma_sg_req
*sgreq
;
560 struct tegra_dma_desc
*dma_desc
;
563 sgreq
= list_first_entry(&tdc
->pending_sg_req
, typeof(*sgreq
), node
);
564 dma_desc
= sgreq
->dma_desc
;
565 dma_desc
->bytes_transferred
+= sgreq
->req_len
;
567 /* Callback need to be call */
568 if (!dma_desc
->cb_count
)
569 list_add_tail(&dma_desc
->cb_node
, &tdc
->cb_desc
);
570 dma_desc
->cb_count
++;
572 /* If not last req then put at end of pending list */
573 if (!list_is_last(&sgreq
->node
, &tdc
->pending_sg_req
)) {
574 list_move_tail(&sgreq
->node
, &tdc
->pending_sg_req
);
575 sgreq
->configured
= false;
576 st
= handle_continuous_head_request(tdc
, sgreq
, to_terminate
);
578 dma_desc
->dma_status
= DMA_ERROR
;
583 static void tegra_dma_tasklet(unsigned long data
)
585 struct tegra_dma_channel
*tdc
= (struct tegra_dma_channel
*)data
;
586 dma_async_tx_callback callback
= NULL
;
587 void *callback_param
= NULL
;
588 struct tegra_dma_desc
*dma_desc
;
592 spin_lock_irqsave(&tdc
->lock
, flags
);
593 while (!list_empty(&tdc
->cb_desc
)) {
594 dma_desc
= list_first_entry(&tdc
->cb_desc
,
595 typeof(*dma_desc
), cb_node
);
596 list_del(&dma_desc
->cb_node
);
597 callback
= dma_desc
->txd
.callback
;
598 callback_param
= dma_desc
->txd
.callback_param
;
599 cb_count
= dma_desc
->cb_count
;
600 dma_desc
->cb_count
= 0;
601 spin_unlock_irqrestore(&tdc
->lock
, flags
);
602 while (cb_count
-- && callback
)
603 callback(callback_param
);
604 spin_lock_irqsave(&tdc
->lock
, flags
);
606 spin_unlock_irqrestore(&tdc
->lock
, flags
);
609 static irqreturn_t
tegra_dma_isr(int irq
, void *dev_id
)
611 struct tegra_dma_channel
*tdc
= dev_id
;
612 unsigned long status
;
615 spin_lock_irqsave(&tdc
->lock
, flags
);
617 status
= tdc_read(tdc
, TEGRA_APBDMA_CHAN_STATUS
);
618 if (status
& TEGRA_APBDMA_STATUS_ISE_EOC
) {
619 tdc_write(tdc
, TEGRA_APBDMA_CHAN_STATUS
, status
);
620 tdc
->isr_handler(tdc
, false);
621 tasklet_schedule(&tdc
->tasklet
);
622 spin_unlock_irqrestore(&tdc
->lock
, flags
);
626 spin_unlock_irqrestore(&tdc
->lock
, flags
);
627 dev_info(tdc2dev(tdc
),
628 "Interrupt already served status 0x%08lx\n", status
);
632 static dma_cookie_t
tegra_dma_tx_submit(struct dma_async_tx_descriptor
*txd
)
634 struct tegra_dma_desc
*dma_desc
= txd_to_tegra_dma_desc(txd
);
635 struct tegra_dma_channel
*tdc
= to_tegra_dma_chan(txd
->chan
);
639 spin_lock_irqsave(&tdc
->lock
, flags
);
640 dma_desc
->dma_status
= DMA_IN_PROGRESS
;
641 cookie
= dma_cookie_assign(&dma_desc
->txd
);
642 list_splice_tail_init(&dma_desc
->tx_list
, &tdc
->pending_sg_req
);
643 spin_unlock_irqrestore(&tdc
->lock
, flags
);
647 static void tegra_dma_issue_pending(struct dma_chan
*dc
)
649 struct tegra_dma_channel
*tdc
= to_tegra_dma_chan(dc
);
652 spin_lock_irqsave(&tdc
->lock
, flags
);
653 if (list_empty(&tdc
->pending_sg_req
)) {
654 dev_err(tdc2dev(tdc
), "No DMA request\n");
658 tdc_start_head_req(tdc
);
660 /* Continuous single mode: Configure next req */
663 * Wait for 1 burst time for configure DMA for
666 udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME
);
667 tdc_configure_next_head_desc(tdc
);
671 spin_unlock_irqrestore(&tdc
->lock
, flags
);
675 static void tegra_dma_terminate_all(struct dma_chan
*dc
)
677 struct tegra_dma_channel
*tdc
= to_tegra_dma_chan(dc
);
678 struct tegra_dma_sg_req
*sgreq
;
679 struct tegra_dma_desc
*dma_desc
;
681 unsigned long status
;
684 spin_lock_irqsave(&tdc
->lock
, flags
);
685 if (list_empty(&tdc
->pending_sg_req
)) {
686 spin_unlock_irqrestore(&tdc
->lock
, flags
);
693 /* Pause DMA before checking the queue status */
694 tegra_dma_global_pause(tdc
, true);
696 status
= tdc_read(tdc
, TEGRA_APBDMA_CHAN_STATUS
);
697 if (status
& TEGRA_APBDMA_STATUS_ISE_EOC
) {
698 dev_dbg(tdc2dev(tdc
), "%s():handling isr\n", __func__
);
699 tdc
->isr_handler(tdc
, true);
700 status
= tdc_read(tdc
, TEGRA_APBDMA_CHAN_STATUS
);
703 was_busy
= tdc
->busy
;
706 if (!list_empty(&tdc
->pending_sg_req
) && was_busy
) {
707 sgreq
= list_first_entry(&tdc
->pending_sg_req
,
708 typeof(*sgreq
), node
);
709 sgreq
->dma_desc
->bytes_transferred
+=
710 get_current_xferred_count(tdc
, sgreq
, status
);
712 tegra_dma_global_resume(tdc
);
715 tegra_dma_abort_all(tdc
);
717 while (!list_empty(&tdc
->cb_desc
)) {
718 dma_desc
= list_first_entry(&tdc
->cb_desc
,
719 typeof(*dma_desc
), cb_node
);
720 list_del(&dma_desc
->cb_node
);
721 dma_desc
->cb_count
= 0;
723 spin_unlock_irqrestore(&tdc
->lock
, flags
);
726 static enum dma_status
tegra_dma_tx_status(struct dma_chan
*dc
,
727 dma_cookie_t cookie
, struct dma_tx_state
*txstate
)
729 struct tegra_dma_channel
*tdc
= to_tegra_dma_chan(dc
);
730 struct tegra_dma_desc
*dma_desc
;
731 struct tegra_dma_sg_req
*sg_req
;
734 unsigned int residual
;
736 spin_lock_irqsave(&tdc
->lock
, flags
);
738 ret
= dma_cookie_status(dc
, cookie
, txstate
);
739 if (ret
== DMA_SUCCESS
) {
740 dma_set_residue(txstate
, 0);
741 spin_unlock_irqrestore(&tdc
->lock
, flags
);
745 /* Check on wait_ack desc status */
746 list_for_each_entry(dma_desc
, &tdc
->free_dma_desc
, node
) {
747 if (dma_desc
->txd
.cookie
== cookie
) {
748 residual
= dma_desc
->bytes_requested
-
749 (dma_desc
->bytes_transferred
%
750 dma_desc
->bytes_requested
);
751 dma_set_residue(txstate
, residual
);
752 ret
= dma_desc
->dma_status
;
753 spin_unlock_irqrestore(&tdc
->lock
, flags
);
758 /* Check in pending list */
759 list_for_each_entry(sg_req
, &tdc
->pending_sg_req
, node
) {
760 dma_desc
= sg_req
->dma_desc
;
761 if (dma_desc
->txd
.cookie
== cookie
) {
762 residual
= dma_desc
->bytes_requested
-
763 (dma_desc
->bytes_transferred
%
764 dma_desc
->bytes_requested
);
765 dma_set_residue(txstate
, residual
);
766 ret
= dma_desc
->dma_status
;
767 spin_unlock_irqrestore(&tdc
->lock
, flags
);
772 dev_dbg(tdc2dev(tdc
), "cookie %d does not found\n", cookie
);
773 spin_unlock_irqrestore(&tdc
->lock
, flags
);
777 static int tegra_dma_device_control(struct dma_chan
*dc
, enum dma_ctrl_cmd cmd
,
781 case DMA_SLAVE_CONFIG
:
782 return tegra_dma_slave_config(dc
,
783 (struct dma_slave_config
*)arg
);
785 case DMA_TERMINATE_ALL
:
786 tegra_dma_terminate_all(dc
);
796 static inline int get_bus_width(struct tegra_dma_channel
*tdc
,
797 enum dma_slave_buswidth slave_bw
)
800 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
801 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_8
;
802 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
803 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_16
;
804 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
805 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32
;
806 case DMA_SLAVE_BUSWIDTH_8_BYTES
:
807 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64
;
809 dev_warn(tdc2dev(tdc
),
810 "slave bw is not supported, using 32bits\n");
811 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32
;
815 static inline int get_burst_size(struct tegra_dma_channel
*tdc
,
816 u32 burst_size
, enum dma_slave_buswidth slave_bw
, int len
)
822 * burst_size from client is in terms of the bus_width.
823 * convert them into AHB memory width which is 4 byte.
825 burst_byte
= burst_size
* slave_bw
;
826 burst_ahb_width
= burst_byte
/ 4;
828 /* If burst size is 0 then calculate the burst size based on length */
829 if (!burst_ahb_width
) {
831 return TEGRA_APBDMA_AHBSEQ_BURST_1
;
832 else if ((len
>> 4) & 0x1)
833 return TEGRA_APBDMA_AHBSEQ_BURST_4
;
835 return TEGRA_APBDMA_AHBSEQ_BURST_8
;
837 if (burst_ahb_width
< 4)
838 return TEGRA_APBDMA_AHBSEQ_BURST_1
;
839 else if (burst_ahb_width
< 8)
840 return TEGRA_APBDMA_AHBSEQ_BURST_4
;
842 return TEGRA_APBDMA_AHBSEQ_BURST_8
;
845 static int get_transfer_param(struct tegra_dma_channel
*tdc
,
846 enum dma_transfer_direction direction
, unsigned long *apb_addr
,
847 unsigned long *apb_seq
, unsigned long *csr
, unsigned int *burst_size
,
848 enum dma_slave_buswidth
*slave_bw
)
853 *apb_addr
= tdc
->dma_sconfig
.dst_addr
;
854 *apb_seq
= get_bus_width(tdc
, tdc
->dma_sconfig
.dst_addr_width
);
855 *burst_size
= tdc
->dma_sconfig
.dst_maxburst
;
856 *slave_bw
= tdc
->dma_sconfig
.dst_addr_width
;
857 *csr
= TEGRA_APBDMA_CSR_DIR
;
861 *apb_addr
= tdc
->dma_sconfig
.src_addr
;
862 *apb_seq
= get_bus_width(tdc
, tdc
->dma_sconfig
.src_addr_width
);
863 *burst_size
= tdc
->dma_sconfig
.src_maxburst
;
864 *slave_bw
= tdc
->dma_sconfig
.src_addr_width
;
869 dev_err(tdc2dev(tdc
), "Dma direction is not supported\n");
875 static struct dma_async_tx_descriptor
*tegra_dma_prep_slave_sg(
876 struct dma_chan
*dc
, struct scatterlist
*sgl
, unsigned int sg_len
,
877 enum dma_transfer_direction direction
, unsigned long flags
,
880 struct tegra_dma_channel
*tdc
= to_tegra_dma_chan(dc
);
881 struct tegra_dma_desc
*dma_desc
;
883 struct scatterlist
*sg
;
884 unsigned long csr
, ahb_seq
, apb_ptr
, apb_seq
;
885 struct list_head req_list
;
886 struct tegra_dma_sg_req
*sg_req
= NULL
;
888 enum dma_slave_buswidth slave_bw
;
891 if (!tdc
->config_init
) {
892 dev_err(tdc2dev(tdc
), "dma channel is not configured\n");
896 dev_err(tdc2dev(tdc
), "Invalid segment length %d\n", sg_len
);
900 ret
= get_transfer_param(tdc
, direction
, &apb_ptr
, &apb_seq
, &csr
,
901 &burst_size
, &slave_bw
);
905 INIT_LIST_HEAD(&req_list
);
907 ahb_seq
= TEGRA_APBDMA_AHBSEQ_INTR_ENB
;
908 ahb_seq
|= TEGRA_APBDMA_AHBSEQ_WRAP_NONE
<<
909 TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT
;
910 ahb_seq
|= TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32
;
912 csr
|= TEGRA_APBDMA_CSR_ONCE
| TEGRA_APBDMA_CSR_FLOW
;
913 csr
|= tdc
->dma_sconfig
.slave_id
<< TEGRA_APBDMA_CSR_REQ_SEL_SHIFT
;
914 if (flags
& DMA_PREP_INTERRUPT
)
915 csr
|= TEGRA_APBDMA_CSR_IE_EOC
;
917 apb_seq
|= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1
;
919 dma_desc
= tegra_dma_desc_get(tdc
);
921 dev_err(tdc2dev(tdc
), "Dma descriptors not available\n");
924 INIT_LIST_HEAD(&dma_desc
->tx_list
);
925 INIT_LIST_HEAD(&dma_desc
->cb_node
);
926 dma_desc
->cb_count
= 0;
927 dma_desc
->bytes_requested
= 0;
928 dma_desc
->bytes_transferred
= 0;
929 dma_desc
->dma_status
= DMA_IN_PROGRESS
;
931 /* Make transfer requests */
932 for_each_sg(sgl
, sg
, sg_len
, i
) {
935 mem
= sg_dma_address(sg
);
936 len
= sg_dma_len(sg
);
938 if ((len
& 3) || (mem
& 3) ||
939 (len
> tdc
->tdma
->chip_data
->max_dma_count
)) {
940 dev_err(tdc2dev(tdc
),
941 "Dma length/memory address is not supported\n");
942 tegra_dma_desc_put(tdc
, dma_desc
);
946 sg_req
= tegra_dma_sg_req_get(tdc
);
948 dev_err(tdc2dev(tdc
), "Dma sg-req not available\n");
949 tegra_dma_desc_put(tdc
, dma_desc
);
953 ahb_seq
|= get_burst_size(tdc
, burst_size
, slave_bw
, len
);
954 dma_desc
->bytes_requested
+= len
;
956 sg_req
->ch_regs
.apb_ptr
= apb_ptr
;
957 sg_req
->ch_regs
.ahb_ptr
= mem
;
958 sg_req
->ch_regs
.csr
= csr
| ((len
- 4) & 0xFFFC);
959 sg_req
->ch_regs
.apb_seq
= apb_seq
;
960 sg_req
->ch_regs
.ahb_seq
= ahb_seq
;
961 sg_req
->configured
= false;
962 sg_req
->last_sg
= false;
963 sg_req
->dma_desc
= dma_desc
;
964 sg_req
->req_len
= len
;
966 list_add_tail(&sg_req
->node
, &dma_desc
->tx_list
);
968 sg_req
->last_sg
= true;
969 if (flags
& DMA_CTRL_ACK
)
970 dma_desc
->txd
.flags
= DMA_CTRL_ACK
;
973 * Make sure that mode should not be conflicting with currently
976 if (!tdc
->isr_handler
) {
977 tdc
->isr_handler
= handle_once_dma_done
;
981 dev_err(tdc2dev(tdc
), "DMA configured in cyclic mode\n");
982 tegra_dma_desc_put(tdc
, dma_desc
);
987 return &dma_desc
->txd
;
990 struct dma_async_tx_descriptor
*tegra_dma_prep_dma_cyclic(
991 struct dma_chan
*dc
, dma_addr_t buf_addr
, size_t buf_len
,
992 size_t period_len
, enum dma_transfer_direction direction
,
993 unsigned long flags
, void *context
)
995 struct tegra_dma_channel
*tdc
= to_tegra_dma_chan(dc
);
996 struct tegra_dma_desc
*dma_desc
= NULL
;
997 struct tegra_dma_sg_req
*sg_req
= NULL
;
998 unsigned long csr
, ahb_seq
, apb_ptr
, apb_seq
;
1001 dma_addr_t mem
= buf_addr
;
1003 enum dma_slave_buswidth slave_bw
;
1006 if (!buf_len
|| !period_len
) {
1007 dev_err(tdc2dev(tdc
), "Invalid buffer/period len\n");
1011 if (!tdc
->config_init
) {
1012 dev_err(tdc2dev(tdc
), "DMA slave is not configured\n");
1017 * We allow to take more number of requests till DMA is
1018 * not started. The driver will loop over all requests.
1019 * Once DMA is started then new requests can be queued only after
1020 * terminating the DMA.
1023 dev_err(tdc2dev(tdc
), "Request not allowed when dma running\n");
1028 * We only support cycle transfer when buf_len is multiple of
1031 if (buf_len
% period_len
) {
1032 dev_err(tdc2dev(tdc
), "buf_len is not multiple of period_len\n");
1037 if ((len
& 3) || (buf_addr
& 3) ||
1038 (len
> tdc
->tdma
->chip_data
->max_dma_count
)) {
1039 dev_err(tdc2dev(tdc
), "Req len/mem address is not correct\n");
1043 ret
= get_transfer_param(tdc
, direction
, &apb_ptr
, &apb_seq
, &csr
,
1044 &burst_size
, &slave_bw
);
1049 ahb_seq
= TEGRA_APBDMA_AHBSEQ_INTR_ENB
;
1050 ahb_seq
|= TEGRA_APBDMA_AHBSEQ_WRAP_NONE
<<
1051 TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT
;
1052 ahb_seq
|= TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32
;
1054 csr
|= TEGRA_APBDMA_CSR_FLOW
| TEGRA_APBDMA_CSR_IE_EOC
;
1055 csr
|= tdc
->dma_sconfig
.slave_id
<< TEGRA_APBDMA_CSR_REQ_SEL_SHIFT
;
1057 apb_seq
|= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1
;
1059 dma_desc
= tegra_dma_desc_get(tdc
);
1061 dev_err(tdc2dev(tdc
), "not enough descriptors available\n");
1065 INIT_LIST_HEAD(&dma_desc
->tx_list
);
1066 INIT_LIST_HEAD(&dma_desc
->cb_node
);
1067 dma_desc
->cb_count
= 0;
1069 dma_desc
->bytes_transferred
= 0;
1070 dma_desc
->bytes_requested
= buf_len
;
1071 remain_len
= buf_len
;
1073 /* Split transfer equal to period size */
1074 while (remain_len
) {
1075 sg_req
= tegra_dma_sg_req_get(tdc
);
1077 dev_err(tdc2dev(tdc
), "Dma sg-req not available\n");
1078 tegra_dma_desc_put(tdc
, dma_desc
);
1082 ahb_seq
|= get_burst_size(tdc
, burst_size
, slave_bw
, len
);
1083 sg_req
->ch_regs
.apb_ptr
= apb_ptr
;
1084 sg_req
->ch_regs
.ahb_ptr
= mem
;
1085 sg_req
->ch_regs
.csr
= csr
| ((len
- 4) & 0xFFFC);
1086 sg_req
->ch_regs
.apb_seq
= apb_seq
;
1087 sg_req
->ch_regs
.ahb_seq
= ahb_seq
;
1088 sg_req
->configured
= false;
1089 sg_req
->half_done
= false;
1090 sg_req
->last_sg
= false;
1091 sg_req
->dma_desc
= dma_desc
;
1092 sg_req
->req_len
= len
;
1094 list_add_tail(&sg_req
->node
, &dma_desc
->tx_list
);
1098 sg_req
->last_sg
= true;
1099 dma_desc
->txd
.flags
= 0;
1102 * Make sure that mode should not be conflicting with currently
1105 if (!tdc
->isr_handler
) {
1106 tdc
->isr_handler
= handle_cont_sngl_cycle_dma_done
;
1110 dev_err(tdc2dev(tdc
), "DMA configuration conflict\n");
1111 tegra_dma_desc_put(tdc
, dma_desc
);
1116 return &dma_desc
->txd
;
1119 static int tegra_dma_alloc_chan_resources(struct dma_chan
*dc
)
1121 struct tegra_dma_channel
*tdc
= to_tegra_dma_chan(dc
);
1122 struct tegra_dma
*tdma
= tdc
->tdma
;
1125 dma_cookie_init(&tdc
->dma_chan
);
1126 tdc
->config_init
= false;
1127 ret
= clk_prepare_enable(tdma
->dma_clk
);
1129 dev_err(tdc2dev(tdc
), "clk_prepare_enable failed: %d\n", ret
);
1133 static void tegra_dma_free_chan_resources(struct dma_chan
*dc
)
1135 struct tegra_dma_channel
*tdc
= to_tegra_dma_chan(dc
);
1136 struct tegra_dma
*tdma
= tdc
->tdma
;
1138 struct tegra_dma_desc
*dma_desc
;
1139 struct tegra_dma_sg_req
*sg_req
;
1140 struct list_head dma_desc_list
;
1141 struct list_head sg_req_list
;
1142 unsigned long flags
;
1144 INIT_LIST_HEAD(&dma_desc_list
);
1145 INIT_LIST_HEAD(&sg_req_list
);
1147 dev_dbg(tdc2dev(tdc
), "Freeing channel %d\n", tdc
->id
);
1150 tegra_dma_terminate_all(dc
);
1152 spin_lock_irqsave(&tdc
->lock
, flags
);
1153 list_splice_init(&tdc
->pending_sg_req
, &sg_req_list
);
1154 list_splice_init(&tdc
->free_sg_req
, &sg_req_list
);
1155 list_splice_init(&tdc
->free_dma_desc
, &dma_desc_list
);
1156 INIT_LIST_HEAD(&tdc
->cb_desc
);
1157 tdc
->config_init
= false;
1158 spin_unlock_irqrestore(&tdc
->lock
, flags
);
1160 while (!list_empty(&dma_desc_list
)) {
1161 dma_desc
= list_first_entry(&dma_desc_list
,
1162 typeof(*dma_desc
), node
);
1163 list_del(&dma_desc
->node
);
1167 while (!list_empty(&sg_req_list
)) {
1168 sg_req
= list_first_entry(&sg_req_list
, typeof(*sg_req
), node
);
1169 list_del(&sg_req
->node
);
1172 clk_disable_unprepare(tdma
->dma_clk
);
1175 /* Tegra20 specific DMA controller information */
1176 static const struct tegra_dma_chip_data tegra20_dma_chip_data
= {
1178 .max_dma_count
= 1024UL * 64,
1181 #if defined(CONFIG_OF)
1182 /* Tegra30 specific DMA controller information */
1183 static const struct tegra_dma_chip_data tegra30_dma_chip_data
= {
1185 .max_dma_count
= 1024UL * 64,
1188 static const struct of_device_id tegra_dma_of_match
[] = {
1190 .compatible
= "nvidia,tegra30-apbdma",
1191 .data
= &tegra30_dma_chip_data
,
1193 .compatible
= "nvidia,tegra20-apbdma",
1194 .data
= &tegra20_dma_chip_data
,
1198 MODULE_DEVICE_TABLE(of
, tegra_dma_of_match
);
1201 static int tegra_dma_probe(struct platform_device
*pdev
)
1203 struct resource
*res
;
1204 struct tegra_dma
*tdma
;
1207 const struct tegra_dma_chip_data
*cdata
= NULL
;
1209 if (pdev
->dev
.of_node
) {
1210 const struct of_device_id
*match
;
1211 match
= of_match_device(of_match_ptr(tegra_dma_of_match
),
1214 dev_err(&pdev
->dev
, "Error: No device match found\n");
1217 cdata
= match
->data
;
1219 /* If no device tree then fallback to tegra20 */
1220 cdata
= &tegra20_dma_chip_data
;
1223 tdma
= devm_kzalloc(&pdev
->dev
, sizeof(*tdma
) + cdata
->nr_channels
*
1224 sizeof(struct tegra_dma_channel
), GFP_KERNEL
);
1226 dev_err(&pdev
->dev
, "Error: memory allocation failed\n");
1230 tdma
->dev
= &pdev
->dev
;
1231 tdma
->chip_data
= cdata
;
1232 platform_set_drvdata(pdev
, tdma
);
1234 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1236 dev_err(&pdev
->dev
, "No mem resource for DMA\n");
1240 tdma
->base_addr
= devm_ioremap_resource(&pdev
->dev
, res
);
1241 if (IS_ERR(tdma
->base_addr
))
1242 return PTR_ERR(tdma
->base_addr
);
1244 tdma
->dma_clk
= devm_clk_get(&pdev
->dev
, NULL
);
1245 if (IS_ERR(tdma
->dma_clk
)) {
1246 dev_err(&pdev
->dev
, "Error: Missing controller clock\n");
1247 return PTR_ERR(tdma
->dma_clk
);
1250 spin_lock_init(&tdma
->global_lock
);
1252 pm_runtime_enable(&pdev
->dev
);
1253 if (!pm_runtime_enabled(&pdev
->dev
)) {
1254 ret
= tegra_dma_runtime_resume(&pdev
->dev
);
1256 dev_err(&pdev
->dev
, "dma_runtime_resume failed %d\n",
1258 goto err_pm_disable
;
1262 /* Enable clock before accessing registers */
1263 ret
= clk_prepare_enable(tdma
->dma_clk
);
1265 dev_err(&pdev
->dev
, "clk_prepare_enable failed: %d\n", ret
);
1266 goto err_pm_disable
;
1269 /* Reset DMA controller */
1270 tegra_periph_reset_assert(tdma
->dma_clk
);
1272 tegra_periph_reset_deassert(tdma
->dma_clk
);
1274 /* Enable global DMA registers */
1275 tdma_write(tdma
, TEGRA_APBDMA_GENERAL
, TEGRA_APBDMA_GENERAL_ENABLE
);
1276 tdma_write(tdma
, TEGRA_APBDMA_CONTROL
, 0);
1277 tdma_write(tdma
, TEGRA_APBDMA_IRQ_MASK_SET
, 0xFFFFFFFFul
);
1279 clk_disable_unprepare(tdma
->dma_clk
);
1281 INIT_LIST_HEAD(&tdma
->dma_dev
.channels
);
1282 for (i
= 0; i
< cdata
->nr_channels
; i
++) {
1283 struct tegra_dma_channel
*tdc
= &tdma
->channels
[i
];
1285 tdc
->chan_base_offset
= TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET
+
1286 i
* TEGRA_APBDMA_CHANNEL_REGISTER_SIZE
;
1288 res
= platform_get_resource(pdev
, IORESOURCE_IRQ
, i
);
1291 dev_err(&pdev
->dev
, "No irq resource for chan %d\n", i
);
1294 tdc
->irq
= res
->start
;
1295 snprintf(tdc
->name
, sizeof(tdc
->name
), "apbdma.%d", i
);
1296 ret
= devm_request_irq(&pdev
->dev
, tdc
->irq
,
1297 tegra_dma_isr
, 0, tdc
->name
, tdc
);
1300 "request_irq failed with err %d channel %d\n",
1305 tdc
->dma_chan
.device
= &tdma
->dma_dev
;
1306 dma_cookie_init(&tdc
->dma_chan
);
1307 list_add_tail(&tdc
->dma_chan
.device_node
,
1308 &tdma
->dma_dev
.channels
);
1312 tasklet_init(&tdc
->tasklet
, tegra_dma_tasklet
,
1313 (unsigned long)tdc
);
1314 spin_lock_init(&tdc
->lock
);
1316 INIT_LIST_HEAD(&tdc
->pending_sg_req
);
1317 INIT_LIST_HEAD(&tdc
->free_sg_req
);
1318 INIT_LIST_HEAD(&tdc
->free_dma_desc
);
1319 INIT_LIST_HEAD(&tdc
->cb_desc
);
1322 dma_cap_set(DMA_SLAVE
, tdma
->dma_dev
.cap_mask
);
1323 dma_cap_set(DMA_PRIVATE
, tdma
->dma_dev
.cap_mask
);
1324 dma_cap_set(DMA_CYCLIC
, tdma
->dma_dev
.cap_mask
);
1326 tdma
->dma_dev
.dev
= &pdev
->dev
;
1327 tdma
->dma_dev
.device_alloc_chan_resources
=
1328 tegra_dma_alloc_chan_resources
;
1329 tdma
->dma_dev
.device_free_chan_resources
=
1330 tegra_dma_free_chan_resources
;
1331 tdma
->dma_dev
.device_prep_slave_sg
= tegra_dma_prep_slave_sg
;
1332 tdma
->dma_dev
.device_prep_dma_cyclic
= tegra_dma_prep_dma_cyclic
;
1333 tdma
->dma_dev
.device_control
= tegra_dma_device_control
;
1334 tdma
->dma_dev
.device_tx_status
= tegra_dma_tx_status
;
1335 tdma
->dma_dev
.device_issue_pending
= tegra_dma_issue_pending
;
1337 ret
= dma_async_device_register(&tdma
->dma_dev
);
1340 "Tegra20 APB DMA driver registration failed %d\n", ret
);
1344 dev_info(&pdev
->dev
, "Tegra20 APB DMA driver register %d channels\n",
1345 cdata
->nr_channels
);
1350 struct tegra_dma_channel
*tdc
= &tdma
->channels
[i
];
1351 tasklet_kill(&tdc
->tasklet
);
1355 pm_runtime_disable(&pdev
->dev
);
1356 if (!pm_runtime_status_suspended(&pdev
->dev
))
1357 tegra_dma_runtime_suspend(&pdev
->dev
);
1361 static int tegra_dma_remove(struct platform_device
*pdev
)
1363 struct tegra_dma
*tdma
= platform_get_drvdata(pdev
);
1365 struct tegra_dma_channel
*tdc
;
1367 dma_async_device_unregister(&tdma
->dma_dev
);
1369 for (i
= 0; i
< tdma
->chip_data
->nr_channels
; ++i
) {
1370 tdc
= &tdma
->channels
[i
];
1371 tasklet_kill(&tdc
->tasklet
);
1374 pm_runtime_disable(&pdev
->dev
);
1375 if (!pm_runtime_status_suspended(&pdev
->dev
))
1376 tegra_dma_runtime_suspend(&pdev
->dev
);
1381 static int tegra_dma_runtime_suspend(struct device
*dev
)
1383 struct platform_device
*pdev
= to_platform_device(dev
);
1384 struct tegra_dma
*tdma
= platform_get_drvdata(pdev
);
1386 clk_disable_unprepare(tdma
->dma_clk
);
1390 static int tegra_dma_runtime_resume(struct device
*dev
)
1392 struct platform_device
*pdev
= to_platform_device(dev
);
1393 struct tegra_dma
*tdma
= platform_get_drvdata(pdev
);
1396 ret
= clk_prepare_enable(tdma
->dma_clk
);
1398 dev_err(dev
, "clk_enable failed: %d\n", ret
);
1404 static const struct dev_pm_ops tegra_dma_dev_pm_ops
= {
1405 #ifdef CONFIG_PM_RUNTIME
1406 .runtime_suspend
= tegra_dma_runtime_suspend
,
1407 .runtime_resume
= tegra_dma_runtime_resume
,
1411 static struct platform_driver tegra_dmac_driver
= {
1413 .name
= "tegra-apbdma",
1414 .owner
= THIS_MODULE
,
1415 .pm
= &tegra_dma_dev_pm_ops
,
1416 .of_match_table
= of_match_ptr(tegra_dma_of_match
),
1418 .probe
= tegra_dma_probe
,
1419 .remove
= tegra_dma_remove
,
1422 module_platform_driver(tegra_dmac_driver
);
1424 MODULE_ALIAS("platform:tegra20-apbdma");
1425 MODULE_DESCRIPTION("NVIDIA Tegra APB DMA Controller driver");
1426 MODULE_AUTHOR("Laxman Dewangan <ldewangan@nvidia.com>");
1427 MODULE_LICENSE("GPL v2");