ARM: tegra: dma: not required to move requestor when stopping.

[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / arch / arm / mach-tegra / dma.c

/*
 * arch/arm/mach-tegra/dma.c
 *
 * System DMA driver for NVIDIA Tegra SoCs
 *
 * Copyright (c) 2008-2009, NVIDIA Corporation.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#include <linux/io.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/spinlock.h>
#include <linux/err.h>
#include <linux/irq.h>
#include <linux/delay.h>
#include <linux/clk.h>
#include <mach/dma.h>
#include <mach/irqs.h>
#include <mach/iomap.h>
#include <mach/suspend.h>

#include "apbio.h"

#define APB_DMA_GEN                             0x000
#define GEN_ENABLE                              (1<<31)

#define APB_DMA_CNTRL                           0x010

#define APB_DMA_IRQ_MASK                        0x01c

#define APB_DMA_IRQ_MASK_SET                    0x020

#define APB_DMA_CHAN_CSR                        0x000
#define CSR_ENB                                 (1<<31)
#define CSR_IE_EOC                              (1<<30)
#define CSR_HOLD                                (1<<29)
#define CSR_DIR                                 (1<<28)
#define CSR_ONCE                                (1<<27)
#define CSR_FLOW                                (1<<21)
#define CSR_REQ_SEL_SHIFT                       16
#define CSR_WCOUNT_SHIFT                        2
#define CSR_WCOUNT_MASK                         0xFFFC

#define APB_DMA_CHAN_STA                                0x004
#define STA_BUSY                                (1<<31)
#define STA_ISE_EOC                             (1<<30)
#define STA_HALT                                (1<<29)
#define STA_PING_PONG                           (1<<28)
#define STA_COUNT_SHIFT                         2
#define STA_COUNT_MASK                          0xFFFC

#define APB_DMA_CHAN_AHB_PTR                            0x010

#define APB_DMA_CHAN_AHB_SEQ                            0x014
#define AHB_SEQ_INTR_ENB                        (1<<31)
#define AHB_SEQ_BUS_WIDTH_SHIFT                 28
#define AHB_SEQ_BUS_WIDTH_MASK                  (0x7<<AHB_SEQ_BUS_WIDTH_SHIFT)
#define AHB_SEQ_BUS_WIDTH_8                     (0<<AHB_SEQ_BUS_WIDTH_SHIFT)
#define AHB_SEQ_BUS_WIDTH_16                    (1<<AHB_SEQ_BUS_WIDTH_SHIFT)
#define AHB_SEQ_BUS_WIDTH_32                    (2<<AHB_SEQ_BUS_WIDTH_SHIFT)
#define AHB_SEQ_BUS_WIDTH_64                    (3<<AHB_SEQ_BUS_WIDTH_SHIFT)
#define AHB_SEQ_BUS_WIDTH_128                   (4<<AHB_SEQ_BUS_WIDTH_SHIFT)
#define AHB_SEQ_DATA_SWAP                       (1<<27)
#define AHB_SEQ_BURST_MASK                      (0x7<<24)
#define AHB_SEQ_BURST_1                         (4<<24)
#define AHB_SEQ_BURST_4                         (5<<24)
#define AHB_SEQ_BURST_8                         (6<<24)
#define AHB_SEQ_DBL_BUF                         (1<<19)
#define AHB_SEQ_WRAP_SHIFT                      16
#define AHB_SEQ_WRAP_MASK                       (0x7<<AHB_SEQ_WRAP_SHIFT)

#define APB_DMA_CHAN_APB_PTR                            0x018

#define APB_DMA_CHAN_APB_SEQ                            0x01c
#define APB_SEQ_BUS_WIDTH_SHIFT                 28
#define APB_SEQ_BUS_WIDTH_MASK                  (0x7<<APB_SEQ_BUS_WIDTH_SHIFT)
#define APB_SEQ_BUS_WIDTH_8                     (0<<APB_SEQ_BUS_WIDTH_SHIFT)
#define APB_SEQ_BUS_WIDTH_16                    (1<<APB_SEQ_BUS_WIDTH_SHIFT)
#define APB_SEQ_BUS_WIDTH_32                    (2<<APB_SEQ_BUS_WIDTH_SHIFT)
#define APB_SEQ_BUS_WIDTH_64                    (3<<APB_SEQ_BUS_WIDTH_SHIFT)
#define APB_SEQ_BUS_WIDTH_128                   (4<<APB_SEQ_BUS_WIDTH_SHIFT)
#define APB_SEQ_DATA_SWAP                       (1<<27)
#define APB_SEQ_WRAP_SHIFT                      16
#define APB_SEQ_WRAP_MASK                       (0x7<<APB_SEQ_WRAP_SHIFT)

#define TEGRA_SYSTEM_DMA_CH_NR                  16
#define TEGRA_SYSTEM_DMA_AVP_CH_NUM             4
#define TEGRA_SYSTEM_DMA_CH_MIN                 0
#define TEGRA_SYSTEM_DMA_CH_MAX \
        (TEGRA_SYSTEM_DMA_CH_NR - TEGRA_SYSTEM_DMA_AVP_CH_NUM - 1)

#define NV_DMA_MAX_TRASFER_SIZE 0x10000

static const unsigned int ahb_addr_wrap_table[8] = {
        0, 32, 64, 128, 256, 512, 1024, 2048
};

static const unsigned int apb_addr_wrap_table[8] = {
        0, 1, 2, 4, 8, 16, 32, 64
};

static const unsigned int bus_width_table[5] = {
        8, 16, 32, 64, 128
};

#define TEGRA_DMA_NAME_SIZE 16
struct tegra_dma_channel {
        struct list_head        list;
        int                     id;
        spinlock_t              lock;
        char                    name[TEGRA_DMA_NAME_SIZE];
        void  __iomem           *addr;
        int                     mode;
        int                     irq;
        int                     req_transfer_count;
};

#define  NV_DMA_MAX_CHANNELS  32

static bool tegra_dma_initialized;
static DEFINE_MUTEX(tegra_dma_lock);
static DEFINE_SPINLOCK(enable_lock);

static DECLARE_BITMAP(channel_usage, NV_DMA_MAX_CHANNELS);
static struct tegra_dma_channel dma_channels[NV_DMA_MAX_CHANNELS];

static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
        struct tegra_dma_req *req);
static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
        struct tegra_dma_req *req);
static void tegra_dma_stop(struct tegra_dma_channel *ch);

void tegra_dma_flush(struct tegra_dma_channel *ch)
{
}
EXPORT_SYMBOL(tegra_dma_flush);

void tegra_dma_dequeue(struct tegra_dma_channel *ch)
{
        struct tegra_dma_req *req;

        if (tegra_dma_is_empty(ch))
                return;

        req = list_entry(ch->list.next, typeof(*req), node);

        tegra_dma_dequeue_req(ch, req);
        return;
}

static void tegra_dma_stop(struct tegra_dma_channel *ch)
{
        u32 csr;
        u32 status;

        csr = readl(ch->addr + APB_DMA_CHAN_CSR);
        csr &= ~CSR_IE_EOC;
        writel(csr, ch->addr + APB_DMA_CHAN_CSR);

        csr &= ~CSR_ENB;
        writel(csr, ch->addr + APB_DMA_CHAN_CSR);

        status = readl(ch->addr + APB_DMA_CHAN_STA);
        if (status & STA_ISE_EOC)
                writel(status, ch->addr + APB_DMA_CHAN_STA);
}

static int tegra_dma_cancel(struct tegra_dma_channel *ch)
{
        unsigned long irq_flags;

        spin_lock_irqsave(&ch->lock, irq_flags);
        while (!list_empty(&ch->list))
                list_del(ch->list.next);

        tegra_dma_stop(ch);

        spin_unlock_irqrestore(&ch->lock, irq_flags);
        return 0;
}

static unsigned int get_channel_status(struct tegra_dma_channel *ch,
                        struct tegra_dma_req *req, bool is_stop_dma)
{
        void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
        unsigned int status;

        if (is_stop_dma) {
                /*
                 * STOP the DMA and get the transfer count.
                 * Getting the transfer count is tricky.
                 *  - Globally disable DMA on all channels
                 *  - Read the channel's status register to know the number
                 *    of pending bytes to be transfered.
                 *  - Stop the dma channel
                 *  - Globally re-enable DMA to resume other transfers
                 */
                spin_lock(&enable_lock);
                writel(0, addr + APB_DMA_GEN);
                udelay(20);
                status = readl(ch->addr + APB_DMA_CHAN_STA);
                tegra_dma_stop(ch);
                writel(GEN_ENABLE, addr + APB_DMA_GEN);
                spin_unlock(&enable_lock);
                if (status & STA_ISE_EOC) {
                        pr_err("Got Dma Int here clearing");
                        writel(status, ch->addr + APB_DMA_CHAN_STA);
                }
                req->status = TEGRA_DMA_REQ_ERROR_ABORTED;
        } else {
                status = readl(ch->addr + APB_DMA_CHAN_STA);
        }
        return status;
}

/* should be called with the channel lock held */
static unsigned int dma_active_count(struct tegra_dma_channel *ch,
        struct tegra_dma_req *req, unsigned int status)
{
        unsigned int to_transfer;
        unsigned int req_transfer_count;
        unsigned int bytes_transferred;

        to_transfer = ((status & STA_COUNT_MASK) >> STA_COUNT_SHIFT) + 1;
        req_transfer_count = ch->req_transfer_count + 1;
        bytes_transferred = req_transfer_count;
        if (status & STA_BUSY)
                bytes_transferred -= to_transfer;
        /*
         * In continuous transfer mode, DMA only tracks the count of the
         * half DMA buffer. So, if the DMA already finished half the DMA
         * then add the half buffer to the completed count.
         */
        if (ch->mode & TEGRA_DMA_MODE_CONTINOUS) {
                if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL)
                        bytes_transferred += req_transfer_count;
                if (status & STA_ISE_EOC)
                        bytes_transferred += req_transfer_count;
        }
        bytes_transferred *= 4;
        return bytes_transferred;
}

int tegra_dma_dequeue_req(struct tegra_dma_channel *ch,
        struct tegra_dma_req *_req)
{
        unsigned int status;
        struct tegra_dma_req *req = NULL;
        int found = 0;
        unsigned long irq_flags;
        int stop = 0;

        spin_lock_irqsave(&ch->lock, irq_flags);

        if (list_entry(ch->list.next, struct tegra_dma_req, node) == _req)
                stop = 1;

        list_for_each_entry(req, &ch->list, node) {
                if (req == _req) {
                        list_del(&req->node);
                        found = 1;
                        break;
                }
        }
        if (!found) {
                spin_unlock_irqrestore(&ch->lock, irq_flags);
                return 0;
        }

        if (!stop)
                goto skip_stop_dma;

        status = get_channel_status(ch, req, true);
        req->bytes_transferred = dma_active_count(ch, req, status);

        if (!list_empty(&ch->list)) {
                /* if the list is not empty, queue the next request */
                struct tegra_dma_req *next_req;
                next_req = list_entry(ch->list.next,
                        typeof(*next_req), node);
                tegra_dma_update_hw(ch, next_req);
        }

skip_stop_dma:
        req->status = -TEGRA_DMA_REQ_ERROR_ABORTED;

        spin_unlock_irqrestore(&ch->lock, irq_flags);

        /* Callback should be called without any lock */
        req->complete(req);
        return 0;
}
EXPORT_SYMBOL(tegra_dma_dequeue_req);

bool tegra_dma_is_empty(struct tegra_dma_channel *ch)
{
        unsigned long irq_flags;
        bool is_empty;

        spin_lock_irqsave(&ch->lock, irq_flags);
        if (list_empty(&ch->list))
                is_empty = true;
        else
                is_empty = false;
        spin_unlock_irqrestore(&ch->lock, irq_flags);
        return is_empty;
}
EXPORT_SYMBOL(tegra_dma_is_empty);

bool tegra_dma_is_req_inflight(struct tegra_dma_channel *ch,
        struct tegra_dma_req *_req)
{
        unsigned long irq_flags;
        struct tegra_dma_req *req;

        spin_lock_irqsave(&ch->lock, irq_flags);
        list_for_each_entry(req, &ch->list, node) {
                if (req == _req) {
                        spin_unlock_irqrestore(&ch->lock, irq_flags);
                        return true;
                }
        }
        spin_unlock_irqrestore(&ch->lock, irq_flags);
        return false;
}
EXPORT_SYMBOL(tegra_dma_is_req_inflight);

int tegra_dma_enqueue_req(struct tegra_dma_channel *ch,
        struct tegra_dma_req *req)
{
        unsigned long irq_flags;
        struct tegra_dma_req *_req;
        int start_dma = 0;

        if (req->size > NV_DMA_MAX_TRASFER_SIZE ||
                req->source_addr & 0x3 || req->dest_addr & 0x3) {
                pr_err("Invalid DMA request for channel %d\n", ch->id);
                return -EINVAL;
        }

        spin_lock_irqsave(&ch->lock, irq_flags);

        list_for_each_entry(_req, &ch->list, node) {
                if (req == _req) {
                    spin_unlock_irqrestore(&ch->lock, irq_flags);
                    return -EEXIST;
                }
        }

        req->bytes_transferred = 0;
        req->status = 0;
        req->buffer_status = 0;
        if (list_empty(&ch->list))
                start_dma = 1;

        list_add_tail(&req->node, &ch->list);

        if (start_dma)
                tegra_dma_update_hw(ch, req);

        spin_unlock_irqrestore(&ch->lock, irq_flags);

        return 0;
}
EXPORT_SYMBOL(tegra_dma_enqueue_req);

struct tegra_dma_channel *tegra_dma_allocate_channel(int mode)
{
        int channel;
        struct tegra_dma_channel *ch = NULL;

        if (!tegra_dma_initialized)
                return NULL;

        mutex_lock(&tegra_dma_lock);

        /* first channel is the shared channel */
        if (mode & TEGRA_DMA_SHARED) {
                channel = TEGRA_SYSTEM_DMA_CH_MIN;
        } else {
                channel = find_first_zero_bit(channel_usage,
                        ARRAY_SIZE(dma_channels));
                if (channel >= ARRAY_SIZE(dma_channels))
                        goto out;
        }
        __set_bit(channel, channel_usage);
        ch = &dma_channels[channel];
        ch->mode = mode;

out:
        mutex_unlock(&tegra_dma_lock);
        return ch;
}
EXPORT_SYMBOL(tegra_dma_allocate_channel);

void tegra_dma_free_channel(struct tegra_dma_channel *ch)
{
        if (ch->mode & TEGRA_DMA_SHARED)
                return;
        tegra_dma_cancel(ch);
        mutex_lock(&tegra_dma_lock);
        __clear_bit(ch->id, channel_usage);
        mutex_unlock(&tegra_dma_lock);
}
EXPORT_SYMBOL(tegra_dma_free_channel);

static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
        struct tegra_dma_req *req)
{
        u32 apb_ptr;
        u32 ahb_ptr;

        if (req->to_memory) {
                apb_ptr = req->source_addr;
                ahb_ptr = req->dest_addr;
        } else {
                apb_ptr = req->dest_addr;
                ahb_ptr = req->source_addr;
        }
        writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
        writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);

        req->status = TEGRA_DMA_REQ_INFLIGHT;
        return;
}

static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
        struct tegra_dma_req *req)
{
        int ahb_addr_wrap;
        int apb_addr_wrap;
        int ahb_bus_width;
        int apb_bus_width;
        int index;

        u32 ahb_seq;
        u32 apb_seq;
        u32 ahb_ptr;
        u32 apb_ptr;
        u32 csr;

        csr = CSR_IE_EOC | CSR_FLOW;
        ahb_seq = AHB_SEQ_INTR_ENB | AHB_SEQ_BURST_1;
        apb_seq = 0;

        csr |= req->req_sel << CSR_REQ_SEL_SHIFT;

        /* One shot mode is always single buffered,
         * continuous mode is always double buffered
         * */
        if (ch->mode & TEGRA_DMA_MODE_ONESHOT) {
                csr |= CSR_ONCE;
                ch->req_transfer_count = (req->size >> 2) - 1;
        } else {
                ahb_seq |= AHB_SEQ_DBL_BUF;

                /* In double buffered mode, we set the size to half the
                 * requested size and interrupt when half the buffer
                 * is full */
                ch->req_transfer_count = (req->size >> 3) - 1;
        }

        csr |= ch->req_transfer_count << CSR_WCOUNT_SHIFT;

        if (req->to_memory) {
                apb_ptr = req->source_addr;
                ahb_ptr = req->dest_addr;

                apb_addr_wrap = req->source_wrap;
                ahb_addr_wrap = req->dest_wrap;
                apb_bus_width = req->source_bus_width;
                ahb_bus_width = req->dest_bus_width;

        } else {
                csr |= CSR_DIR;
                apb_ptr = req->dest_addr;
                ahb_ptr = req->source_addr;

                apb_addr_wrap = req->dest_wrap;
                ahb_addr_wrap = req->source_wrap;
                apb_bus_width = req->dest_bus_width;
                ahb_bus_width = req->source_bus_width;
        }

        apb_addr_wrap >>= 2;
        ahb_addr_wrap >>= 2;

        /* set address wrap for APB size */
        index = 0;
        do  {
                if (apb_addr_wrap_table[index] == apb_addr_wrap)
                        break;
                index++;
        } while (index < ARRAY_SIZE(apb_addr_wrap_table));
        BUG_ON(index == ARRAY_SIZE(apb_addr_wrap_table));
        apb_seq |= index << APB_SEQ_WRAP_SHIFT;

        /* set address wrap for AHB size */
        index = 0;
        do  {
                if (ahb_addr_wrap_table[index] == ahb_addr_wrap)
                        break;
                index++;
        } while (index < ARRAY_SIZE(ahb_addr_wrap_table));
        BUG_ON(index == ARRAY_SIZE(ahb_addr_wrap_table));
        ahb_seq |= index << AHB_SEQ_WRAP_SHIFT;

        for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
                if (bus_width_table[index] == ahb_bus_width)
                        break;
        }
        BUG_ON(index == ARRAY_SIZE(bus_width_table));
        ahb_seq |= index << AHB_SEQ_BUS_WIDTH_SHIFT;

        for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
                if (bus_width_table[index] == apb_bus_width)
                        break;
        }
        BUG_ON(index == ARRAY_SIZE(bus_width_table));
        apb_seq |= index << APB_SEQ_BUS_WIDTH_SHIFT;

        writel(csr, ch->addr + APB_DMA_CHAN_CSR);
        writel(apb_seq, ch->addr + APB_DMA_CHAN_APB_SEQ);
        writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
        writel(ahb_seq, ch->addr + APB_DMA_CHAN_AHB_SEQ);
        writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);

        csr |= CSR_ENB;
        writel(csr, ch->addr + APB_DMA_CHAN_CSR);

        req->status = TEGRA_DMA_REQ_INFLIGHT;
}

static void handle_oneshot_dma(struct tegra_dma_channel *ch)
{
        struct tegra_dma_req *req;
        unsigned long irq_flags;

        spin_lock_irqsave(&ch->lock, irq_flags);
        if (list_empty(&ch->list)) {
                spin_unlock_irqrestore(&ch->lock, irq_flags);
                return;
        }

        req = list_entry(ch->list.next, typeof(*req), node);
        if (req) {
                int bytes_transferred;

                bytes_transferred = ch->req_transfer_count;
                bytes_transferred += 1;
                bytes_transferred <<= 2;

                list_del(&req->node);
                req->bytes_transferred = bytes_transferred;
                req->status = TEGRA_DMA_REQ_SUCCESS;

                spin_unlock_irqrestore(&ch->lock, irq_flags);
                /* Callback should be called without any lock */
                pr_debug("%s: transferred %d bytes\n", __func__,
                        req->bytes_transferred);
                req->complete(req);
                spin_lock_irqsave(&ch->lock, irq_flags);
        }

        if (!list_empty(&ch->list)) {
                req = list_entry(ch->list.next, typeof(*req), node);
                /* the complete function we just called may have enqueued
                   another req, in which case dma has already started */
                if (req->status != TEGRA_DMA_REQ_INFLIGHT)
                        tegra_dma_update_hw(ch, req);
        }
        spin_unlock_irqrestore(&ch->lock, irq_flags);
}

static void handle_continuous_dma(struct tegra_dma_channel *ch)
{
        struct tegra_dma_req *req;
        unsigned long irq_flags;

        spin_lock_irqsave(&ch->lock, irq_flags);
        if (list_empty(&ch->list)) {
                spin_unlock_irqrestore(&ch->lock, irq_flags);
                return;
        }

        req = list_entry(ch->list.next, typeof(*req), node);
        if (req) {
                if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_EMPTY) {
                        bool is_dma_ping_complete;
                        is_dma_ping_complete = (readl(ch->addr + APB_DMA_CHAN_STA)
                                                & STA_PING_PONG) ? true : false;
                        if (req->to_memory)
                                is_dma_ping_complete = !is_dma_ping_complete;
                        /* Out of sync - Release current buffer */
                        if (!is_dma_ping_complete) {
                                int bytes_transferred;

                                bytes_transferred = ch->req_transfer_count;
                                bytes_transferred += 1;
                                bytes_transferred <<= 3;
                                req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
                                req->bytes_transferred = bytes_transferred;
                                req->status = TEGRA_DMA_REQ_SUCCESS;
                                tegra_dma_stop(ch);

                                if (!list_is_last(&req->node, &ch->list)) {
                                        struct tegra_dma_req *next_req;

                                        next_req = list_entry(req->node.next,
                                                typeof(*next_req), node);
                                        tegra_dma_update_hw(ch, next_req);
                                }

                                list_del(&req->node);

                                /* DMA lock is NOT held when callbak is called */
                                spin_unlock_irqrestore(&ch->lock, irq_flags);
                                req->complete(req);
                                return;
                        }
                        /* Load the next request into the hardware, if available
                         * */
                        if (!list_is_last(&req->node, &ch->list)) {
                                struct tegra_dma_req *next_req;

                                next_req = list_entry(req->node.next,
                                        typeof(*next_req), node);
                                tegra_dma_update_hw_partial(ch, next_req);
                        }
                        req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL;
                        req->status = TEGRA_DMA_REQ_SUCCESS;
                        /* DMA lock is NOT held when callback is called */
                        spin_unlock_irqrestore(&ch->lock, irq_flags);
                        if (likely(req->threshold))
                                req->threshold(req);
                        return;

                } else if (req->buffer_status ==
                        TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL) {
                        /* Callback when the buffer is completely full (i.e on
                         * the second  interrupt */
                        int bytes_transferred;

                        bytes_transferred = ch->req_transfer_count;
                        bytes_transferred += 1;
                        bytes_transferred <<= 3;

                        req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
                        req->bytes_transferred = bytes_transferred;
                        req->status = TEGRA_DMA_REQ_SUCCESS;
                        list_del(&req->node);

                        /* DMA lock is NOT held when callbak is called */
                        spin_unlock_irqrestore(&ch->lock, irq_flags);
                        req->complete(req);
                        return;

                } else {
                        BUG();
                }
        }
        spin_unlock_irqrestore(&ch->lock, irq_flags);
}

static irqreturn_t dma_isr(int irq, void *data)
{
        struct tegra_dma_channel *ch = data;
        unsigned long status;

        status = readl(ch->addr + APB_DMA_CHAN_STA);
        if (status & STA_ISE_EOC)
                writel(status, ch->addr + APB_DMA_CHAN_STA);
        else {
                pr_warning("Got a spurious ISR for DMA channel %d\n", ch->id);
                return IRQ_HANDLED;
        }
        return IRQ_WAKE_THREAD;
}

static irqreturn_t dma_thread_fn(int irq, void *data)
{
        struct tegra_dma_channel *ch = data;

        if (ch->mode & TEGRA_DMA_MODE_ONESHOT)
                handle_oneshot_dma(ch);
        else
                handle_continuous_dma(ch);


        return IRQ_HANDLED;
}

int __init tegra_dma_init(void)
{
        int ret = 0;
        int i;
        unsigned int irq;
        void __iomem *addr;
        struct clk *c;

        bitmap_fill(channel_usage, NV_DMA_MAX_CHANNELS);

        c = clk_get_sys("tegra-dma", NULL);
        if (IS_ERR(c)) {
                pr_err("Unable to get clock for APB DMA\n");
                ret = PTR_ERR(c);
                goto fail;
        }
        ret = clk_enable(c);
        if (ret != 0) {
                pr_err("Unable to enable clock for APB DMA\n");
                goto fail;
        }

        addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
        writel(GEN_ENABLE, addr + APB_DMA_GEN);
        writel(0, addr + APB_DMA_CNTRL);
        writel(0xFFFFFFFFul >> (31 - TEGRA_SYSTEM_DMA_CH_MAX),
               addr + APB_DMA_IRQ_MASK_SET);

        for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
                struct tegra_dma_channel *ch = &dma_channels[i];

                ch->id = i;
                snprintf(ch->name, TEGRA_DMA_NAME_SIZE, "dma_channel_%d", i);

                ch->addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
                        TEGRA_APB_DMA_CH0_SIZE * i);

                spin_lock_init(&ch->lock);
                INIT_LIST_HEAD(&ch->list);

                irq = INT_APB_DMA_CH0 + i;
                ret = request_threaded_irq(irq, dma_isr, dma_thread_fn, 0,
                        dma_channels[i].name, ch);
                if (ret) {
                        pr_err("Failed to register IRQ %d for DMA %d\n",
                                irq, i);
                        goto fail;
                }
                ch->irq = irq;

                __clear_bit(i, channel_usage);
        }
        /* mark the shared channel allocated */
        __set_bit(TEGRA_SYSTEM_DMA_CH_MIN, channel_usage);

        tegra_dma_initialized = true;

        return 0;
fail:
        writel(0, addr + APB_DMA_GEN);
        for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
                struct tegra_dma_channel *ch = &dma_channels[i];
                if (ch->irq)
                        free_irq(ch->irq, ch);
        }
        return ret;
}
postcore_initcall(tegra_dma_init);

#ifdef CONFIG_PM
static u32 apb_dma[5*TEGRA_SYSTEM_DMA_CH_NR + 3];

void tegra_dma_suspend(void)
{
        void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
        u32 *ctx = apb_dma;
        int i;

        *ctx++ = readl(addr + APB_DMA_GEN);
        *ctx++ = readl(addr + APB_DMA_CNTRL);
        *ctx++ = readl(addr + APB_DMA_IRQ_MASK);

        for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) {
                addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
                                  TEGRA_APB_DMA_CH0_SIZE * i);

                *ctx++ = readl(addr + APB_DMA_CHAN_CSR);
                *ctx++ = readl(addr + APB_DMA_CHAN_AHB_PTR);
                *ctx++ = readl(addr + APB_DMA_CHAN_AHB_SEQ);
                *ctx++ = readl(addr + APB_DMA_CHAN_APB_PTR);
                *ctx++ = readl(addr + APB_DMA_CHAN_APB_SEQ);
        }
}

void tegra_dma_resume(void)
{
        void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
        u32 *ctx = apb_dma;
        int i;

        writel(*ctx++, addr + APB_DMA_GEN);
        writel(*ctx++, addr + APB_DMA_CNTRL);
        writel(*ctx++, addr + APB_DMA_IRQ_MASK);

        for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) {
                addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
                                  TEGRA_APB_DMA_CH0_SIZE * i);

                writel(*ctx++, addr + APB_DMA_CHAN_CSR);
                writel(*ctx++, addr + APB_DMA_CHAN_AHB_PTR);
                writel(*ctx++, addr + APB_DMA_CHAN_AHB_SEQ);
                writel(*ctx++, addr + APB_DMA_CHAN_APB_PTR);
                writel(*ctx++, addr + APB_DMA_CHAN_APB_SEQ);
        }
}

#endif