Linux 4.19-rc7

[linux-2.6/btrfs-unstable.git] / drivers / gpu / host1x / cdma.c

/*
 * Tegra host1x Command DMA
 *
 * Copyright (c) 2010-2013, NVIDIA Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */


#include <asm/cacheflush.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/host1x.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/kfifo.h>
#include <linux/slab.h>
#include <trace/events/host1x.h>

#include "cdma.h"
#include "channel.h"
#include "dev.h"
#include "debug.h"
#include "job.h"

/*
 * push_buffer
 *
 * The push buffer is a circular array of words to be fetched by command DMA.
 * Note that it works slightly differently to the sync queue; fence == pos
 * means that the push buffer is full, not empty.
 */

#define HOST1X_PUSHBUFFER_SLOTS 512

/*
 * Clean up push buffer resources
 */
static void host1x_pushbuffer_destroy(struct push_buffer *pb)
{
        struct host1x_cdma *cdma = pb_to_cdma(pb);
        struct host1x *host1x = cdma_to_host1x(cdma);

        if (!pb->mapped)
                return;

        if (host1x->domain) {
                iommu_unmap(host1x->domain, pb->dma, pb->alloc_size);
                free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma));
        }

        dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys);

        pb->mapped = NULL;
        pb->phys = 0;
}

/*
 * Init push buffer resources
 */
static int host1x_pushbuffer_init(struct push_buffer *pb)
{
        struct host1x_cdma *cdma = pb_to_cdma(pb);
        struct host1x *host1x = cdma_to_host1x(cdma);
        struct iova *alloc;
        u32 size;
        int err;

        pb->mapped = NULL;
        pb->phys = 0;
        pb->size = HOST1X_PUSHBUFFER_SLOTS * 8;

        size = pb->size + 4;

        /* initialize buffer pointers */
        pb->fence = pb->size - 8;
        pb->pos = 0;

        if (host1x->domain) {
                unsigned long shift;

                size = iova_align(&host1x->iova, size);

                pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
                                          GFP_KERNEL);
                if (!pb->mapped)
                        return -ENOMEM;

                shift = iova_shift(&host1x->iova);
                alloc = alloc_iova(&host1x->iova, size >> shift,
                                   host1x->iova_end >> shift, true);
                if (!alloc) {
                        err = -ENOMEM;
                        goto iommu_free_mem;
                }

                pb->dma = iova_dma_addr(&host1x->iova, alloc);
                err = iommu_map(host1x->domain, pb->dma, pb->phys, size,
                                IOMMU_READ);
                if (err)
                        goto iommu_free_iova;
        } else {
                pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
                                          GFP_KERNEL);
                if (!pb->mapped)
                        return -ENOMEM;

                pb->dma = pb->phys;
        }

        pb->alloc_size = size;

        host1x_hw_pushbuffer_init(host1x, pb);

        return 0;

iommu_free_iova:
        __free_iova(&host1x->iova, alloc);
iommu_free_mem:
        dma_free_wc(host1x->dev, size, pb->mapped, pb->phys);

        return err;
}

/*
 * Push two words to the push buffer
 * Caller must ensure push buffer is not full
 */
static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)
{
        u32 *p = (u32 *)((void *)pb->mapped + pb->pos);

        WARN_ON(pb->pos == pb->fence);
        *(p++) = op1;
        *(p++) = op2;
        pb->pos = (pb->pos + 8) & (pb->size - 1);
}

/*
 * Pop a number of two word slots from the push buffer
 * Caller must ensure push buffer is not empty
 */
static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots)
{
        /* Advance the next write position */
        pb->fence = (pb->fence + slots * 8) & (pb->size - 1);
}

/*
 * Return the number of two word slots free in the push buffer
 */
static u32 host1x_pushbuffer_space(struct push_buffer *pb)
{
        return ((pb->fence - pb->pos) & (pb->size - 1)) / 8;
}

/*
 * Sleep (if necessary) until the requested event happens
 *   - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
 *     - Returns 1
 *   - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
 *     - Return the amount of space (> 0)
 * Must be called with the cdma lock held.
 */
unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
                                     enum cdma_event event)
{
        for (;;) {
                struct push_buffer *pb = &cdma->push_buffer;
                unsigned int space;

                switch (event) {
                case CDMA_EVENT_SYNC_QUEUE_EMPTY:
                        space = list_empty(&cdma->sync_queue) ? 1 : 0;
                        break;

                case CDMA_EVENT_PUSH_BUFFER_SPACE:
                        space = host1x_pushbuffer_space(pb);
                        break;

                default:
                        WARN_ON(1);
                        return -EINVAL;
                }

                if (space)
                        return space;

                trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
                                       event);

                /* If somebody has managed to already start waiting, yield */
                if (cdma->event != CDMA_EVENT_NONE) {
                        mutex_unlock(&cdma->lock);
                        schedule();
                        mutex_lock(&cdma->lock);
                        continue;
                }

                cdma->event = event;

                mutex_unlock(&cdma->lock);
                down(&cdma->sem);
                mutex_lock(&cdma->lock);
        }

        return 0;
}

/*
 * Start timer that tracks the time spent by the job.
 * Must be called with the cdma lock held.
 */
static void cdma_start_timer_locked(struct host1x_cdma *cdma,
                                    struct host1x_job *job)
{
        struct host1x *host = cdma_to_host1x(cdma);

        if (cdma->timeout.client) {
                /* timer already started */
                return;
        }

        cdma->timeout.client = job->client;
        cdma->timeout.syncpt = host1x_syncpt_get(host, job->syncpt_id);
        cdma->timeout.syncpt_val = job->syncpt_end;
        cdma->timeout.start_ktime = ktime_get();

        schedule_delayed_work(&cdma->timeout.wq,
                              msecs_to_jiffies(job->timeout));
}

/*
 * Stop timer when a buffer submission completes.
 * Must be called with the cdma lock held.
 */
static void stop_cdma_timer_locked(struct host1x_cdma *cdma)
{
        cancel_delayed_work(&cdma->timeout.wq);
        cdma->timeout.client = NULL;
}

/*
 * For all sync queue entries that have already finished according to the
 * current sync point registers:
 *  - unpin & unref their mems
 *  - pop their push buffer slots
 *  - remove them from the sync queue
 * This is normally called from the host code's worker thread, but can be
 * called manually if necessary.
 * Must be called with the cdma lock held.
 */
static void update_cdma_locked(struct host1x_cdma *cdma)
{
        bool signal = false;
        struct host1x *host1x = cdma_to_host1x(cdma);
        struct host1x_job *job, *n;

        /* If CDMA is stopped, queue is cleared and we can return */
        if (!cdma->running)
                return;

        /*
         * Walk the sync queue, reading the sync point registers as necessary,
         * to consume as many sync queue entries as possible without blocking
         */
        list_for_each_entry_safe(job, n, &cdma->sync_queue, list) {
                struct host1x_syncpt *sp =
                        host1x_syncpt_get(host1x, job->syncpt_id);

                /* Check whether this syncpt has completed, and bail if not */
                if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) {
                        /* Start timer on next pending syncpt */
                        if (job->timeout)
                                cdma_start_timer_locked(cdma, job);

                        break;
                }

                /* Cancel timeout, when a buffer completes */
                if (cdma->timeout.client)
                        stop_cdma_timer_locked(cdma);

                /* Unpin the memory */
                host1x_job_unpin(job);

                /* Pop push buffer slots */
                if (job->num_slots) {
                        struct push_buffer *pb = &cdma->push_buffer;

                        host1x_pushbuffer_pop(pb, job->num_slots);

                        if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
                                signal = true;
                }

                list_del(&job->list);
                host1x_job_put(job);
        }

        if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY &&
            list_empty(&cdma->sync_queue))
                signal = true;

        if (signal) {
                cdma->event = CDMA_EVENT_NONE;
                up(&cdma->sem);
        }
}

void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
                                   struct device *dev)
{
        struct host1x *host1x = cdma_to_host1x(cdma);
        u32 restart_addr, syncpt_incrs, syncpt_val;
        struct host1x_job *job = NULL;

        syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);

        dev_dbg(dev, "%s: starting cleanup (thresh %d)\n",
                __func__, syncpt_val);

        /*
         * Move the sync_queue read pointer to the first entry that hasn't
         * completed based on the current HW syncpt value. It's likely there
         * won't be any (i.e. we're still at the head), but covers the case
         * where a syncpt incr happens just prior/during the teardown.
         */

        dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n",
                __func__);

        list_for_each_entry(job, &cdma->sync_queue, list) {
                if (syncpt_val < job->syncpt_end)
                        break;

                host1x_job_dump(dev, job);
        }

        /*
         * Walk the sync_queue, first incrementing with the CPU syncpts that
         * are partially executed (the first buffer) or fully skipped while
         * still in the current context (slots are also NOP-ed).
         *
         * At the point contexts are interleaved, syncpt increments must be
         * done inline with the pushbuffer from a GATHER buffer to maintain
         * the order (slots are modified to be a GATHER of syncpt incrs).
         *
         * Note: save in restart_addr the location where the timed out buffer
         * started in the PB, so we can start the refetch from there (with the
         * modified NOP-ed PB slots). This lets things appear to have completed
         * properly for this buffer and resources are freed.
         */

        dev_dbg(dev, "%s: perform CPU incr on pending same ctx buffers\n",
                __func__);

        if (!list_empty(&cdma->sync_queue))
                restart_addr = job->first_get;
        else
                restart_addr = cdma->last_pos;

        /* do CPU increments as long as this context continues */
        list_for_each_entry_from(job, &cdma->sync_queue, list) {
                /* different context, gets us out of this loop */
                if (job->client != cdma->timeout.client)
                        break;

                /* won't need a timeout when replayed */
                job->timeout = 0;

                syncpt_incrs = job->syncpt_end - syncpt_val;
                dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);

                host1x_job_dump(dev, job);

                /* safe to use CPU to incr syncpts */
                host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get,
                                                syncpt_incrs, job->syncpt_end,
                                                job->num_slots);

                syncpt_val += syncpt_incrs;
        }

        /*
         * The following sumbits from the same client may be dependent on the
         * failed submit and therefore they may fail. Force a small timeout
         * to make the queue cleanup faster.
         */

        list_for_each_entry_from(job, &cdma->sync_queue, list)
                if (job->client == cdma->timeout.client)
                        job->timeout = min_t(unsigned int, job->timeout, 500);

        dev_dbg(dev, "%s: finished sync_queue modification\n", __func__);

        /* roll back DMAGET and start up channel again */
        host1x_hw_cdma_resume(host1x, cdma, restart_addr);
}

/*
 * Create a cdma
 */
int host1x_cdma_init(struct host1x_cdma *cdma)
{
        int err;

        mutex_init(&cdma->lock);
        sema_init(&cdma->sem, 0);

        INIT_LIST_HEAD(&cdma->sync_queue);

        cdma->event = CDMA_EVENT_NONE;
        cdma->running = false;
        cdma->torndown = false;

        err = host1x_pushbuffer_init(&cdma->push_buffer);
        if (err)
                return err;

        return 0;
}

/*
 * Destroy a cdma
 */
int host1x_cdma_deinit(struct host1x_cdma *cdma)
{
        struct push_buffer *pb = &cdma->push_buffer;
        struct host1x *host1x = cdma_to_host1x(cdma);

        if (cdma->running) {
                pr_warn("%s: CDMA still running\n", __func__);
                return -EBUSY;
        }

        host1x_pushbuffer_destroy(pb);
        host1x_hw_cdma_timeout_destroy(host1x, cdma);

        return 0;
}

/*
 * Begin a cdma submit
 */
int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job)
{
        struct host1x *host1x = cdma_to_host1x(cdma);

        mutex_lock(&cdma->lock);

        if (job->timeout) {
                /* init state on first submit with timeout value */
                if (!cdma->timeout.initialized) {
                        int err;

                        err = host1x_hw_cdma_timeout_init(host1x, cdma,
                                                          job->syncpt_id);
                        if (err) {
                                mutex_unlock(&cdma->lock);
                                return err;
                        }
                }
        }

        if (!cdma->running)
                host1x_hw_cdma_start(host1x, cdma);

        cdma->slots_free = 0;
        cdma->slots_used = 0;
        cdma->first_get = cdma->push_buffer.pos;

        trace_host1x_cdma_begin(dev_name(job->channel->dev));
        return 0;
}

/*
 * Push two words into a push buffer slot
 * Blocks as necessary if the push buffer is full.
 */
void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)
{
        struct host1x *host1x = cdma_to_host1x(cdma);
        struct push_buffer *pb = &cdma->push_buffer;
        u32 slots_free = cdma->slots_free;

        if (host1x_debug_trace_cmdbuf)
                trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev),
                                       op1, op2);

        if (slots_free == 0) {
                host1x_hw_cdma_flush(host1x, cdma);
                slots_free = host1x_cdma_wait_locked(cdma,
                                                CDMA_EVENT_PUSH_BUFFER_SPACE);
        }

        cdma->slots_free = slots_free - 1;
        cdma->slots_used++;
        host1x_pushbuffer_push(pb, op1, op2);
}

/*
 * End a cdma submit
 * Kick off DMA, add job to the sync queue, and a number of slots to be freed
 * from the pushbuffer. The handles for a submit must all be pinned at the same
 * time, but they can be unpinned in smaller chunks.
 */
void host1x_cdma_end(struct host1x_cdma *cdma,
                     struct host1x_job *job)
{
        struct host1x *host1x = cdma_to_host1x(cdma);
        bool idle = list_empty(&cdma->sync_queue);

        host1x_hw_cdma_flush(host1x, cdma);

        job->first_get = cdma->first_get;
        job->num_slots = cdma->slots_used;
        host1x_job_get(job);
        list_add_tail(&job->list, &cdma->sync_queue);

        /* start timer on idle -> active transitions */
        if (job->timeout && idle)
                cdma_start_timer_locked(cdma, job);

        trace_host1x_cdma_end(dev_name(job->channel->dev));
        mutex_unlock(&cdma->lock);
}

/*
 * Update cdma state according to current sync point values
 */
void host1x_cdma_update(struct host1x_cdma *cdma)
{
        mutex_lock(&cdma->lock);
        update_cdma_locked(cdma);
        mutex_unlock(&cdma->lock);
}