gpu/host1x/cdma.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Tegra host1x Command DMA
 *
 * Copyright (c) 2010-2013, NVIDIA Corporation.
 */


#include <asm/cacheflush.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/host1x.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/kfifo.h>
#include <linux/overflow.h>
#include <linux/slab.h>
#include <trace/events/host1x.h>

#include "cdma.h"
#include "channel.h"
#include "dev.h"
#include "debug.h"
#include "job.h"

/*
 * push_buffer
 *
 * The push buffer is a circular array of words to be fetched by command DMA.
 * Note that it works slightly differently to the sync queue; fence == pos
 * means that the push buffer is full, not empty.
 */

/*
 * Typically the commands written into the push buffer are a pair of words. We
 * use slots to represent each of these pairs and to simplify things. Note the
 * strange number of slots allocated here. 512 slots will fit exactly within a
 * single memory page. We also need one additional word at the end of the push
 * buffer for the RESTART opcode that will instruct the CDMA to jump back to
 * the beginning of the push buffer. With 512 slots, this means that we'll use
 * 2 memory pages and waste 4092 bytes of the second page that will never be
 * used.
 */
#define HOST1X_PUSHBUFFER_SLOTS	511

/*
 * Clean up push buffer resources
 */
static void host1x_pushbuffer_destroy(struct push_buffer *pb)
{
	struct host1x_cdma *cdma = pb_to_cdma(pb);
	struct host1x *host1x = cdma_to_host1x(cdma);

	if (!pb->mapped)
		return;

	if (host1x->domain) {
		iommu_unmap(host1x->domain, pb->dma, pb->alloc_size);
		free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma));
	}

	dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys);

	pb->mapped = NULL;
	pb->phys = 0;
}

/*
 * Init push buffer resources
 */
static int host1x_pushbuffer_init(struct push_buffer *pb)
{
	struct host1x_cdma *cdma = pb_to_cdma(pb);
	struct host1x *host1x = cdma_to_host1x(cdma);
	struct iova *alloc;
	u32 size;
	int err;

	pb->mapped = NULL;
	pb->phys = 0;
	pb->size = HOST1X_PUSHBUFFER_SLOTS * 8;

	size = pb->size + 4;

	/* initialize buffer pointers */
	pb->fence = pb->size - 8;
	pb->pos = 0;

	if (host1x->domain) {
		unsigned long shift;

		size = iova_align(&host1x->iova, size);

		pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
					  GFP_KERNEL);
		if (!pb->mapped)
			return -ENOMEM;

		shift = iova_shift(&host1x->iova);
		alloc = alloc_iova(&host1x->iova, size >> shift,
				   host1x->iova_end >> shift, true);
		if (!alloc) {
			err = -ENOMEM;
			goto iommu_free_mem;
		}

		pb->dma = iova_dma_addr(&host1x->iova, alloc);
		err = iommu_map(host1x->domain, pb->dma, pb->phys, size,
				IOMMU_READ, GFP_KERNEL);
		if (err)
			goto iommu_free_iova;
	} else {
		pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
					  GFP_KERNEL);
		if (!pb->mapped)
			return -ENOMEM;

		pb->dma = pb->phys;
	}

	pb->alloc_size = size;

	host1x_hw_pushbuffer_init(host1x, pb);

	return 0;

iommu_free_iova:
	__free_iova(&host1x->iova, alloc);
iommu_free_mem:
	dma_free_wc(host1x->dev, size, pb->mapped, pb->phys);

	return err;
}

/*
 * Push two words to the push buffer
 * Caller must ensure push buffer is not full
 */
static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)
{
	u32 *p = (u32 *)((void *)pb->mapped + pb->pos);

	WARN_ON(pb->pos == pb->fence);
	*(p++) = op1;
	*(p++) = op2;
	pb->pos += 8;

	if (pb->pos >= pb->size)
		pb->pos -= pb->size;
}

/*
 * Pop a number of two word slots from the push buffer
 * Caller must ensure push buffer is not empty
 */
static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots)
{
	/* Advance the next write position */
	pb->fence += slots * 8;

	if (pb->fence >= pb->size)
		pb->fence -= pb->size;
}

/*
 * Return the number of two word slots free in the push buffer
 */
static u32 host1x_pushbuffer_space(struct push_buffer *pb)
{
	unsigned int fence = pb->fence;

	if (pb->fence < pb->pos)
		fence += pb->size;

	return (fence - pb->pos) / 8;
}

/*
 * Sleep (if necessary) until the requested event happens
 *   - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
 *     - Returns 1
 *   - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
 *     - Return the amount of space (> 0)
 * Must be called with the cdma lock held.
 */
unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
				     enum cdma_event event)
{
	for (;;) {
		struct push_buffer *pb = &cdma->push_buffer;
		unsigned int space;

		switch (event) {
		case CDMA_EVENT_SYNC_QUEUE_EMPTY:
			space = list_empty(&cdma->sync_queue) ? 1 : 0;
			break;

		case CDMA_EVENT_PUSH_BUFFER_SPACE:
			space = host1x_pushbuffer_space(pb);
			break;

		default:
			WARN_ON(1);
			return -EINVAL;
		}

		if (space)
			return space;

		trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
				       event);

		/* If somebody has managed to already start waiting, yield */
		if (cdma->event != CDMA_EVENT_NONE) {
			mutex_unlock(&cdma->lock);
			schedule();
			mutex_lock(&cdma->lock);
			continue;
		}

		cdma->event = event;

		mutex_unlock(&cdma->lock);
		wait_for_completion(&cdma->complete);
		mutex_lock(&cdma->lock);
	}

	return 0;
}

/*
 * Sleep (if necessary) until the push buffer has enough free space.
 *
 * Must be called with the cdma lock held.
 */
static int host1x_cdma_wait_pushbuffer_space(struct host1x *host1x,
					     struct host1x_cdma *cdma,
					     unsigned int needed)
{
	while (true) {
		struct push_buffer *pb = &cdma->push_buffer;
		unsigned int space;

		space = host1x_pushbuffer_space(pb);
		if (space >= needed)
			break;

		trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
				       CDMA_EVENT_PUSH_BUFFER_SPACE);

		/* If somebody has managed to already start waiting, yield */
		if (cdma->event != CDMA_EVENT_NONE) {
			mutex_unlock(&cdma->lock);
			schedule();
			mutex_lock(&cdma->lock);
			continue;
		}

		cdma->event = CDMA_EVENT_PUSH_BUFFER_SPACE;

		mutex_unlock(&cdma->lock);
		wait_for_completion(&cdma->complete);
		mutex_lock(&cdma->lock);
	}

	return 0;
}
/*
 * Start timer that tracks the time spent by the job.
 * Must be called with the cdma lock held.
 */
static void cdma_start_timer_locked(struct host1x_cdma *cdma,
				    struct host1x_job *job)
{
	if (cdma->timeout.client) {
		/* timer already started */
		return;
	}

	cdma->timeout.client = job->client;
	cdma->timeout.syncpt = job->syncpt;
	cdma->timeout.syncpt_val = job->syncpt_end;
	cdma->timeout.start_ktime = ktime_get();

	schedule_delayed_work(&cdma->timeout.wq,
			      msecs_to_jiffies(job->timeout));
}

/*
 * Stop timer when a buffer submission completes.
 * Must be called with the cdma lock held.
 */
static void stop_cdma_timer_locked(struct host1x_cdma *cdma)
{
	cancel_delayed_work(&cdma->timeout.wq);
	cdma->timeout.client = NULL;
}

/*
 * For all sync queue entries that have already finished according to the
 * current sync point registers:
 *  - unpin & unref their mems
 *  - pop their push buffer slots
 *  - remove them from the sync queue
 * This is normally called from the host code's worker thread, but can be
 * called manually if necessary.
 * Must be called with the cdma lock held.
 */
static void update_cdma_locked(struct host1x_cdma *cdma)
{
	bool signal = false;
	struct host1x_job *job, *n;

	/*
	 * Walk the sync queue, reading the sync point registers as necessary,
	 * to consume as many sync queue entries as possible without blocking
	 */
	list_for_each_entry_safe(job, n, &cdma->sync_queue, list) {
		struct host1x_syncpt *sp = job->syncpt;

		/* Check whether this syncpt has completed, and bail if not */
		if (!host1x_syncpt_is_expired(sp, job->syncpt_end) &&
		    !job->cancelled) {
			/* Start timer on next pending syncpt */
			if (job->timeout)
				cdma_start_timer_locked(cdma, job);

			break;
		}

		/* Cancel timeout, when a buffer completes */
		if (cdma->timeout.client)
			stop_cdma_timer_locked(cdma);

		/* Unpin the memory */
		host1x_job_unpin(job);

		/* Pop push buffer slots */
		if (job->num_slots) {
			struct push_buffer *pb = &cdma->push_buffer;

			host1x_pushbuffer_pop(pb, job->num_slots);

			if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
				signal = true;
		}

		list_del(&job->list);
		host1x_job_put(job);
	}

	if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY &&
	    list_empty(&cdma->sync_queue))
		signal = true;

	if (signal) {
		cdma->event = CDMA_EVENT_NONE;
		complete(&cdma->complete);
	}
}

void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
				   struct device *dev)
{
	struct host1x *host1x = cdma_to_host1x(cdma);
	u32 restart_addr, syncpt_incrs, syncpt_val;
	struct host1x_job *job, *next_job = NULL;

	syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);

	dev_dbg(dev, "%s: starting cleanup (thresh %d)\n",
		__func__, syncpt_val);

	/*
	 * Move the sync_queue read pointer to the first entry that hasn't
	 * completed based on the current HW syncpt value. It's likely there
	 * won't be any (i.e. we're still at the head), but covers the case
	 * where a syncpt incr happens just prior/during the teardown.
	 */

	dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n",
		__func__);

	list_for_each_entry(job, &cdma->sync_queue, list) {
		if (syncpt_val < job->syncpt_end) {

			if (!list_is_last(&job->list, &cdma->sync_queue))
				next_job = list_next_entry(job, list);

			goto syncpt_incr;
		}

		host1x_job_dump(dev, job);
	}

	/* all jobs have been completed */
	job = NULL;

syncpt_incr:

	/*
	 * Increment with CPU the remaining syncpts of a partially executed job.
	 *
	 * CDMA will continue execution starting with the next job or will get
	 * into idle state.
	 */
	if (next_job)
		restart_addr = next_job->first_get;
	else
		restart_addr = cdma->last_pos;

	if (!job)
		goto resume;

	/* do CPU increments for the remaining syncpts */
	if (job->syncpt_recovery) {
		dev_dbg(dev, "%s: perform CPU incr on pending buffers\n",
			__func__);

		/* won't need a timeout when replayed */
		job->timeout = 0;

		syncpt_incrs = wrapping_sub(u32, job->syncpt_end, syncpt_val);
		dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);

		host1x_job_dump(dev, job);

		/* safe to use CPU to incr syncpts */
		host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get,
						syncpt_incrs, job->syncpt_end,
						job->num_slots);

		dev_dbg(dev, "%s: finished sync_queue modification\n",
			__func__);
	} else {
		struct host1x_job *failed_job = job;

		host1x_job_dump(dev, job);

		host1x_syncpt_set_locked(job->syncpt);
		failed_job->cancelled = true;

		list_for_each_entry_continue(job, &cdma->sync_queue, list) {
			unsigned int i;

			if (job->syncpt != failed_job->syncpt)
				continue;

			for (i = 0; i < job->num_slots; i++) {
				unsigned int slot = (job->first_get/8 + i) %
						    HOST1X_PUSHBUFFER_SLOTS;
				u32 *mapped = cdma->push_buffer.mapped;

				/*
				 * Overwrite opcodes with 0 word writes
				 * to offset 0xbad. This does nothing but
				 * has a easily detected signature in debug
				 * traces.
				 *
				 * On systems with MLOCK enforcement enabled,
				 * the above 0 word writes would fall foul of
				 * the enforcement. As such, in the first slot
				 * put a RESTART_W opcode to the beginning
				 * of the next job. We don't use this for older
				 * chips since those only support the RESTART
				 * opcode with inconvenient alignment requirements.
				 */
				if (i == 0 && host1x->info->has_wide_gather) {
					unsigned int next_job = (job->first_get/8 + job->num_slots)
						% HOST1X_PUSHBUFFER_SLOTS;
					mapped[2*slot+0] = (0xd << 28) | (next_job * 2);
					mapped[2*slot+1] = 0x0;
				} else {
					mapped[2*slot+0] = 0x1bad0000;
					mapped[2*slot+1] = 0x1bad0000;
				}
			}

			job->cancelled = true;
		}

		wmb();

		update_cdma_locked(cdma);
	}

resume:
	/* roll back DMAGET and start up channel again */
	host1x_hw_cdma_resume(host1x, cdma, restart_addr);
}

static void cdma_update_work(struct work_struct *work)
{
	struct host1x_cdma *cdma = container_of(work, struct host1x_cdma, update_work);

	mutex_lock(&cdma->lock);
	update_cdma_locked(cdma);
	mutex_unlock(&cdma->lock);
}

/*
 * Create a cdma
 */
int host1x_cdma_init(struct host1x_cdma *cdma)
{
	int err;

	mutex_init(&cdma->lock);
	init_completion(&cdma->complete);
	INIT_WORK(&cdma->update_work, cdma_update_work);

	INIT_LIST_HEAD(&cdma->sync_queue);

	cdma->event = CDMA_EVENT_NONE;
	cdma->running = false;
	cdma->torndown = false;

	err = host1x_pushbuffer_init(&cdma->push_buffer);
	if (err)
		return err;

	return 0;
}

/*
 * Destroy a cdma
 */
int host1x_cdma_deinit(struct host1x_cdma *cdma)
{
	struct push_buffer *pb = &cdma->push_buffer;
	struct host1x *host1x = cdma_to_host1x(cdma);

	if (cdma->running) {
		pr_warn("%s: CDMA still running\n", __func__);
		return -EBUSY;
	}

	host1x_pushbuffer_destroy(pb);
	host1x_hw_cdma_timeout_destroy(host1x, cdma);

	return 0;
}

/*
 * Begin a cdma submit
 */
int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job)
{
	struct host1x *host1x = cdma_to_host1x(cdma);

	mutex_lock(&cdma->lock);

	/*
	 * Check if syncpoint was locked due to previous job timeout.
	 * This needs to be done within the cdma lock to avoid a race
	 * with the timeout handler.
	 */
	if (job->syncpt->locked) {
		mutex_unlock(&cdma->lock);
		return -EPERM;
	}

	if (job->timeout) {
		/* init state on first submit with timeout value */
		if (!cdma->timeout.initialized) {
			int err;

			err = host1x_hw_cdma_timeout_init(host1x, cdma);
			if (err) {
				mutex_unlock(&cdma->lock);
				return err;
			}
		}
	}

	if (!cdma->running)
		host1x_hw_cdma_start(host1x, cdma);

	cdma->slots_free = 0;
	cdma->slots_used = 0;
	cdma->first_get = cdma->push_buffer.pos;

	trace_host1x_cdma_begin(dev_name(job->channel->dev));
	return 0;
}

/*
 * Push two words into a push buffer slot
 * Blocks as necessary if the push buffer is full.
 */
void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)
{
	struct push_buffer *pb = &cdma->push_buffer;
	u32 slots_free = cdma->slots_free;

	if (host1x_debug_trace_cmdbuf)
		trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev),
				       op1, op2);

	if (slots_free == 0)
		slots_free = host1x_cdma_wait_locked(cdma,
						CDMA_EVENT_PUSH_BUFFER_SPACE);

	cdma->slots_free = slots_free - 1;
	cdma->slots_used++;
	host1x_pushbuffer_push(pb, op1, op2);
}

/*
 * Push four words into two consecutive push buffer slots. Note that extra
 * care needs to be taken not to split the two slots across the end of the
 * push buffer. Otherwise the RESTART opcode at the end of the push buffer
 * that ensures processing will restart at the beginning will break up the
 * four words.
 *
 * Blocks as necessary if the push buffer is full.
 */
void host1x_cdma_push_wide(struct host1x_cdma *cdma, u32 op1, u32 op2,
			   u32 op3, u32 op4)
{
	struct host1x_channel *channel = cdma_to_channel(cdma);
	struct host1x *host1x = cdma_to_host1x(cdma);
	struct push_buffer *pb = &cdma->push_buffer;
	unsigned int space, needed = 2, extra = 0;

	if (host1x_debug_trace_cmdbuf)
		trace_host1x_cdma_push_wide(dev_name(channel->dev), op1, op2,
					    op3, op4);

	/* compute number of extra slots needed for padding */
	if (pb->pos + 16 > pb->size) {
		extra = (pb->size - pb->pos) / 8;
		needed += extra;
	}

	host1x_cdma_wait_pushbuffer_space(host1x, cdma, needed);
	space = host1x_pushbuffer_space(pb);

	cdma->slots_free = space - needed;
	cdma->slots_used += needed;

	if (extra > 0) {
		/*
		 * If there isn't enough space at the tail of the pushbuffer,
		 * insert a RESTART(0) here to go back to the beginning.
		 * The code above adjusted the indexes appropriately.
		 */
		host1x_pushbuffer_push(pb, (0x5 << 28), 0xdead0000);
	}

	host1x_pushbuffer_push(pb, op1, op2);
	host1x_pushbuffer_push(pb, op3, op4);
}

/*
 * End a cdma submit
 * Kick off DMA, add job to the sync queue, and a number of slots to be freed
 * from the pushbuffer. The handles for a submit must all be pinned at the same
 * time, but they can be unpinned in smaller chunks.
 */
void host1x_cdma_end(struct host1x_cdma *cdma,
		     struct host1x_job *job)
{
	struct host1x *host1x = cdma_to_host1x(cdma);
	bool idle = list_empty(&cdma->sync_queue);

	host1x_hw_cdma_flush(host1x, cdma);

	job->first_get = cdma->first_get;
	job->num_slots = cdma->slots_used;
	host1x_job_get(job);
	list_add_tail(&job->list, &cdma->sync_queue);

	/* start timer on idle -> active transitions */
	if (job->timeout && idle)
		cdma_start_timer_locked(cdma, job);

	trace_host1x_cdma_end(dev_name(job->channel->dev));
	mutex_unlock(&cdma->lock);
}

/*
 * Update cdma state according to current sync point values
 */
void host1x_cdma_update(struct host1x_cdma *cdma)
{
	schedule_work(&cdma->update_work);
}