engine/fifo/chan.c

/*
 * Copyright 2012 Red Hat Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Ben Skeggs
 */
#include "chan.h"
#include "chid.h"
#include "cgrp.h"
#include "runl.h"
#include "priv.h"

#include <core/ramht.h>
#include <subdev/mmu.h>
#include <engine/dma.h>

#include <nvif/if0020.h>

const struct nvkm_event_func
nvkm_chan_event = {
};

void
nvkm_chan_cctx_bind(struct nvkm_chan *chan, struct nvkm_engn *engn, struct nvkm_cctx *cctx)
{
	struct nvkm_cgrp *cgrp = chan->cgrp;
	struct nvkm_runl *runl = cgrp->runl;
	struct nvkm_engine *engine = engn->engine;

	if (!engn->func->bind)
		return;

	CHAN_TRACE(chan, "%sbind cctx %d[%s]", cctx ? "" : "un", engn->id, engine->subdev.name);

	/* Prevent any channel in channel group from being rescheduled, kick them
	 * off host and any engine(s) they're loaded on.
	 */
	if (cgrp->hw)
		nvkm_runl_block(runl);
	else
		nvkm_chan_block(chan);
	nvkm_chan_preempt(chan, true);

	/* Update context pointer. */
	engn->func->bind(engn, cctx, chan);

	/* Resume normal operation. */
	if (cgrp->hw)
		nvkm_runl_allow(runl);
	else
		nvkm_chan_allow(chan);
}

void
nvkm_chan_cctx_put(struct nvkm_chan *chan, struct nvkm_cctx **pcctx)
{
	struct nvkm_cctx *cctx = *pcctx;

	if (cctx) {
		struct nvkm_engn *engn = cctx->vctx->ectx->engn;

		if (refcount_dec_and_mutex_lock(&cctx->refs, &chan->cgrp->mutex)) {
			CHAN_TRACE(chan, "dtor cctx %d[%s]", engn->id, engn->engine->subdev.name);
			nvkm_cgrp_vctx_put(chan->cgrp, &cctx->vctx);
			list_del(&cctx->head);
			kfree(cctx);
			mutex_unlock(&chan->cgrp->mutex);
		}

		*pcctx = NULL;
	}
}

int
nvkm_chan_cctx_get(struct nvkm_chan *chan, struct nvkm_engn *engn, struct nvkm_cctx **pcctx,
		   struct nvkm_client *client)
{
	struct nvkm_cgrp *cgrp = chan->cgrp;
	struct nvkm_vctx *vctx;
	struct nvkm_cctx *cctx;
	int ret;

	/* Look for an existing channel context for this engine+VEID. */
	mutex_lock(&cgrp->mutex);
	cctx = nvkm_list_find(cctx, &chan->cctxs, head,
			      cctx->vctx->ectx->engn == engn && cctx->vctx->vmm == chan->vmm);
	if (cctx) {
		refcount_inc(&cctx->refs);
		*pcctx = cctx;
		mutex_unlock(&chan->cgrp->mutex);
		return 0;
	}

	/* Nope - create a fresh one.  But, sub-context first. */
	ret = nvkm_cgrp_vctx_get(cgrp, engn, chan, &vctx, client);
	if (ret) {
		CHAN_ERROR(chan, "vctx %d[%s]: %d", engn->id, engn->engine->subdev.name, ret);
		goto done;
	}

	/* Now, create the channel context - to track engine binding. */
	CHAN_TRACE(chan, "ctor cctx %d[%s]", engn->id, engn->engine->subdev.name);
	if (!(cctx = *pcctx = kzalloc(sizeof(*cctx), GFP_KERNEL))) {
		nvkm_cgrp_vctx_put(cgrp, &vctx);
		ret = -ENOMEM;
		goto done;
	}

	cctx->vctx = vctx;
	refcount_set(&cctx->refs, 1);
	refcount_set(&cctx->uses, 0);
	list_add_tail(&cctx->head, &chan->cctxs);
done:
	mutex_unlock(&cgrp->mutex);
	return ret;
}

int
nvkm_chan_preempt_locked(struct nvkm_chan *chan, bool wait)
{
	struct nvkm_runl *runl = chan->cgrp->runl;

	CHAN_TRACE(chan, "preempt");
	chan->func->preempt(chan);
	if (!wait)
		return 0;

	return nvkm_runl_preempt_wait(runl);
}

int
nvkm_chan_preempt(struct nvkm_chan *chan, bool wait)
{
	int ret;

	if (!chan->func->preempt)
		return 0;

	mutex_lock(&chan->cgrp->runl->mutex);
	ret = nvkm_chan_preempt_locked(chan, wait);
	mutex_unlock(&chan->cgrp->runl->mutex);
	return ret;
}

void
nvkm_chan_remove_locked(struct nvkm_chan *chan)
{
	struct nvkm_cgrp *cgrp = chan->cgrp;
	struct nvkm_runl *runl = cgrp->runl;

	if (list_empty(&chan->head))
		return;

	CHAN_TRACE(chan, "remove");
	if (!--cgrp->chan_nr) {
		runl->cgrp_nr--;
		list_del(&cgrp->head);
	}
	runl->chan_nr--;
	list_del_init(&chan->head);
	atomic_set(&runl->changed, 1);
}

void
nvkm_chan_remove(struct nvkm_chan *chan, bool preempt)
{
	struct nvkm_runl *runl = chan->cgrp->runl;

	mutex_lock(&runl->mutex);
	if (preempt && chan->func->preempt)
		nvkm_chan_preempt_locked(chan, true);
	nvkm_chan_remove_locked(chan);
	nvkm_runl_update_locked(runl, true);
	mutex_unlock(&runl->mutex);
}

void
nvkm_chan_insert(struct nvkm_chan *chan)
{
	struct nvkm_cgrp *cgrp = chan->cgrp;
	struct nvkm_runl *runl = cgrp->runl;

	mutex_lock(&runl->mutex);
	if (WARN_ON(!list_empty(&chan->head))) {
		mutex_unlock(&runl->mutex);
		return;
	}

	CHAN_TRACE(chan, "insert");
	list_add_tail(&chan->head, &cgrp->chans);
	runl->chan_nr++;
	if (!cgrp->chan_nr++) {
		list_add_tail(&cgrp->head, &cgrp->runl->cgrps);
		runl->cgrp_nr++;
	}
	atomic_set(&runl->changed, 1);
	nvkm_runl_update_locked(runl, true);
	mutex_unlock(&runl->mutex);
}

static void
nvkm_chan_block_locked(struct nvkm_chan *chan)
{
	CHAN_TRACE(chan, "block %d", atomic_read(&chan->blocked));
	if (atomic_inc_return(&chan->blocked) == 1)
		chan->func->stop(chan);
}

void
nvkm_chan_error(struct nvkm_chan *chan, bool preempt)
{
	unsigned long flags;

	spin_lock_irqsave(&chan->lock, flags);
	if (atomic_inc_return(&chan->errored) == 1) {
		CHAN_ERROR(chan, "errored - disabling channel");
		nvkm_chan_block_locked(chan);
		if (preempt)
			chan->func->preempt(chan);
		nvkm_event_ntfy(&chan->cgrp->runl->chid->event, chan->id, NVKM_CHAN_EVENT_ERRORED);
	}
	spin_unlock_irqrestore(&chan->lock, flags);
}

void
nvkm_chan_block(struct nvkm_chan *chan)
{
	spin_lock_irq(&chan->lock);
	nvkm_chan_block_locked(chan);
	spin_unlock_irq(&chan->lock);
}

void
nvkm_chan_allow(struct nvkm_chan *chan)
{
	spin_lock_irq(&chan->lock);
	CHAN_TRACE(chan, "allow %d", atomic_read(&chan->blocked));
	if (atomic_dec_and_test(&chan->blocked))
		chan->func->start(chan);
	spin_unlock_irq(&chan->lock);
}

void
nvkm_chan_del(struct nvkm_chan **pchan)
{
	struct nvkm_chan *chan = *pchan;

	if (!chan)
		return;

	if (chan->func->ramfc->clear)
		chan->func->ramfc->clear(chan);

	nvkm_ramht_del(&chan->ramht);
	nvkm_gpuobj_del(&chan->pgd);
	nvkm_gpuobj_del(&chan->eng);
	nvkm_gpuobj_del(&chan->cache);
	nvkm_gpuobj_del(&chan->ramfc);

	if (chan->cgrp) {
		if (!chan->func->id_put)
			nvkm_chid_put(chan->cgrp->runl->chid, chan->id, &chan->cgrp->lock);
		else
			chan->func->id_put(chan);

		nvkm_cgrp_unref(&chan->cgrp);
	}

	nvkm_memory_unref(&chan->userd.mem);

	if (chan->vmm) {
		nvkm_vmm_part(chan->vmm, chan->inst->memory);
		nvkm_vmm_unref(&chan->vmm);
	}

	nvkm_gpuobj_del(&chan->push);
	nvkm_gpuobj_del(&chan->inst);
	kfree(chan);
}

void
nvkm_chan_put(struct nvkm_chan **pchan, unsigned long irqflags)
{
	struct nvkm_chan *chan = *pchan;

	if (!chan)
		return;

	*pchan = NULL;
	spin_unlock_irqrestore(&chan->cgrp->lock, irqflags);
}

struct nvkm_chan *
nvkm_chan_get_inst(struct nvkm_engine *engine, u64 inst, unsigned long *pirqflags)
{
	struct nvkm_fifo *fifo = engine->subdev.device->fifo;
	struct nvkm_runl *runl;
	struct nvkm_engn *engn;
	struct nvkm_chan *chan;

	nvkm_runl_foreach(runl, fifo) {
		nvkm_runl_foreach_engn(engn, runl) {
			if (engine == &fifo->engine || engn->engine == engine) {
				chan = nvkm_runl_chan_get_inst(runl, inst, pirqflags);
				if (chan || engn->engine == engine)
					return chan;
			}
		}
	}

	return NULL;
}

struct nvkm_chan *
nvkm_chan_get_chid(struct nvkm_engine *engine, int id, unsigned long *pirqflags)
{
	struct nvkm_fifo *fifo = engine->subdev.device->fifo;
	struct nvkm_runl *runl;
	struct nvkm_engn *engn;

	nvkm_runl_foreach(runl, fifo) {
		nvkm_runl_foreach_engn(engn, runl) {
			if (fifo->chid || engn->engine == engine)
				return nvkm_runl_chan_get_chid(runl, id, pirqflags);
		}
	}

	return NULL;
}

int
nvkm_chan_new_(const struct nvkm_chan_func *func, struct nvkm_runl *runl, int runq,
	       struct nvkm_cgrp *cgrp, const char *name, bool priv, u32 devm, struct nvkm_vmm *vmm,
	       struct nvkm_dmaobj *dmaobj, u64 offset, u64 length,
	       struct nvkm_memory *userd, u64 ouserd, struct nvkm_chan **pchan)
{
	struct nvkm_fifo *fifo = runl->fifo;
	struct nvkm_device *device = fifo->engine.subdev.device;
	struct nvkm_chan *chan;
	int ret;

	/* Validate arguments against class requirements. */
	if ((runq && runq >= runl->func->runqs) ||
	    (!func->inst->vmm != !vmm) ||
	    ((func->userd->bar < 0) == !userd) ||
	    (!func->ramfc->ctxdma != !dmaobj) ||
	    ((func->ramfc->devm < devm) && devm != BIT(0)) ||
	    (!func->ramfc->priv && priv)) {
		RUNL_DEBUG(runl, "args runq:%d:%d vmm:%d:%p userd:%d:%p "
				 "push:%d:%p devm:%08x:%08x priv:%d:%d",
			   runl->func->runqs, runq, func->inst->vmm, vmm,
			   func->userd->bar < 0, userd, func->ramfc->ctxdma, dmaobj,
			   func->ramfc->devm, devm, func->ramfc->priv, priv);
		return -EINVAL;
	}

	if (!(chan = *pchan = kzalloc(sizeof(*chan), GFP_KERNEL)))
		return -ENOMEM;

	chan->func = func;
	strscpy(chan->name, name, sizeof(chan->name));
	chan->runq = runq;
	chan->id = -1;
	spin_lock_init(&chan->lock);
	atomic_set(&chan->blocked, 1);
	atomic_set(&chan->errored, 0);
	INIT_LIST_HEAD(&chan->cctxs);
	INIT_LIST_HEAD(&chan->head);

	/* Join channel group.
	 *
	 * GK110 and newer support channel groups (aka TSGs), where individual channels
	 * share a timeslice, and, engine context(s).
	 *
	 * As such, engine contexts are tracked in nvkm_cgrp and we need them even when
	 * channels aren't in an API channel group, and on HW that doesn't support TSGs.
	 */
	if (!cgrp) {
		ret = nvkm_cgrp_new(runl, chan->name, vmm, fifo->func->cgrp.force, &chan->cgrp);
		if (ret) {
			RUNL_DEBUG(runl, "cgrp %d", ret);
			return ret;
		}

		cgrp = chan->cgrp;
	} else {
		if (cgrp->runl != runl || cgrp->vmm != vmm) {
			RUNL_DEBUG(runl, "cgrp %d %d", cgrp->runl != runl, cgrp->vmm != vmm);
			return -EINVAL;
		}

		chan->cgrp = nvkm_cgrp_ref(cgrp);
	}

	/* Allocate instance block. */
	ret = nvkm_gpuobj_new(device, func->inst->size, 0x1000, func->inst->zero, NULL,
			      &chan->inst);
	if (ret) {
		RUNL_DEBUG(runl, "inst %d", ret);
		return ret;
	}

	/* Initialise virtual address-space. */
	if (func->inst->vmm) {
		if (WARN_ON(vmm->mmu != device->mmu))
			return -EINVAL;

		ret = nvkm_vmm_join(vmm, chan->inst->memory);
		if (ret) {
			RUNL_DEBUG(runl, "vmm %d", ret);
			return ret;
		}

		chan->vmm = nvkm_vmm_ref(vmm);
	}

	/* Allocate HW ctxdma for push buffer. */
	if (func->ramfc->ctxdma) {
		ret = nvkm_object_bind(&dmaobj->object, chan->inst, -16, &chan->push);
		if (ret) {
			RUNL_DEBUG(runl, "bind %d", ret);
			return ret;
		}
	}

	/* Allocate channel ID. */
	if (!chan->func->id_get) {
		chan->id = nvkm_chid_get(runl->chid, chan);
		if (chan->id >= 0) {
			if (func->userd->bar < 0) {
				if (ouserd + chan->func->userd->size >=
					nvkm_memory_size(userd)) {
					RUNL_DEBUG(runl, "ouserd %llx", ouserd);
					return -EINVAL;
				}

				ret = nvkm_memory_kmap(userd, &chan->userd.mem);
				if (ret) {
					RUNL_DEBUG(runl, "userd %d", ret);
					return ret;
				}

				chan->userd.base = ouserd;
			} else {
				chan->userd.mem = nvkm_memory_ref(fifo->userd.mem);
				chan->userd.base = chan->id * chan->func->userd->size;
			}
		}
	} else {
		chan->id = chan->func->id_get(chan, userd, ouserd);
	}

	if (chan->id < 0) {
		RUNL_ERROR(runl, "!chids");
		return -ENOSPC;
	}

	if (cgrp->id < 0)
		cgrp->id = chan->id;

	/* Initialise USERD. */
	if (chan->func->userd->clear)
		chan->func->userd->clear(chan);

	/* Initialise RAMFC. */
	ret = chan->func->ramfc->write(chan, offset, length, devm, priv);
	if (ret) {
		RUNL_DEBUG(runl, "ramfc %d", ret);
		return ret;
	}

	return 0;
}