1ebb945a9SBen Skeggs /*
2ebb945a9SBen Skeggs * Copyright 2012 Red Hat Inc.
3ebb945a9SBen Skeggs *
4ebb945a9SBen Skeggs * Permission is hereby granted, free of charge, to any person obtaining a
5ebb945a9SBen Skeggs * copy of this software and associated documentation files (the "Software"),
6ebb945a9SBen Skeggs * to deal in the Software without restriction, including without limitation
7ebb945a9SBen Skeggs * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8ebb945a9SBen Skeggs * and/or sell copies of the Software, and to permit persons to whom the
9ebb945a9SBen Skeggs * Software is furnished to do so, subject to the following conditions:
10ebb945a9SBen Skeggs *
11ebb945a9SBen Skeggs * The above copyright notice and this permission notice shall be included in
12ebb945a9SBen Skeggs * all copies or substantial portions of the Software.
13ebb945a9SBen Skeggs *
14ebb945a9SBen Skeggs * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15ebb945a9SBen Skeggs * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16ebb945a9SBen Skeggs * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17ebb945a9SBen Skeggs * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18ebb945a9SBen Skeggs * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19ebb945a9SBen Skeggs * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20ebb945a9SBen Skeggs * OTHER DEALINGS IN THE SOFTWARE.
21ebb945a9SBen Skeggs *
22ebb945a9SBen Skeggs * Authors: Ben Skeggs
23ebb945a9SBen Skeggs */
24cd346a89SBen Skeggs #include <nvif/push006c.h>
25ebb945a9SBen Skeggs
26fdb751efSBen Skeggs #include <nvif/class.h>
27845f2725SBen Skeggs #include <nvif/cl0002.h>
286c9705f6SBen Skeggs #include <nvif/if0020.h>
29fdb751efSBen Skeggs
304dc28134SBen Skeggs #include "nouveau_drv.h"
31ebb945a9SBen Skeggs #include "nouveau_dma.h"
32ebb945a9SBen Skeggs #include "nouveau_bo.h"
33ebb945a9SBen Skeggs #include "nouveau_chan.h"
34ebb945a9SBen Skeggs #include "nouveau_fence.h"
35ebb945a9SBen Skeggs #include "nouveau_abi16.h"
3624e8375bSBen Skeggs #include "nouveau_vmm.h"
37eeaf06acSBen Skeggs #include "nouveau_svm.h"
38ebb945a9SBen Skeggs
39ebb945a9SBen Skeggs MODULE_PARM_DESC(vram_pushbuf, "Create DMA push buffers in VRAM");
40703fa264SPierre Moreau int nouveau_vram_pushbuf;
41ebb945a9SBen Skeggs module_param_named(vram_pushbuf, nouveau_vram_pushbuf, int, 0400);
42ebb945a9SBen Skeggs
437576c4caSDanilo Krummrich void
nouveau_channel_kill(struct nouveau_channel * chan)447576c4caSDanilo Krummrich nouveau_channel_kill(struct nouveau_channel *chan)
457576c4caSDanilo Krummrich {
467576c4caSDanilo Krummrich atomic_set(&chan->killed, 1);
477576c4caSDanilo Krummrich if (chan->fence)
487576c4caSDanilo Krummrich nouveau_fence_context_kill(chan->fence, -ENODEV);
497576c4caSDanilo Krummrich }
507576c4caSDanilo Krummrich
51d8cc37d8SBen Skeggs static int
nouveau_channel_killed(struct nvif_event * event,void * repv,u32 repc)526c9705f6SBen Skeggs nouveau_channel_killed(struct nvif_event *event, void *repv, u32 repc)
53d8cc37d8SBen Skeggs {
546c9705f6SBen Skeggs struct nouveau_channel *chan = container_of(event, typeof(*chan), kill);
55d8cc37d8SBen Skeggs struct nouveau_cli *cli = chan->cli;
566c9705f6SBen Skeggs
57d8cc37d8SBen Skeggs NV_PRINTK(warn, cli, "channel %d killed!\n", chan->chid);
587576c4caSDanilo Krummrich
597576c4caSDanilo Krummrich if (unlikely(!atomic_read(&chan->killed)))
607576c4caSDanilo Krummrich nouveau_channel_kill(chan);
616c9705f6SBen Skeggs
626c9705f6SBen Skeggs return NVIF_EVENT_DROP;
63d8cc37d8SBen Skeggs }
64d8cc37d8SBen Skeggs
65ebb945a9SBen Skeggs int
nouveau_channel_idle(struct nouveau_channel * chan)66ebb945a9SBen Skeggs nouveau_channel_idle(struct nouveau_channel *chan)
67ebb945a9SBen Skeggs {
6884cd0a55SBen Skeggs if (likely(chan && chan->fence && !atomic_read(&chan->killed))) {
69a01ca78cSBen Skeggs struct nouveau_cli *cli = chan->cli;
70ebb945a9SBen Skeggs struct nouveau_fence *fence = NULL;
71ebb945a9SBen Skeggs int ret;
72ebb945a9SBen Skeggs
73978474dcSDanilo Krummrich ret = nouveau_fence_new(&fence, chan);
74ebb945a9SBen Skeggs if (!ret) {
75*27ce65f6SDave Airlie ret = nouveau_fence_wait(fence, false, false);
76ebb945a9SBen Skeggs nouveau_fence_unref(&fence);
77ebb945a9SBen Skeggs }
78ebb945a9SBen Skeggs
79fbd58ebdSBen Skeggs if (ret) {
80fcf3f91cSBen Skeggs NV_PRINTK(err, cli, "failed to idle channel %d [%s]\n",
81fcf3f91cSBen Skeggs chan->chid, cli->name);
82ebb945a9SBen Skeggs return ret;
83ebb945a9SBen Skeggs }
84fbd58ebdSBen Skeggs }
85fbd58ebdSBen Skeggs return 0;
86fbd58ebdSBen Skeggs }
87ebb945a9SBen Skeggs
88ebb945a9SBen Skeggs void
nouveau_channel_del(struct nouveau_channel ** pchan)89ebb945a9SBen Skeggs nouveau_channel_del(struct nouveau_channel **pchan)
90ebb945a9SBen Skeggs {
91ebb945a9SBen Skeggs struct nouveau_channel *chan = *pchan;
92ebb945a9SBen Skeggs if (chan) {
93d7722134SBen Skeggs if (chan->fence)
94d7722134SBen Skeggs nouveau_fence(chan->cli->drm)->context_del(chan);
95fbd58ebdSBen Skeggs
96ebb945a9SBen Skeggs if (nvif_object_constructed(&chan->user))
97eeaf06acSBen Skeggs nouveau_svmm_part(chan->vmm->svmm, chan->inst);
98eeaf06acSBen Skeggs
99eeaf06acSBen Skeggs nvif_object_dtor(&chan->blit);
100eeaf06acSBen Skeggs nvif_object_dtor(&chan->nvsw);
101835a65f5SKarol Herbst nvif_object_dtor(&chan->gart);
1029ac596a4SBen Skeggs nvif_object_dtor(&chan->vram);
1039ac596a4SBen Skeggs nvif_event_dtor(&chan->kill);
1049ac596a4SBen Skeggs nvif_object_dtor(&chan->user);
1056c9705f6SBen Skeggs nvif_mem_dtor(&chan->mem_userd);
1069ac596a4SBen Skeggs nvif_object_dtor(&chan->push.ctxdma);
10706db7fdeSBen Skeggs nouveau_vma_del(&chan->push.vma);
1089ac596a4SBen Skeggs nouveau_bo_unmap(chan->push.buffer);
10924e8375bSBen Skeggs if (chan->push.buffer && chan->push.buffer->bo.pin_count)
110ebb945a9SBen Skeggs nouveau_bo_unpin(chan->push.buffer);
1116797cea1SChristian König nouveau_bo_fini(chan->push.buffer);
112124ea297SMarcin Slusarz kfree(chan);
113ebb945a9SBen Skeggs }
114ebb945a9SBen Skeggs *pchan = NULL;
115ebb945a9SBen Skeggs }
116ebb945a9SBen Skeggs
117ebb945a9SBen Skeggs static void
nouveau_channel_kick(struct nvif_push * push)118ebb945a9SBen Skeggs nouveau_channel_kick(struct nvif_push *push)
119fdb06e2bSBen Skeggs {
120fdb06e2bSBen Skeggs struct nouveau_channel *chan = container_of(push, typeof(*chan), chan.push);
121fdb06e2bSBen Skeggs chan->dma.cur = chan->dma.cur + (chan->chan.push.cur - chan->chan.push.bgn);
122fdb06e2bSBen Skeggs FIRE_RING(chan);
123fdb06e2bSBen Skeggs chan->chan.push.bgn = chan->chan.push.cur;
124fdb06e2bSBen Skeggs }
125fdb06e2bSBen Skeggs
126fdb06e2bSBen Skeggs static int
nouveau_channel_wait(struct nvif_push * push,u32 size)127fdb06e2bSBen Skeggs nouveau_channel_wait(struct nvif_push *push, u32 size)
128fdb06e2bSBen Skeggs {
129fdb06e2bSBen Skeggs struct nouveau_channel *chan = container_of(push, typeof(*chan), chan.push);
130fdb06e2bSBen Skeggs int ret;
131fdb06e2bSBen Skeggs chan->dma.cur = chan->dma.cur + (chan->chan.push.cur - chan->chan.push.bgn);
132fdb06e2bSBen Skeggs ret = RING_SPACE(chan, size);
133fdb06e2bSBen Skeggs if (ret == 0) {
134fdb06e2bSBen Skeggs chan->chan.push.bgn = chan->chan.push.mem.object.map.ptr;
135fdb06e2bSBen Skeggs chan->chan.push.bgn = chan->chan.push.bgn + chan->dma.cur;
136fdb06e2bSBen Skeggs chan->chan.push.cur = chan->chan.push.bgn;
137fdb06e2bSBen Skeggs chan->chan.push.end = chan->chan.push.bgn + size;
138fdb06e2bSBen Skeggs }
139fdb06e2bSBen Skeggs return ret;
140fdb06e2bSBen Skeggs }
141fdb06e2bSBen Skeggs
142fdb06e2bSBen Skeggs static int
nouveau_channel_prep(struct nouveau_cli * cli,u32 size,struct nouveau_channel ** pchan)143fdb06e2bSBen Skeggs nouveau_channel_prep(struct nouveau_cli *cli,
144ebb945a9SBen Skeggs u32 size, struct nouveau_channel **pchan)
1450ad72863SBen Skeggs {
146fcf3f91cSBen Skeggs struct nouveau_drm *drm = cli->drm;
147ebb945a9SBen Skeggs struct nvif_device *device = &cli->device;
148a01ca78cSBen Skeggs struct nv_dma_v0 args = {};
1494acfd707SBen Skeggs struct nouveau_channel *chan;
150ebb945a9SBen Skeggs u32 target;
151ebb945a9SBen Skeggs int ret;
152ebb945a9SBen Skeggs
153ebb945a9SBen Skeggs chan = *pchan = kzalloc(sizeof(*chan), GFP_KERNEL);
154ebb945a9SBen Skeggs if (!chan)
155ebb945a9SBen Skeggs return -ENOMEM;
156ebb945a9SBen Skeggs
157ebb945a9SBen Skeggs chan->cli = cli;
158a01ca78cSBen Skeggs chan->vmm = nouveau_cli_vmm(cli);
159ebb945a9SBen Skeggs atomic_set(&chan->killed, 0);
1607b05a7c0SDanilo Krummrich
161d8cc37d8SBen Skeggs /* allocate memory for dma push buffer */
162ebb945a9SBen Skeggs target = NOUVEAU_GEM_DOMAIN_GART | NOUVEAU_GEM_DOMAIN_COHERENT;
163ebb945a9SBen Skeggs if (nouveau_vram_pushbuf)
16481b61579SChristian König target = NOUVEAU_GEM_DOMAIN_VRAM;
165ebb945a9SBen Skeggs
16681b61579SChristian König ret = nouveau_bo_new(cli, size, 0, target, 0, 0, NULL, NULL,
167ebb945a9SBen Skeggs &chan->push.buffer);
168bab7cc18SBen Skeggs if (ret == 0) {
169ebb945a9SBen Skeggs ret = nouveau_bo_pin(chan->push.buffer, target, false);
170ebb945a9SBen Skeggs if (ret == 0)
171ad76b3f7SBen Skeggs ret = nouveau_bo_map(chan->push.buffer);
172ebb945a9SBen Skeggs }
173ebb945a9SBen Skeggs
174ebb945a9SBen Skeggs if (ret) {
175ebb945a9SBen Skeggs nouveau_channel_del(pchan);
176ebb945a9SBen Skeggs return ret;
177ebb945a9SBen Skeggs }
178ebb945a9SBen Skeggs
179ebb945a9SBen Skeggs chan->chan.push.mem.object.parent = cli->base.object.parent;
180ebb945a9SBen Skeggs chan->chan.push.mem.object.client = &cli->base;
181fdb06e2bSBen Skeggs chan->chan.push.mem.object.name = "chanPush";
182fdb06e2bSBen Skeggs chan->chan.push.mem.object.map.ptr = chan->push.buffer->kmap.virtual;
183fdb06e2bSBen Skeggs chan->chan.push.wait = nouveau_channel_wait;
184fdb06e2bSBen Skeggs chan->chan.push.kick = nouveau_channel_kick;
185fdb06e2bSBen Skeggs
186fdb06e2bSBen Skeggs /* create dma object covering the *entire* memory space that the
187fdb06e2bSBen Skeggs * pushbuf lives in, this is because the GEM code requires that
188fdb06e2bSBen Skeggs * we be able to call out to other (indirect) push buffers
189ebb945a9SBen Skeggs */
190ebb945a9SBen Skeggs chan->push.addr = chan->push.buffer->offset;
191ebb945a9SBen Skeggs
192ebb945a9SBen Skeggs if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) {
1930dc9b286SNirmoy Das ret = nouveau_vma_new(chan->push.buffer, chan->vmm,
194ebb945a9SBen Skeggs &chan->push.vma);
195967e7bdeSBen Skeggs if (ret) {
196bfe91afaSBen Skeggs nouveau_channel_del(pchan);
197ebb945a9SBen Skeggs return ret;
198ebb945a9SBen Skeggs }
199ebb945a9SBen Skeggs
200ebb945a9SBen Skeggs chan->push.addr = chan->push.vma->addr;
201ebb945a9SBen Skeggs
202ebb945a9SBen Skeggs if (device->info.family >= NV_DEVICE_INFO_V0_FERMI)
20392b4eaafSBen Skeggs return 0;
20492b4eaafSBen Skeggs
20592b4eaafSBen Skeggs args.target = NV_DMA_V0_TARGET_VM;
20692b4eaafSBen Skeggs args.access = NV_DMA_V0_ACCESS_VM;
20792b4eaafSBen Skeggs args.start = 0;
2084acfd707SBen Skeggs args.limit = chan->vmm->vmm.limit - 1;
2094acfd707SBen Skeggs } else
210ebb945a9SBen Skeggs if (chan->push.buffer->bo.resource->mem_type == TTM_PL_VRAM) {
211bfe91afaSBen Skeggs if (device->info.family == NV_DEVICE_INFO_V0_TNT) {
212ebb945a9SBen Skeggs /* nv04 vram pushbuf hack, retarget to its location in
213d3116756SChristian König * the framebuffer bar rather than direct vram access..
214967e7bdeSBen Skeggs * nfi why this exists, it came from the -nv ddx.
215ebb945a9SBen Skeggs */
216ebb945a9SBen Skeggs args.target = NV_DMA_V0_TARGET_PCI;
217ebb945a9SBen Skeggs args.access = NV_DMA_V0_ACCESS_RDWR;
218ebb945a9SBen Skeggs args.start = nvxx_device(drm)->func->resource_addr(nvxx_device(drm), 1);
2194acfd707SBen Skeggs args.limit = args.start + device->info.ram_user - 1;
2204acfd707SBen Skeggs } else {
2217e8820feSBen Skeggs args.target = NV_DMA_V0_TARGET_VRAM;
2227e8820feSBen Skeggs args.access = NV_DMA_V0_ACCESS_RDWR;
223f392ec4bSBen Skeggs args.start = 0;
224ebb945a9SBen Skeggs args.limit = device->info.ram_user - 1;
2254acfd707SBen Skeggs }
2264acfd707SBen Skeggs } else {
227ebb945a9SBen Skeggs if (drm->agp.bridge) {
228f392ec4bSBen Skeggs args.target = NV_DMA_V0_TARGET_AGP;
229ebb945a9SBen Skeggs args.access = NV_DMA_V0_ACCESS_RDWR;
230ebb945a9SBen Skeggs args.start = drm->agp.base;
231340b0e7cSBen Skeggs args.limit = drm->agp.base + drm->agp.size - 1;
2324acfd707SBen Skeggs } else {
2334acfd707SBen Skeggs args.target = NV_DMA_V0_TARGET_VM;
234ebb945a9SBen Skeggs args.access = NV_DMA_V0_ACCESS_RDWR;
235ebb945a9SBen Skeggs args.start = 0;
236ebb945a9SBen Skeggs args.limit = chan->vmm->vmm.limit - 1;
237ebb945a9SBen Skeggs }
2384acfd707SBen Skeggs }
2394acfd707SBen Skeggs
240ebb945a9SBen Skeggs ret = nvif_object_ctor(&device->object, "abi16PushCtxDma", 0,
241bfe91afaSBen Skeggs NV_DMA_FROM_MEMORY, &args, sizeof(args),
242ebb945a9SBen Skeggs &chan->push.ctxdma);
243ebb945a9SBen Skeggs if (ret) {
244ebb945a9SBen Skeggs nouveau_channel_del(pchan);
2459ac596a4SBen Skeggs return ret;
2469ac596a4SBen Skeggs }
2479ac596a4SBen Skeggs
248ebb945a9SBen Skeggs return 0;
249ebb945a9SBen Skeggs }
250ebb945a9SBen Skeggs
251ebb945a9SBen Skeggs static int
nouveau_channel_ctor(struct nouveau_cli * cli,bool priv,u64 runm,struct nouveau_channel ** pchan)252ebb945a9SBen Skeggs nouveau_channel_ctor(struct nouveau_cli *cli, bool priv, u64 runm,
253ebb945a9SBen Skeggs struct nouveau_channel **pchan)
254ebb945a9SBen Skeggs {
255ebb945a9SBen Skeggs const struct nvif_mclass hosts[] = {
2565b8a43aeSMarcin Slusarz { AMPERE_CHANNEL_GPFIFO_B, 0 },
25706db7fdeSBen Skeggs { AMPERE_CHANNEL_GPFIFO_A, 0 },
258fcf3f91cSBen Skeggs { TURING_CHANNEL_GPFIFO_A, 0 },
259ebb945a9SBen Skeggs { VOLTA_CHANNEL_GPFIFO_A, 0 },
260152be542SDanilo Krummrich { PASCAL_CHANNEL_GPFIFO_A, 0 },
26106db7fdeSBen Skeggs { MAXWELL_CHANNEL_GPFIFO_A, 0 },
2627f4f35eaSBen Skeggs { KEPLER_CHANNEL_GPFIFO_B, 0 },
26306db7fdeSBen Skeggs { KEPLER_CHANNEL_GPFIFO_A, 0 },
26406db7fdeSBen Skeggs { FERMI_CHANNEL_GPFIFO , 0 },
26506db7fdeSBen Skeggs { G82_CHANNEL_GPFIFO , 0 },
26606db7fdeSBen Skeggs { NV50_CHANNEL_GPFIFO , 0 },
26706db7fdeSBen Skeggs { NV40_CHANNEL_DMA , 0 },
26806db7fdeSBen Skeggs { NV17_CHANNEL_DMA , 0 },
26906db7fdeSBen Skeggs { NV10_CHANNEL_DMA , 0 },
27006db7fdeSBen Skeggs { NV03_CHANNEL_DMA , 0 },
27106db7fdeSBen Skeggs {}
27206db7fdeSBen Skeggs };
27306db7fdeSBen Skeggs struct {
27406db7fdeSBen Skeggs struct nvif_chan_v0 chan;
27506db7fdeSBen Skeggs char name[TASK_COMM_LEN+16];
27606db7fdeSBen Skeggs } args;
27706db7fdeSBen Skeggs struct nvif_device *device = &cli->device;
27806db7fdeSBen Skeggs struct nouveau_channel *chan;
27906db7fdeSBen Skeggs const u64 plength = 0x10000;
28006db7fdeSBen Skeggs const u64 ioffset = plength;
28106db7fdeSBen Skeggs const u64 ilength = 0x02000;
28206db7fdeSBen Skeggs char name[TASK_COMM_LEN];
283ebb945a9SBen Skeggs int cid, ret;
28406db7fdeSBen Skeggs u64 size;
28506db7fdeSBen Skeggs
28606db7fdeSBen Skeggs cid = nvif_mclass(&device->object, hosts);
28706db7fdeSBen Skeggs if (cid < 0)
28806db7fdeSBen Skeggs return cid;
28906db7fdeSBen Skeggs
29006db7fdeSBen Skeggs if (hosts[cid].oclass < NV50_CHANNEL_GPFIFO)
29106db7fdeSBen Skeggs size = plength;
29206db7fdeSBen Skeggs else
29306db7fdeSBen Skeggs size = ioffset + ilength;
29406db7fdeSBen Skeggs
29506db7fdeSBen Skeggs /* allocate dma push buffer */
29606db7fdeSBen Skeggs ret = nouveau_channel_prep(cli, size, &chan);
29706db7fdeSBen Skeggs *pchan = chan;
29806db7fdeSBen Skeggs if (ret)
299ebb945a9SBen Skeggs return ret;
300ebb945a9SBen Skeggs
30106db7fdeSBen Skeggs /* create channel object */
302ebb945a9SBen Skeggs args.chan.version = 0;
303ebb945a9SBen Skeggs args.chan.namelen = sizeof(args.name);
304ebb945a9SBen Skeggs args.chan.runlist = __ffs64(runm);
305ebb945a9SBen Skeggs args.chan.runq = 0;
306ebb945a9SBen Skeggs args.chan.priv = priv;
30706db7fdeSBen Skeggs args.chan.devm = BIT(0);
30806db7fdeSBen Skeggs if (hosts[cid].oclass < NV50_CHANNEL_GPFIFO) {
30906db7fdeSBen Skeggs args.chan.vmm = 0;
31006db7fdeSBen Skeggs args.chan.ctxdma = nvif_handle(&chan->push.ctxdma);
31106db7fdeSBen Skeggs args.chan.offset = chan->push.addr;
31206db7fdeSBen Skeggs args.chan.length = 0;
31306db7fdeSBen Skeggs } else {
31406db7fdeSBen Skeggs args.chan.vmm = nvif_handle(&chan->vmm->vmm.object);
31506db7fdeSBen Skeggs if (hosts[cid].oclass < FERMI_CHANNEL_GPFIFO)
31606db7fdeSBen Skeggs args.chan.ctxdma = nvif_handle(&chan->push.ctxdma);
31706db7fdeSBen Skeggs else
31806db7fdeSBen Skeggs args.chan.ctxdma = 0;
31906db7fdeSBen Skeggs args.chan.offset = ioffset + chan->push.addr;
32006db7fdeSBen Skeggs args.chan.length = ilength;
32106db7fdeSBen Skeggs }
32206db7fdeSBen Skeggs args.chan.huserd = 0;
32306db7fdeSBen Skeggs args.chan.ouserd = 0;
32406db7fdeSBen Skeggs
32506db7fdeSBen Skeggs /* allocate userd */
326bbf8906bSBen Skeggs if (hosts[cid].oclass >= VOLTA_CHANNEL_GPFIFO_A) {
32706db7fdeSBen Skeggs ret = nvif_mem_ctor(&cli->mmu, "abi16ChanUSERD", NVIF_CLASS_MEM_GF100,
32806db7fdeSBen Skeggs NVIF_MEM_VRAM | NVIF_MEM_COHERENT | NVIF_MEM_MAPPABLE,
329ebb945a9SBen Skeggs 0, PAGE_SIZE, NULL, 0, &chan->mem_userd);
33006db7fdeSBen Skeggs if (ret)
33106db7fdeSBen Skeggs return ret;
33206db7fdeSBen Skeggs
33306db7fdeSBen Skeggs args.chan.huserd = nvif_handle(&chan->mem_userd.object);
33406db7fdeSBen Skeggs args.chan.ouserd = 0;
33506db7fdeSBen Skeggs
33606db7fdeSBen Skeggs chan->userd = &chan->mem_userd.object;
33706db7fdeSBen Skeggs } else {
33806db7fdeSBen Skeggs chan->userd = &chan->user;
33906db7fdeSBen Skeggs }
34006db7fdeSBen Skeggs
34106db7fdeSBen Skeggs get_task_comm(name, current);
34206db7fdeSBen Skeggs snprintf(args.name, sizeof(args.name), "%s[%d]", name, task_pid_nr(current));
34306db7fdeSBen Skeggs
34406db7fdeSBen Skeggs ret = nvif_object_ctor(&device->object, "abi16ChanUser", 0, hosts[cid].oclass,
34506db7fdeSBen Skeggs &args, sizeof(args), &chan->user);
34606db7fdeSBen Skeggs if (ret) {
34706db7fdeSBen Skeggs nouveau_channel_del(pchan);
34806db7fdeSBen Skeggs return ret;
34906db7fdeSBen Skeggs }
35006db7fdeSBen Skeggs
35106db7fdeSBen Skeggs chan->runlist = args.chan.runlist;
352ebb945a9SBen Skeggs chan->chid = args.chan.chid;
353ebb945a9SBen Skeggs chan->inst = args.chan.inst;
354ebb945a9SBen Skeggs chan->token = args.chan.token;
355ebb945a9SBen Skeggs return 0;
35606db7fdeSBen Skeggs }
35706db7fdeSBen Skeggs
35806db7fdeSBen Skeggs static int
nouveau_channel_init(struct nouveau_channel * chan,u32 vram,u32 gart)35906db7fdeSBen Skeggs nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart)
36006db7fdeSBen Skeggs {
36106db7fdeSBen Skeggs struct nouveau_cli *cli = chan->cli;
36206db7fdeSBen Skeggs struct nouveau_drm *drm = cli->drm;
363ebb945a9SBen Skeggs struct nvif_device *device = &cli->device;
364ebb945a9SBen Skeggs struct nv_dma_v0 args = {};
365ebb945a9SBen Skeggs int ret, i;
3660ad72863SBen Skeggs
367d8cc37d8SBen Skeggs ret = nvif_object_map(chan->userd, NULL, 0);
3684acfd707SBen Skeggs if (ret)
369ebb945a9SBen Skeggs return ret;
370ebb945a9SBen Skeggs
37106db7fdeSBen Skeggs if (chan->user.oclass >= FERMI_CHANNEL_GPFIFO) {
372097d56cdSBen Skeggs struct {
373097d56cdSBen Skeggs struct nvif_event_v0 base;
3746c6ae061SBen Skeggs struct nvif_chan_event_v0 host;
3757f4f35eaSBen Skeggs } args;
3766c9705f6SBen Skeggs
3776c9705f6SBen Skeggs args.host.version = 0;
3786c9705f6SBen Skeggs args.host.type = NVIF_CHAN_EVENT_V0_KILLED;
3796c9705f6SBen Skeggs
3806c9705f6SBen Skeggs ret = nvif_event_ctor(&chan->user, "abi16ChanKilled", chan->chid,
3816c9705f6SBen Skeggs nouveau_channel_killed, false,
3826c9705f6SBen Skeggs &args.base, sizeof(args), &chan->kill);
3836c9705f6SBen Skeggs if (ret == 0)
3846c9705f6SBen Skeggs ret = nvif_event_allow(&chan->kill);
3856c9705f6SBen Skeggs if (ret) {
3866c9705f6SBen Skeggs NV_ERROR(drm, "Failed to request channel kill "
387d8cc37d8SBen Skeggs "notification: %d\n", ret);
3886c9705f6SBen Skeggs return ret;
389d8cc37d8SBen Skeggs }
390d8cc37d8SBen Skeggs }
391d8cc37d8SBen Skeggs
392d8cc37d8SBen Skeggs /* allocate dma objects to cover all allowed vram, and gart */
393d8cc37d8SBen Skeggs if (device->info.family < NV_DEVICE_INFO_V0_FERMI) {
394d8cc37d8SBen Skeggs if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) {
395d8cc37d8SBen Skeggs args.target = NV_DMA_V0_TARGET_VM;
396ebb945a9SBen Skeggs args.access = NV_DMA_V0_ACCESS_VM;
397967e7bdeSBen Skeggs args.start = 0;
398967e7bdeSBen Skeggs args.limit = chan->vmm->vmm.limit - 1;
3994acfd707SBen Skeggs } else {
4004acfd707SBen Skeggs args.target = NV_DMA_V0_TARGET_VRAM;
401ebb945a9SBen Skeggs args.access = NV_DMA_V0_ACCESS_RDWR;
402bfe91afaSBen Skeggs args.start = 0;
403ebb945a9SBen Skeggs args.limit = device->info.ram_user - 1;
4044acfd707SBen Skeggs }
4054acfd707SBen Skeggs
406ebb945a9SBen Skeggs ret = nvif_object_ctor(&chan->user, "abi16ChanVramCtxDma", vram,
407f392ec4bSBen Skeggs NV_DMA_IN_MEMORY, &args, sizeof(args),
408ebb945a9SBen Skeggs &chan->vram);
409ebb945a9SBen Skeggs if (ret)
4109ac596a4SBen Skeggs return ret;
4119ac596a4SBen Skeggs
4129ac596a4SBen Skeggs if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) {
413ebb945a9SBen Skeggs args.target = NV_DMA_V0_TARGET_VM;
414ebb945a9SBen Skeggs args.access = NV_DMA_V0_ACCESS_VM;
415ebb945a9SBen Skeggs args.start = 0;
416967e7bdeSBen Skeggs args.limit = chan->vmm->vmm.limit - 1;
4174acfd707SBen Skeggs } else
4184acfd707SBen Skeggs if (drm->agp.bridge) {
419ebb945a9SBen Skeggs args.target = NV_DMA_V0_TARGET_AGP;
420bfe91afaSBen Skeggs args.access = NV_DMA_V0_ACCESS_RDWR;
421ebb945a9SBen Skeggs args.start = drm->agp.base;
422340b0e7cSBen Skeggs args.limit = drm->agp.base + drm->agp.size - 1;
4234acfd707SBen Skeggs } else {
4244acfd707SBen Skeggs args.target = NV_DMA_V0_TARGET_VM;
425ebb945a9SBen Skeggs args.access = NV_DMA_V0_ACCESS_RDWR;
426ebb945a9SBen Skeggs args.start = 0;
427ebb945a9SBen Skeggs args.limit = chan->vmm->vmm.limit - 1;
428ebb945a9SBen Skeggs }
4294acfd707SBen Skeggs
4304acfd707SBen Skeggs ret = nvif_object_ctor(&chan->user, "abi16ChanGartCtxDma", gart,
431ebb945a9SBen Skeggs NV_DMA_IN_MEMORY, &args, sizeof(args),
432bfe91afaSBen Skeggs &chan->gart);
433ebb945a9SBen Skeggs if (ret)
434ebb945a9SBen Skeggs return ret;
4359ac596a4SBen Skeggs }
4369ac596a4SBen Skeggs
4379ac596a4SBen Skeggs /* initialise dma tracking parameters */
438ebb945a9SBen Skeggs switch (chan->user.oclass) {
439ebb945a9SBen Skeggs case NV03_CHANNEL_DMA:
440ebb945a9SBen Skeggs case NV10_CHANNEL_DMA:
441ebb945a9SBen Skeggs case NV17_CHANNEL_DMA:
442ebb945a9SBen Skeggs case NV40_CHANNEL_DMA:
443bbe08a0eSDanilo Krummrich chan->user_put = 0x40;
444bbe08a0eSDanilo Krummrich chan->user_get = 0x44;
445bbe08a0eSDanilo Krummrich chan->dma.max = (0x10000 / 4) - 2;
446bbe08a0eSDanilo Krummrich break;
447bbe08a0eSDanilo Krummrich default:
448ebb945a9SBen Skeggs chan->user_put = 0x40;
449ebb945a9SBen Skeggs chan->user_get = 0x44;
450ebb945a9SBen Skeggs chan->user_get_hi = 0x60;
451ebb945a9SBen Skeggs chan->dma.ib_base = 0x10000 / 4;
452ebb945a9SBen Skeggs chan->dma.ib_max = NV50_DMA_IB_MAX;
453ebb945a9SBen Skeggs chan->dma.ib_put = 0;
454ebb945a9SBen Skeggs chan->dma.ib_free = chan->dma.ib_max - chan->dma.ib_put;
455ebb945a9SBen Skeggs chan->dma.max = chan->dma.ib_base;
456ebb945a9SBen Skeggs break;
457d59e75eeSDanilo Krummrich }
458ebb945a9SBen Skeggs
459ebb945a9SBen Skeggs chan->dma.put = 0;
460ebb945a9SBen Skeggs chan->dma.cur = chan->dma.put;
461ebb945a9SBen Skeggs chan->dma.free = chan->dma.max - chan->dma.cur;
462ebb945a9SBen Skeggs
463ebb945a9SBen Skeggs ret = PUSH_WAIT(&chan->chan.push, NOUVEAU_DMA_SKIPS);
464ebb945a9SBen Skeggs if (ret)
465ebb945a9SBen Skeggs return ret;
466ebb945a9SBen Skeggs
467ebb945a9SBen Skeggs for (i = 0; i < NOUVEAU_DMA_SKIPS; i++)
468cd346a89SBen Skeggs PUSH_DATA(&chan->chan.push, 0x00000000);
469ebb945a9SBen Skeggs
470ebb945a9SBen Skeggs /* allocate software object class (used for fences on <= nv05) */
471ebb945a9SBen Skeggs if (device->info.family < NV_DEVICE_INFO_V0_CELSIUS) {
472ebb945a9SBen Skeggs ret = nvif_object_ctor(&chan->user, "abi16NvswFence", 0x006e,
473cd346a89SBen Skeggs NVIF_CLASS_SW_NV04,
474ebb945a9SBen Skeggs NULL, 0, &chan->nvsw);
47569a6146dSBen Skeggs if (ret)
476967e7bdeSBen Skeggs return ret;
4779ac596a4SBen Skeggs
47808f7633cSBen Skeggs ret = PUSH_WAIT(&chan->chan.push, 2);
4790ad72863SBen Skeggs if (ret)
480ebb945a9SBen Skeggs return ret;
481ebb945a9SBen Skeggs
482ebb945a9SBen Skeggs PUSH_NVSQ(&chan->chan.push, NV_SW, 0x0000, chan->nvsw.handle);
483cd346a89SBen Skeggs PUSH_KICK(&chan->chan.push);
484ebb945a9SBen Skeggs }
485ebb945a9SBen Skeggs
486ebb945a9SBen Skeggs /* initialise synchronisation */
487cd346a89SBen Skeggs return nouveau_fence(drm)->context_new(chan);
488cd346a89SBen Skeggs }
489ebb945a9SBen Skeggs
490ebb945a9SBen Skeggs int
nouveau_channel_new(struct nouveau_cli * cli,bool priv,u64 runm,u32 vram,u32 gart,struct nouveau_channel ** pchan)491ebb945a9SBen Skeggs nouveau_channel_new(struct nouveau_cli *cli,
4924894f662SBen Skeggs bool priv, u64 runm, u32 vram, u32 gart, struct nouveau_channel **pchan)
493ebb945a9SBen Skeggs {
494ebb945a9SBen Skeggs int ret;
495ebb945a9SBen Skeggs
4960ad72863SBen Skeggs ret = nouveau_channel_ctor(cli, priv, runm, pchan);
4976de12538SBen Skeggs if (ret) {
498ebb945a9SBen Skeggs NV_PRINTK(dbg, cli, "channel create, %d\n", ret);
499a01ca78cSBen Skeggs return ret;
500ebb945a9SBen Skeggs }
501ebb945a9SBen Skeggs
50206db7fdeSBen Skeggs ret = nouveau_channel_init(*pchan, vram, gart);
503ebb945a9SBen Skeggs if (ret) {
50406db7fdeSBen Skeggs NV_PRINTK(err, cli, "channel failed to initialise, %d\n", ret);
50559f216cfSBen Skeggs nouveau_channel_del(pchan);
506ebb945a9SBen Skeggs return ret;
507ebb945a9SBen Skeggs }
5086de12538SBen Skeggs
509ebb945a9SBen Skeggs ret = nouveau_svmm_join((*pchan)->vmm->svmm, (*pchan)->inst);
5109ad97edeSBen Skeggs if (ret)
511ebb945a9SBen Skeggs nouveau_channel_del(pchan);
51259f216cfSBen Skeggs
513ebb945a9SBen Skeggs return ret;
514ebb945a9SBen Skeggs }
515eeaf06acSBen Skeggs
516eeaf06acSBen Skeggs void
nouveau_channels_fini(struct nouveau_drm * drm)517eeaf06acSBen Skeggs nouveau_channels_fini(struct nouveau_drm *drm)
518eeaf06acSBen Skeggs {
51967e26e41SBen Skeggs kfree(drm->runl);
520ebb945a9SBen Skeggs }
521eb47db4fSBen Skeggs
522eb39c613SBen Skeggs int
nouveau_channels_init(struct nouveau_drm * drm)523eb39c613SBen Skeggs nouveau_channels_init(struct nouveau_drm *drm)
524eb39c613SBen Skeggs {
525eb39c613SBen Skeggs struct {
526eb39c613SBen Skeggs struct nv_device_info_v1 m;
527eb39c613SBen Skeggs struct {
528eb47db4fSBen Skeggs struct nv_device_info_v1_data channels;
529eb47db4fSBen Skeggs struct nv_device_info_v1_data runlists;
530eb47db4fSBen Skeggs } v;
531eb47db4fSBen Skeggs } args = {
532eb47db4fSBen Skeggs .m.version = 1,
533eb47db4fSBen Skeggs .m.count = sizeof(args.v) / sizeof(args.v.channels),
534eb47db4fSBen Skeggs .v.channels.mthd = NV_DEVICE_HOST_CHANNELS,
535eb39c613SBen Skeggs .v.runlists.mthd = NV_DEVICE_HOST_RUNLISTS,
536eb47db4fSBen Skeggs };
537eb47db4fSBen Skeggs struct nvif_object *device = &drm->client.device.object;
538eb47db4fSBen Skeggs int ret, i;
539eb47db4fSBen Skeggs
540f8fabd31SBen Skeggs ret = nvif_object_mthd(device, NV_DEVICE_V0_INFO, &args, sizeof(args));
541eb39c613SBen Skeggs if (ret ||
542eb47db4fSBen Skeggs args.v.runlists.mthd == NV_DEVICE_INFO_INVALID || !args.v.runlists.data ||
543eb47db4fSBen Skeggs args.v.channels.mthd == NV_DEVICE_INFO_INVALID)
544eb39c613SBen Skeggs return -ENODEV;
545eb39c613SBen Skeggs
546eb39c613SBen Skeggs drm->chan_nr = drm->chan_total = args.v.channels.data;
547eb39c613SBen Skeggs drm->runl_nr = fls64(args.v.runlists.data);
548eb39c613SBen Skeggs drm->runl = kcalloc(drm->runl_nr, sizeof(*drm->runl), GFP_KERNEL);
549eb39c613SBen Skeggs if (!drm->runl)
550eb39c613SBen Skeggs return -ENOMEM;
551eb39c613SBen Skeggs
552eb39c613SBen Skeggs if (drm->chan_nr == 0) {
553eb39c613SBen Skeggs for (i = 0; i < drm->runl_nr; i++) {
554eb39c613SBen Skeggs if (!(args.v.runlists.data & BIT(i)))
555eb39c613SBen Skeggs continue;
556eb39c613SBen Skeggs
557eb39c613SBen Skeggs args.v.channels.mthd = NV_DEVICE_HOST_RUNLIST_CHANNELS;
558eb39c613SBen Skeggs args.v.channels.data = i;
559eb39c613SBen Skeggs
560eb39c613SBen Skeggs ret = nvif_object_mthd(device, NV_DEVICE_V0_INFO, &args, sizeof(args));
561eb39c613SBen Skeggs if (ret || args.v.channels.mthd == NV_DEVICE_INFO_INVALID)
562eb39c613SBen Skeggs return -ENODEV;
563eb39c613SBen Skeggs
564eb39c613SBen Skeggs drm->runl[i].chan_nr = args.v.channels.data;
565eb47db4fSBen Skeggs drm->runl[i].chan_id_base = drm->chan_total;
566eb47db4fSBen Skeggs drm->runl[i].context_base = dma_fence_context_alloc(drm->runl[i].chan_nr);
567eb47db4fSBen Skeggs
568eb47db4fSBen Skeggs drm->chan_total += drm->runl[i].chan_nr;
569eb47db4fSBen Skeggs }
570eb39c613SBen Skeggs } else {
571eb39c613SBen Skeggs drm->runl[0].context_base = dma_fence_context_alloc(drm->chan_nr);
572eb39c613SBen Skeggs for (i = 1; i < drm->runl_nr; i++)
573eb39c613SBen Skeggs drm->runl[i].context_base = drm->runl[0].context_base;
574eb39c613SBen Skeggs
575eb39c613SBen Skeggs }
576eb39c613SBen Skeggs
577eb39c613SBen Skeggs return 0;
578eb39c613SBen Skeggs }
579eb39c613SBen Skeggs