1ebb945a9SBen Skeggs /* 2ebb945a9SBen Skeggs * Copyright 2012 Red Hat Inc. 3ebb945a9SBen Skeggs * 4ebb945a9SBen Skeggs * Permission is hereby granted, free of charge, to any person obtaining a 5ebb945a9SBen Skeggs * copy of this software and associated documentation files (the "Software"), 6ebb945a9SBen Skeggs * to deal in the Software without restriction, including without limitation 7ebb945a9SBen Skeggs * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8ebb945a9SBen Skeggs * and/or sell copies of the Software, and to permit persons to whom the 9ebb945a9SBen Skeggs * Software is furnished to do so, subject to the following conditions: 10ebb945a9SBen Skeggs * 11ebb945a9SBen Skeggs * The above copyright notice and this permission notice shall be included in 12ebb945a9SBen Skeggs * all copies or substantial portions of the Software. 13ebb945a9SBen Skeggs * 14ebb945a9SBen Skeggs * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15ebb945a9SBen Skeggs * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16ebb945a9SBen Skeggs * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17ebb945a9SBen Skeggs * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18ebb945a9SBen Skeggs * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19ebb945a9SBen Skeggs * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20ebb945a9SBen Skeggs * OTHER DEALINGS IN THE SOFTWARE. 21ebb945a9SBen Skeggs * 22ebb945a9SBen Skeggs * Authors: Ben Skeggs 23ebb945a9SBen Skeggs */ 24cd346a89SBen Skeggs #include <nvif/push006c.h> 25ebb945a9SBen Skeggs 26fdb751efSBen Skeggs #include <nvif/class.h> 27845f2725SBen Skeggs #include <nvif/cl0002.h> 288ed1730cSBen Skeggs #include <nvif/cl006b.h> 298ed1730cSBen Skeggs #include <nvif/cl506f.h> 308ed1730cSBen Skeggs #include <nvif/cl906f.h> 318ed1730cSBen Skeggs #include <nvif/cla06f.h> 329d24907cSBen Skeggs #include <nvif/clc36f.h> 33f58ddf95SBen Skeggs #include <nvif/ioctl.h> 34fdb751efSBen Skeggs 354dc28134SBen Skeggs #include "nouveau_drv.h" 36ebb945a9SBen Skeggs #include "nouveau_dma.h" 37ebb945a9SBen Skeggs #include "nouveau_bo.h" 38ebb945a9SBen Skeggs #include "nouveau_chan.h" 39ebb945a9SBen Skeggs #include "nouveau_fence.h" 40ebb945a9SBen Skeggs #include "nouveau_abi16.h" 4124e8375bSBen Skeggs #include "nouveau_vmm.h" 42eeaf06acSBen Skeggs #include "nouveau_svm.h" 43ebb945a9SBen Skeggs 44ebb945a9SBen Skeggs MODULE_PARM_DESC(vram_pushbuf, "Create DMA push buffers in VRAM"); 45703fa264SPierre Moreau int nouveau_vram_pushbuf; 46ebb945a9SBen Skeggs module_param_named(vram_pushbuf, nouveau_vram_pushbuf, int, 0400); 47ebb945a9SBen Skeggs 48d8cc37d8SBen Skeggs static int 49d8cc37d8SBen Skeggs nouveau_channel_killed(struct nvif_notify *ntfy) 50d8cc37d8SBen Skeggs { 51d8cc37d8SBen Skeggs struct nouveau_channel *chan = container_of(ntfy, typeof(*chan), kill); 52d8cc37d8SBen Skeggs struct nouveau_cli *cli = (void *)chan->user.client; 53d8cc37d8SBen Skeggs NV_PRINTK(warn, cli, "channel %d killed!\n", chan->chid); 54d8cc37d8SBen Skeggs atomic_set(&chan->killed, 1); 55ea13e5abSBen Skeggs if (chan->fence) 56ea13e5abSBen Skeggs nouveau_fence_context_kill(chan->fence, -ENODEV); 57d8cc37d8SBen Skeggs return NVIF_NOTIFY_DROP; 58d8cc37d8SBen Skeggs } 59d8cc37d8SBen Skeggs 60ebb945a9SBen Skeggs int 61ebb945a9SBen Skeggs nouveau_channel_idle(struct nouveau_channel *chan) 62ebb945a9SBen Skeggs { 6384cd0a55SBen Skeggs if (likely(chan && chan->fence && !atomic_read(&chan->killed))) { 64a01ca78cSBen Skeggs struct nouveau_cli *cli = (void *)chan->user.client; 65ebb945a9SBen Skeggs struct nouveau_fence *fence = NULL; 66ebb945a9SBen Skeggs int ret; 67ebb945a9SBen Skeggs 68264ce192SBen Skeggs ret = nouveau_fence_new(chan, false, &fence); 69ebb945a9SBen Skeggs if (!ret) { 70ebb945a9SBen Skeggs ret = nouveau_fence_wait(fence, false, false); 71ebb945a9SBen Skeggs nouveau_fence_unref(&fence); 72ebb945a9SBen Skeggs } 73ebb945a9SBen Skeggs 74fbd58ebdSBen Skeggs if (ret) { 75fcf3f91cSBen Skeggs NV_PRINTK(err, cli, "failed to idle channel %d [%s]\n", 76fcf3f91cSBen Skeggs chan->chid, nvxx_client(&cli->base)->name); 77ebb945a9SBen Skeggs return ret; 78ebb945a9SBen Skeggs } 79fbd58ebdSBen Skeggs } 80fbd58ebdSBen Skeggs return 0; 81fbd58ebdSBen Skeggs } 82ebb945a9SBen Skeggs 83ebb945a9SBen Skeggs void 84ebb945a9SBen Skeggs nouveau_channel_del(struct nouveau_channel **pchan) 85ebb945a9SBen Skeggs { 86ebb945a9SBen Skeggs struct nouveau_channel *chan = *pchan; 87ebb945a9SBen Skeggs if (chan) { 88d7722134SBen Skeggs struct nouveau_cli *cli = (void *)chan->user.client; 89d7722134SBen Skeggs 90fbd58ebdSBen Skeggs if (chan->fence) 91ebb945a9SBen Skeggs nouveau_fence(chan->drm)->context_del(chan); 92eeaf06acSBen Skeggs 93eeaf06acSBen Skeggs if (cli) 94eeaf06acSBen Skeggs nouveau_svmm_part(chan->vmm->svmm, chan->inst); 95eeaf06acSBen Skeggs 969ac596a4SBen Skeggs nvif_object_dtor(&chan->nvsw); 979ac596a4SBen Skeggs nvif_object_dtor(&chan->gart); 989ac596a4SBen Skeggs nvif_object_dtor(&chan->vram); 99f7a7d22aSBen Skeggs nvif_notify_dtor(&chan->kill); 1009ac596a4SBen Skeggs nvif_object_dtor(&chan->user); 1019ac596a4SBen Skeggs nvif_object_dtor(&chan->push.ctxdma); 10224e8375bSBen Skeggs nouveau_vma_del(&chan->push.vma); 103ebb945a9SBen Skeggs nouveau_bo_unmap(chan->push.buffer); 1046797cea1SChristian König if (chan->push.buffer && chan->push.buffer->bo.pin_count) 105124ea297SMarcin Slusarz nouveau_bo_unpin(chan->push.buffer); 106ebb945a9SBen Skeggs nouveau_bo_ref(NULL, &chan->push.buffer); 107ebb945a9SBen Skeggs kfree(chan); 108ebb945a9SBen Skeggs } 109ebb945a9SBen Skeggs *pchan = NULL; 110ebb945a9SBen Skeggs } 111ebb945a9SBen Skeggs 112fdb06e2bSBen Skeggs static void 113fdb06e2bSBen Skeggs nouveau_channel_kick(struct nvif_push *push) 114fdb06e2bSBen Skeggs { 115fdb06e2bSBen Skeggs struct nouveau_channel *chan = container_of(push, typeof(*chan), chan._push); 116fdb06e2bSBen Skeggs chan->dma.cur = chan->dma.cur + (chan->chan._push.cur - chan->chan._push.bgn); 117fdb06e2bSBen Skeggs FIRE_RING(chan); 118fdb06e2bSBen Skeggs chan->chan._push.bgn = chan->chan._push.cur; 119fdb06e2bSBen Skeggs } 120fdb06e2bSBen Skeggs 121fdb06e2bSBen Skeggs static int 122fdb06e2bSBen Skeggs nouveau_channel_wait(struct nvif_push *push, u32 size) 123fdb06e2bSBen Skeggs { 124fdb06e2bSBen Skeggs struct nouveau_channel *chan = container_of(push, typeof(*chan), chan._push); 125fdb06e2bSBen Skeggs int ret; 126fdb06e2bSBen Skeggs chan->dma.cur = chan->dma.cur + (chan->chan._push.cur - chan->chan._push.bgn); 127fdb06e2bSBen Skeggs ret = RING_SPACE(chan, size); 128fdb06e2bSBen Skeggs if (ret == 0) { 129fdb06e2bSBen Skeggs chan->chan._push.bgn = chan->chan._push.mem.object.map.ptr; 130fdb06e2bSBen Skeggs chan->chan._push.bgn = chan->chan._push.bgn + chan->dma.cur; 131fdb06e2bSBen Skeggs chan->chan._push.cur = chan->chan._push.bgn; 132fdb06e2bSBen Skeggs chan->chan._push.end = chan->chan._push.bgn + size; 133fdb06e2bSBen Skeggs } 134fdb06e2bSBen Skeggs return ret; 135fdb06e2bSBen Skeggs } 136fdb06e2bSBen Skeggs 137ebb945a9SBen Skeggs static int 1380ad72863SBen Skeggs nouveau_channel_prep(struct nouveau_drm *drm, struct nvif_device *device, 139fcf3f91cSBen Skeggs u32 size, struct nouveau_channel **pchan) 140ebb945a9SBen Skeggs { 141a01ca78cSBen Skeggs struct nouveau_cli *cli = (void *)device->object.client; 1424acfd707SBen Skeggs struct nv_dma_v0 args = {}; 143ebb945a9SBen Skeggs struct nouveau_channel *chan; 144ebb945a9SBen Skeggs u32 target; 145ebb945a9SBen Skeggs int ret; 146ebb945a9SBen Skeggs 147ebb945a9SBen Skeggs chan = *pchan = kzalloc(sizeof(*chan), GFP_KERNEL); 148ebb945a9SBen Skeggs if (!chan) 149ebb945a9SBen Skeggs return -ENOMEM; 150ebb945a9SBen Skeggs 151a01ca78cSBen Skeggs chan->device = device; 152ebb945a9SBen Skeggs chan->drm = drm; 153bfe91afaSBen Skeggs chan->vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 154d8cc37d8SBen Skeggs atomic_set(&chan->killed, 0); 155ebb945a9SBen Skeggs 156ebb945a9SBen Skeggs /* allocate memory for dma push buffer */ 15781b61579SChristian König target = NOUVEAU_GEM_DOMAIN_GART | NOUVEAU_GEM_DOMAIN_COHERENT; 158ebb945a9SBen Skeggs if (nouveau_vram_pushbuf) 15981b61579SChristian König target = NOUVEAU_GEM_DOMAIN_VRAM; 160ebb945a9SBen Skeggs 161bab7cc18SBen Skeggs ret = nouveau_bo_new(cli, size, 0, target, 0, 0, NULL, NULL, 162ebb945a9SBen Skeggs &chan->push.buffer); 163ebb945a9SBen Skeggs if (ret == 0) { 164ad76b3f7SBen Skeggs ret = nouveau_bo_pin(chan->push.buffer, target, false); 165ebb945a9SBen Skeggs if (ret == 0) 166ebb945a9SBen Skeggs ret = nouveau_bo_map(chan->push.buffer); 167ebb945a9SBen Skeggs } 168ebb945a9SBen Skeggs 169ebb945a9SBen Skeggs if (ret) { 170ebb945a9SBen Skeggs nouveau_channel_del(pchan); 171ebb945a9SBen Skeggs return ret; 172ebb945a9SBen Skeggs } 173ebb945a9SBen Skeggs 174fdb06e2bSBen Skeggs chan->chan._push.mem.object.parent = cli->base.object.parent; 175fdb06e2bSBen Skeggs chan->chan._push.mem.object.client = &cli->base; 176fdb06e2bSBen Skeggs chan->chan._push.mem.object.name = "chanPush"; 177fdb06e2bSBen Skeggs chan->chan._push.mem.object.map.ptr = chan->push.buffer->kmap.virtual; 178fdb06e2bSBen Skeggs chan->chan._push.wait = nouveau_channel_wait; 179fdb06e2bSBen Skeggs chan->chan._push.kick = nouveau_channel_kick; 180fdb06e2bSBen Skeggs chan->chan.push = &chan->chan._push; 181fdb06e2bSBen Skeggs 182ebb945a9SBen Skeggs /* create dma object covering the *entire* memory space that the 183ebb945a9SBen Skeggs * pushbuf lives in, this is because the GEM code requires that 184ebb945a9SBen Skeggs * we be able to call out to other (indirect) push buffers 185ebb945a9SBen Skeggs */ 1860dc9b286SNirmoy Das chan->push.addr = chan->push.buffer->offset; 187ebb945a9SBen Skeggs 188967e7bdeSBen Skeggs if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) { 189bfe91afaSBen Skeggs ret = nouveau_vma_new(chan->push.buffer, chan->vmm, 190ebb945a9SBen Skeggs &chan->push.vma); 191ebb945a9SBen Skeggs if (ret) { 192ebb945a9SBen Skeggs nouveau_channel_del(pchan); 193ebb945a9SBen Skeggs return ret; 194ebb945a9SBen Skeggs } 195ebb945a9SBen Skeggs 19692b4eaafSBen Skeggs chan->push.addr = chan->push.vma->addr; 19792b4eaafSBen Skeggs 19892b4eaafSBen Skeggs if (device->info.family >= NV_DEVICE_INFO_V0_FERMI) 19992b4eaafSBen Skeggs return 0; 20092b4eaafSBen Skeggs 2014acfd707SBen Skeggs args.target = NV_DMA_V0_TARGET_VM; 2024acfd707SBen Skeggs args.access = NV_DMA_V0_ACCESS_VM; 203ebb945a9SBen Skeggs args.start = 0; 204bfe91afaSBen Skeggs args.limit = chan->vmm->vmm.limit - 1; 205ebb945a9SBen Skeggs } else 206d3116756SChristian König if (chan->push.buffer->bo.resource->mem_type == TTM_PL_VRAM) { 207967e7bdeSBen Skeggs if (device->info.family == NV_DEVICE_INFO_V0_TNT) { 208ebb945a9SBen Skeggs /* nv04 vram pushbuf hack, retarget to its location in 209ebb945a9SBen Skeggs * the framebuffer bar rather than direct vram access.. 210ebb945a9SBen Skeggs * nfi why this exists, it came from the -nv ddx. 211ebb945a9SBen Skeggs */ 2124acfd707SBen Skeggs args.target = NV_DMA_V0_TARGET_PCI; 2134acfd707SBen Skeggs args.access = NV_DMA_V0_ACCESS_RDWR; 2147e8820feSBen Skeggs args.start = nvxx_device(device)->func-> 2157e8820feSBen Skeggs resource_addr(nvxx_device(device), 1); 216f392ec4bSBen Skeggs args.limit = args.start + device->info.ram_user - 1; 217ebb945a9SBen Skeggs } else { 2184acfd707SBen Skeggs args.target = NV_DMA_V0_TARGET_VRAM; 2194acfd707SBen Skeggs args.access = NV_DMA_V0_ACCESS_RDWR; 220ebb945a9SBen Skeggs args.start = 0; 221f392ec4bSBen Skeggs args.limit = device->info.ram_user - 1; 222ebb945a9SBen Skeggs } 223ebb945a9SBen Skeggs } else { 224340b0e7cSBen Skeggs if (chan->drm->agp.bridge) { 2254acfd707SBen Skeggs args.target = NV_DMA_V0_TARGET_AGP; 2264acfd707SBen Skeggs args.access = NV_DMA_V0_ACCESS_RDWR; 227ebb945a9SBen Skeggs args.start = chan->drm->agp.base; 228ebb945a9SBen Skeggs args.limit = chan->drm->agp.base + 229ebb945a9SBen Skeggs chan->drm->agp.size - 1; 230ebb945a9SBen Skeggs } else { 2314acfd707SBen Skeggs args.target = NV_DMA_V0_TARGET_VM; 2324acfd707SBen Skeggs args.access = NV_DMA_V0_ACCESS_RDWR; 233ebb945a9SBen Skeggs args.start = 0; 234bfe91afaSBen Skeggs args.limit = chan->vmm->vmm.limit - 1; 235ebb945a9SBen Skeggs } 236ebb945a9SBen Skeggs } 237ebb945a9SBen Skeggs 2389ac596a4SBen Skeggs ret = nvif_object_ctor(&device->object, "abi16PushCtxDma", 0, 2399ac596a4SBen Skeggs NV_DMA_FROM_MEMORY, &args, sizeof(args), 2409ac596a4SBen Skeggs &chan->push.ctxdma); 241ebb945a9SBen Skeggs if (ret) { 242ebb945a9SBen Skeggs nouveau_channel_del(pchan); 243ebb945a9SBen Skeggs return ret; 244ebb945a9SBen Skeggs } 245ebb945a9SBen Skeggs 246ebb945a9SBen Skeggs return 0; 247ebb945a9SBen Skeggs } 248ebb945a9SBen Skeggs 2495b8a43aeSMarcin Slusarz static int 2500ad72863SBen Skeggs nouveau_channel_ind(struct nouveau_drm *drm, struct nvif_device *device, 25185532bd9SBen Skeggs u64 runlist, bool priv, struct nouveau_channel **pchan) 252ebb945a9SBen Skeggs { 25349b2dfc0SBen Skeggs static const u16 oclasses[] = { AMPERE_CHANNEL_GPFIFO_B, 25449b2dfc0SBen Skeggs TURING_CHANNEL_GPFIFO_A, 255641d0b30SBen Skeggs VOLTA_CHANNEL_GPFIFO_A, 25637e1c45aSBen Skeggs PASCAL_CHANNEL_GPFIFO_A, 257e8ff9794SBen Skeggs MAXWELL_CHANNEL_GPFIFO_A, 25863f8c9b7SBen Skeggs KEPLER_CHANNEL_GPFIFO_B, 259a1020afeSBen Skeggs KEPLER_CHANNEL_GPFIFO_A, 260bbf8906bSBen Skeggs FERMI_CHANNEL_GPFIFO, 261bbf8906bSBen Skeggs G82_CHANNEL_GPFIFO, 262bbf8906bSBen Skeggs NV50_CHANNEL_GPFIFO, 263c97f8c92SBen Skeggs 0 }; 264ebb945a9SBen Skeggs const u16 *oclass = oclasses; 265bbf8906bSBen Skeggs union { 266bbf8906bSBen Skeggs struct nv50_channel_gpfifo_v0 nv50; 267159045cdSBen Skeggs struct fermi_channel_gpfifo_v0 fermi; 268bbf8906bSBen Skeggs struct kepler_channel_gpfifo_a_v0 kepler; 2699d24907cSBen Skeggs struct volta_channel_gpfifo_a_v0 volta; 270a01ca78cSBen Skeggs } args; 271ebb945a9SBen Skeggs struct nouveau_channel *chan; 272bbf8906bSBen Skeggs u32 size; 273ebb945a9SBen Skeggs int ret; 274ebb945a9SBen Skeggs 275ebb945a9SBen Skeggs /* allocate dma push buffer */ 276fcf3f91cSBen Skeggs ret = nouveau_channel_prep(drm, device, 0x12000, &chan); 277ebb945a9SBen Skeggs *pchan = chan; 278ebb945a9SBen Skeggs if (ret) 279ebb945a9SBen Skeggs return ret; 280ebb945a9SBen Skeggs 281ebb945a9SBen Skeggs /* create channel object */ 282ebb945a9SBen Skeggs do { 2839d24907cSBen Skeggs if (oclass[0] >= VOLTA_CHANNEL_GPFIFO_A) { 2849d24907cSBen Skeggs args.volta.version = 0; 2859d24907cSBen Skeggs args.volta.ilength = 0x02000; 2869d24907cSBen Skeggs args.volta.ioffset = 0x10000 + chan->push.addr; 2879d24907cSBen Skeggs args.volta.runlist = runlist; 288bfe91afaSBen Skeggs args.volta.vmm = nvif_handle(&chan->vmm->vmm.object); 2899d24907cSBen Skeggs args.volta.priv = priv; 2909d24907cSBen Skeggs size = sizeof(args.volta); 2919d24907cSBen Skeggs } else 292bbf8906bSBen Skeggs if (oclass[0] >= KEPLER_CHANNEL_GPFIFO_A) { 293bbf8906bSBen Skeggs args.kepler.version = 0; 294bbf8906bSBen Skeggs args.kepler.ilength = 0x02000; 29524e8375bSBen Skeggs args.kepler.ioffset = 0x10000 + chan->push.addr; 296a7cf0180SBen Skeggs args.kepler.runlist = runlist; 297bfe91afaSBen Skeggs args.kepler.vmm = nvif_handle(&chan->vmm->vmm.object); 29885532bd9SBen Skeggs args.kepler.priv = priv; 299bbf8906bSBen Skeggs size = sizeof(args.kepler); 300159045cdSBen Skeggs } else 301159045cdSBen Skeggs if (oclass[0] >= FERMI_CHANNEL_GPFIFO) { 302159045cdSBen Skeggs args.fermi.version = 0; 303159045cdSBen Skeggs args.fermi.ilength = 0x02000; 30424e8375bSBen Skeggs args.fermi.ioffset = 0x10000 + chan->push.addr; 305bfe91afaSBen Skeggs args.fermi.vmm = nvif_handle(&chan->vmm->vmm.object); 306159045cdSBen Skeggs size = sizeof(args.fermi); 307bbf8906bSBen Skeggs } else { 308bbf8906bSBen Skeggs args.nv50.version = 0; 309bbf8906bSBen Skeggs args.nv50.ilength = 0x02000; 31024e8375bSBen Skeggs args.nv50.ioffset = 0x10000 + chan->push.addr; 311159045cdSBen Skeggs args.nv50.pushbuf = nvif_handle(&chan->push.ctxdma); 312bfe91afaSBen Skeggs args.nv50.vmm = nvif_handle(&chan->vmm->vmm.object); 313bbf8906bSBen Skeggs size = sizeof(args.nv50); 314bbf8906bSBen Skeggs } 315bbf8906bSBen Skeggs 3169ac596a4SBen Skeggs ret = nvif_object_ctor(&device->object, "abi16ChanUser", 0, 3179ac596a4SBen Skeggs *oclass++, &args, size, &chan->user); 318bbf8906bSBen Skeggs if (ret == 0) { 3199d24907cSBen Skeggs if (chan->user.oclass >= VOLTA_CHANNEL_GPFIFO_A) { 3209d24907cSBen Skeggs chan->chid = args.volta.chid; 3219d24907cSBen Skeggs chan->inst = args.volta.inst; 3229d24907cSBen Skeggs chan->token = args.volta.token; 3239d24907cSBen Skeggs } else 32486b442d7SBen Skeggs if (chan->user.oclass >= KEPLER_CHANNEL_GPFIFO_A) { 325a01ca78cSBen Skeggs chan->chid = args.kepler.chid; 32686b442d7SBen Skeggs chan->inst = args.kepler.inst; 32786b442d7SBen Skeggs } else 32886b442d7SBen Skeggs if (chan->user.oclass >= FERMI_CHANNEL_GPFIFO) { 329159045cdSBen Skeggs chan->chid = args.fermi.chid; 33086b442d7SBen Skeggs } else { 331a01ca78cSBen Skeggs chan->chid = args.nv50.chid; 33286b442d7SBen Skeggs } 333ebb945a9SBen Skeggs return ret; 334bbf8906bSBen Skeggs } 335ebb945a9SBen Skeggs } while (*oclass); 336ebb945a9SBen Skeggs 337ebb945a9SBen Skeggs nouveau_channel_del(pchan); 338ebb945a9SBen Skeggs return ret; 339ebb945a9SBen Skeggs } 340ebb945a9SBen Skeggs 341ebb945a9SBen Skeggs static int 3420ad72863SBen Skeggs nouveau_channel_dma(struct nouveau_drm *drm, struct nvif_device *device, 343fcf3f91cSBen Skeggs struct nouveau_channel **pchan) 344ebb945a9SBen Skeggs { 345bbf8906bSBen Skeggs static const u16 oclasses[] = { NV40_CHANNEL_DMA, 346bbf8906bSBen Skeggs NV17_CHANNEL_DMA, 347bbf8906bSBen Skeggs NV10_CHANNEL_DMA, 348bbf8906bSBen Skeggs NV03_CHANNEL_DMA, 349c97f8c92SBen Skeggs 0 }; 350ebb945a9SBen Skeggs const u16 *oclass = oclasses; 351a01ca78cSBen Skeggs struct nv03_channel_dma_v0 args; 352ebb945a9SBen Skeggs struct nouveau_channel *chan; 353ebb945a9SBen Skeggs int ret; 354ebb945a9SBen Skeggs 355ebb945a9SBen Skeggs /* allocate dma push buffer */ 356fcf3f91cSBen Skeggs ret = nouveau_channel_prep(drm, device, 0x10000, &chan); 357ebb945a9SBen Skeggs *pchan = chan; 358ebb945a9SBen Skeggs if (ret) 359ebb945a9SBen Skeggs return ret; 360ebb945a9SBen Skeggs 361ebb945a9SBen Skeggs /* create channel object */ 362bbf8906bSBen Skeggs args.version = 0; 363bf81df9bSBen Skeggs args.pushbuf = nvif_handle(&chan->push.ctxdma); 36424e8375bSBen Skeggs args.offset = chan->push.addr; 365ebb945a9SBen Skeggs 366ebb945a9SBen Skeggs do { 3679ac596a4SBen Skeggs ret = nvif_object_ctor(&device->object, "abi16ChanUser", 0, 3689ac596a4SBen Skeggs *oclass++, &args, sizeof(args), 3699ac596a4SBen Skeggs &chan->user); 370bbf8906bSBen Skeggs if (ret == 0) { 371a01ca78cSBen Skeggs chan->chid = args.chid; 372ebb945a9SBen Skeggs return ret; 373bbf8906bSBen Skeggs } 374ebb945a9SBen Skeggs } while (ret && *oclass); 375ebb945a9SBen Skeggs 376ebb945a9SBen Skeggs nouveau_channel_del(pchan); 377ebb945a9SBen Skeggs return ret; 378ebb945a9SBen Skeggs } 379ebb945a9SBen Skeggs 380ebb945a9SBen Skeggs static int 381ebb945a9SBen Skeggs nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart) 382ebb945a9SBen Skeggs { 3830ad72863SBen Skeggs struct nvif_device *device = chan->device; 384d8cc37d8SBen Skeggs struct nouveau_drm *drm = chan->drm; 3854acfd707SBen Skeggs struct nv_dma_v0 args = {}; 386ebb945a9SBen Skeggs int ret, i; 387ebb945a9SBen Skeggs 388*097d56cdSBen Skeggs ret = nvif_object_map(&chan->user, NULL, 0); 389*097d56cdSBen Skeggs if (ret) 390*097d56cdSBen Skeggs return ret; 3916c6ae061SBen Skeggs 39249b2dfc0SBen Skeggs if (chan->user.oclass >= FERMI_CHANNEL_GPFIFO && 39349b2dfc0SBen Skeggs chan->user.oclass < AMPERE_CHANNEL_GPFIFO_B) { 394f7a7d22aSBen Skeggs ret = nvif_notify_ctor(&chan->user, "abi16ChanKilled", 395f7a7d22aSBen Skeggs nouveau_channel_killed, 396d8cc37d8SBen Skeggs true, NV906F_V0_NTFY_KILLED, 397d8cc37d8SBen Skeggs NULL, 0, 0, &chan->kill); 398d8cc37d8SBen Skeggs if (ret == 0) 399d8cc37d8SBen Skeggs ret = nvif_notify_get(&chan->kill); 400d8cc37d8SBen Skeggs if (ret) { 401d8cc37d8SBen Skeggs NV_ERROR(drm, "Failed to request channel kill " 402d8cc37d8SBen Skeggs "notification: %d\n", ret); 403d8cc37d8SBen Skeggs return ret; 404d8cc37d8SBen Skeggs } 405d8cc37d8SBen Skeggs } 406d8cc37d8SBen Skeggs 407ebb945a9SBen Skeggs /* allocate dma objects to cover all allowed vram, and gart */ 408967e7bdeSBen Skeggs if (device->info.family < NV_DEVICE_INFO_V0_FERMI) { 409967e7bdeSBen Skeggs if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) { 4104acfd707SBen Skeggs args.target = NV_DMA_V0_TARGET_VM; 4114acfd707SBen Skeggs args.access = NV_DMA_V0_ACCESS_VM; 412ebb945a9SBen Skeggs args.start = 0; 413bfe91afaSBen Skeggs args.limit = chan->vmm->vmm.limit - 1; 414ebb945a9SBen Skeggs } else { 4154acfd707SBen Skeggs args.target = NV_DMA_V0_TARGET_VRAM; 4164acfd707SBen Skeggs args.access = NV_DMA_V0_ACCESS_RDWR; 417ebb945a9SBen Skeggs args.start = 0; 418f392ec4bSBen Skeggs args.limit = device->info.ram_user - 1; 419ebb945a9SBen Skeggs } 420ebb945a9SBen Skeggs 4219ac596a4SBen Skeggs ret = nvif_object_ctor(&chan->user, "abi16ChanVramCtxDma", vram, 4229ac596a4SBen Skeggs NV_DMA_IN_MEMORY, &args, sizeof(args), 4239ac596a4SBen Skeggs &chan->vram); 424ebb945a9SBen Skeggs if (ret) 425ebb945a9SBen Skeggs return ret; 426ebb945a9SBen Skeggs 427967e7bdeSBen Skeggs if (device->info.family >= NV_DEVICE_INFO_V0_TESLA) { 4284acfd707SBen Skeggs args.target = NV_DMA_V0_TARGET_VM; 4294acfd707SBen Skeggs args.access = NV_DMA_V0_ACCESS_VM; 430ebb945a9SBen Skeggs args.start = 0; 431bfe91afaSBen Skeggs args.limit = chan->vmm->vmm.limit - 1; 432ebb945a9SBen Skeggs } else 433340b0e7cSBen Skeggs if (chan->drm->agp.bridge) { 4344acfd707SBen Skeggs args.target = NV_DMA_V0_TARGET_AGP; 4354acfd707SBen Skeggs args.access = NV_DMA_V0_ACCESS_RDWR; 436ebb945a9SBen Skeggs args.start = chan->drm->agp.base; 437ebb945a9SBen Skeggs args.limit = chan->drm->agp.base + 438ebb945a9SBen Skeggs chan->drm->agp.size - 1; 439ebb945a9SBen Skeggs } else { 4404acfd707SBen Skeggs args.target = NV_DMA_V0_TARGET_VM; 4414acfd707SBen Skeggs args.access = NV_DMA_V0_ACCESS_RDWR; 442ebb945a9SBen Skeggs args.start = 0; 443bfe91afaSBen Skeggs args.limit = chan->vmm->vmm.limit - 1; 444ebb945a9SBen Skeggs } 445ebb945a9SBen Skeggs 4469ac596a4SBen Skeggs ret = nvif_object_ctor(&chan->user, "abi16ChanGartCtxDma", gart, 4479ac596a4SBen Skeggs NV_DMA_IN_MEMORY, &args, sizeof(args), 4489ac596a4SBen Skeggs &chan->gart); 449ebb945a9SBen Skeggs if (ret) 450ebb945a9SBen Skeggs return ret; 451ebb945a9SBen Skeggs } 452ebb945a9SBen Skeggs 453ebb945a9SBen Skeggs /* initialise dma tracking parameters */ 454a01ca78cSBen Skeggs switch (chan->user.oclass & 0x00ff) { 455503b0f1cSBen Skeggs case 0x006b: 456ebb945a9SBen Skeggs case 0x006e: 457ebb945a9SBen Skeggs chan->user_put = 0x40; 458ebb945a9SBen Skeggs chan->user_get = 0x44; 459ebb945a9SBen Skeggs chan->dma.max = (0x10000 / 4) - 2; 460ebb945a9SBen Skeggs break; 461ebb945a9SBen Skeggs default: 462ebb945a9SBen Skeggs chan->user_put = 0x40; 463ebb945a9SBen Skeggs chan->user_get = 0x44; 464ebb945a9SBen Skeggs chan->user_get_hi = 0x60; 465ebb945a9SBen Skeggs chan->dma.ib_base = 0x10000 / 4; 466ebb945a9SBen Skeggs chan->dma.ib_max = (0x02000 / 8) - 1; 467ebb945a9SBen Skeggs chan->dma.ib_put = 0; 468ebb945a9SBen Skeggs chan->dma.ib_free = chan->dma.ib_max - chan->dma.ib_put; 469ebb945a9SBen Skeggs chan->dma.max = chan->dma.ib_base; 470ebb945a9SBen Skeggs break; 471ebb945a9SBen Skeggs } 472ebb945a9SBen Skeggs 473ebb945a9SBen Skeggs chan->dma.put = 0; 474ebb945a9SBen Skeggs chan->dma.cur = chan->dma.put; 475ebb945a9SBen Skeggs chan->dma.free = chan->dma.max - chan->dma.cur; 476ebb945a9SBen Skeggs 477cd346a89SBen Skeggs ret = PUSH_WAIT(chan->chan.push, NOUVEAU_DMA_SKIPS); 478ebb945a9SBen Skeggs if (ret) 479ebb945a9SBen Skeggs return ret; 480ebb945a9SBen Skeggs 481ebb945a9SBen Skeggs for (i = 0; i < NOUVEAU_DMA_SKIPS; i++) 482cd346a89SBen Skeggs PUSH_DATA(chan->chan.push, 0x00000000); 483ebb945a9SBen Skeggs 48469a6146dSBen Skeggs /* allocate software object class (used for fences on <= nv05) */ 485967e7bdeSBen Skeggs if (device->info.family < NV_DEVICE_INFO_V0_CELSIUS) { 4869ac596a4SBen Skeggs ret = nvif_object_ctor(&chan->user, "abi16NvswFence", 0x006e, 48708f7633cSBen Skeggs NVIF_CLASS_SW_NV04, 4880ad72863SBen Skeggs NULL, 0, &chan->nvsw); 489ebb945a9SBen Skeggs if (ret) 490ebb945a9SBen Skeggs return ret; 491ebb945a9SBen Skeggs 492cd346a89SBen Skeggs ret = PUSH_WAIT(chan->chan.push, 2); 493ebb945a9SBen Skeggs if (ret) 494ebb945a9SBen Skeggs return ret; 495ebb945a9SBen Skeggs 496cd346a89SBen Skeggs PUSH_NVSQ(chan->chan.push, NV_SW, 0x0000, chan->nvsw.handle); 497cd346a89SBen Skeggs PUSH_KICK(chan->chan.push); 498ebb945a9SBen Skeggs } 499ebb945a9SBen Skeggs 500ebb945a9SBen Skeggs /* initialise synchronisation */ 5014894f662SBen Skeggs return nouveau_fence(chan->drm)->context_new(chan); 502ebb945a9SBen Skeggs } 503ebb945a9SBen Skeggs 504ebb945a9SBen Skeggs int 5050ad72863SBen Skeggs nouveau_channel_new(struct nouveau_drm *drm, struct nvif_device *device, 50685532bd9SBen Skeggs u32 arg0, u32 arg1, bool priv, 50785532bd9SBen Skeggs struct nouveau_channel **pchan) 508ebb945a9SBen Skeggs { 509a01ca78cSBen Skeggs struct nouveau_cli *cli = (void *)device->object.client; 510ebb945a9SBen Skeggs int ret; 511ebb945a9SBen Skeggs 51267e26e41SBen Skeggs /* hack until fencenv50 is fixed, and agp access relaxed */ 51385532bd9SBen Skeggs ret = nouveau_channel_ind(drm, device, arg0, priv, pchan); 514ebb945a9SBen Skeggs if (ret) { 5159ad97edeSBen Skeggs NV_PRINTK(dbg, cli, "ib channel create, %d\n", ret); 516fcf3f91cSBen Skeggs ret = nouveau_channel_dma(drm, device, pchan); 517ebb945a9SBen Skeggs if (ret) { 5189ad97edeSBen Skeggs NV_PRINTK(dbg, cli, "dma channel create, %d\n", ret); 51959f216cfSBen Skeggs return ret; 520ebb945a9SBen Skeggs } 521ebb945a9SBen Skeggs } 522ebb945a9SBen Skeggs 52349981046SBen Skeggs ret = nouveau_channel_init(*pchan, arg0, arg1); 524ebb945a9SBen Skeggs if (ret) { 5259ad97edeSBen Skeggs NV_PRINTK(err, cli, "channel failed to initialise, %d\n", ret); 526ebb945a9SBen Skeggs nouveau_channel_del(pchan); 52759f216cfSBen Skeggs return ret; 528ebb945a9SBen Skeggs } 529ebb945a9SBen Skeggs 530eeaf06acSBen Skeggs ret = nouveau_svmm_join((*pchan)->vmm->svmm, (*pchan)->inst); 531eeaf06acSBen Skeggs if (ret) 532eeaf06acSBen Skeggs nouveau_channel_del(pchan); 533eeaf06acSBen Skeggs 53467e26e41SBen Skeggs return ret; 535ebb945a9SBen Skeggs } 536eb47db4fSBen Skeggs 537eb47db4fSBen Skeggs int 538eb47db4fSBen Skeggs nouveau_channels_init(struct nouveau_drm *drm) 539eb47db4fSBen Skeggs { 540eb47db4fSBen Skeggs struct { 541eb47db4fSBen Skeggs struct nv_device_info_v1 m; 542eb47db4fSBen Skeggs struct { 543eb47db4fSBen Skeggs struct nv_device_info_v1_data channels; 544eb47db4fSBen Skeggs } v; 545eb47db4fSBen Skeggs } args = { 546eb47db4fSBen Skeggs .m.version = 1, 547eb47db4fSBen Skeggs .m.count = sizeof(args.v) / sizeof(args.v.channels), 548f8fabd31SBen Skeggs .v.channels.mthd = NV_DEVICE_HOST_CHANNELS, 549eb47db4fSBen Skeggs }; 550eb47db4fSBen Skeggs struct nvif_object *device = &drm->client.device.object; 551eb47db4fSBen Skeggs int ret; 552eb47db4fSBen Skeggs 553eb47db4fSBen Skeggs ret = nvif_object_mthd(device, NV_DEVICE_V0_INFO, &args, sizeof(args)); 554eb47db4fSBen Skeggs if (ret || args.v.channels.mthd == NV_DEVICE_INFO_INVALID) 555eb47db4fSBen Skeggs return -ENODEV; 556eb47db4fSBen Skeggs 557eb47db4fSBen Skeggs drm->chan.nr = args.v.channels.data; 558eb47db4fSBen Skeggs drm->chan.context_base = dma_fence_context_alloc(drm->chan.nr); 559eb47db4fSBen Skeggs return 0; 560eb47db4fSBen Skeggs } 561