1361c3cd8SBen Skeggs /*
2361c3cd8SBen Skeggs * Copyright 2023 Red Hat Inc.
3361c3cd8SBen Skeggs *
4361c3cd8SBen Skeggs * Permission is hereby granted, free of charge, to any person obtaining a
5361c3cd8SBen Skeggs * copy of this software and associated documentation files (the "Software"),
6361c3cd8SBen Skeggs * to deal in the Software without restriction, including without limitation
7361c3cd8SBen Skeggs * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8361c3cd8SBen Skeggs * and/or sell copies of the Software, and to permit persons to whom the
9361c3cd8SBen Skeggs * Software is furnished to do so, subject to the following conditions:
10361c3cd8SBen Skeggs *
11361c3cd8SBen Skeggs * The above copyright notice and this permission notice shall be included in
12361c3cd8SBen Skeggs * all copies or substantial portions of the Software.
13361c3cd8SBen Skeggs *
14361c3cd8SBen Skeggs * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15361c3cd8SBen Skeggs * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16361c3cd8SBen Skeggs * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17361c3cd8SBen Skeggs * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18361c3cd8SBen Skeggs * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19361c3cd8SBen Skeggs * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20361c3cd8SBen Skeggs * OTHER DEALINGS IN THE SOFTWARE.
21361c3cd8SBen Skeggs */
22361c3cd8SBen Skeggs #include "gf100.h"
23361c3cd8SBen Skeggs
24361c3cd8SBen Skeggs #include <core/memory.h>
25361c3cd8SBen Skeggs #include <subdev/gsp.h>
26361c3cd8SBen Skeggs #include <subdev/mmu/vmm.h>
27361c3cd8SBen Skeggs #include <engine/fifo/priv.h>
28361c3cd8SBen Skeggs
29361c3cd8SBen Skeggs #include <nvif/if900d.h>
30361c3cd8SBen Skeggs
31361c3cd8SBen Skeggs #include <nvhw/drf.h>
32361c3cd8SBen Skeggs
33361c3cd8SBen Skeggs #include <nvrm/nvtypes.h>
34*b5bad8c1SDave Airlie #include <nvrm/535.113.01/common/sdk/nvidia/inc/alloc/alloc_channel.h>
35*b5bad8c1SDave Airlie #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080fifo.h>
36*b5bad8c1SDave Airlie #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h>
37*b5bad8c1SDave Airlie #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080internal.h>
38*b5bad8c1SDave Airlie #include <nvrm/535.113.01/nvidia/generated/g_kernel_channel_nvoc.h>
39361c3cd8SBen Skeggs
40361c3cd8SBen Skeggs #define r535_gr(p) container_of((p), struct r535_gr, base)
41361c3cd8SBen Skeggs
42361c3cd8SBen Skeggs #define R515_GR_MAX_CTXBUFS 9
43361c3cd8SBen Skeggs
44361c3cd8SBen Skeggs struct r535_gr {
45361c3cd8SBen Skeggs struct nvkm_gr base;
46361c3cd8SBen Skeggs
47361c3cd8SBen Skeggs struct {
48361c3cd8SBen Skeggs u16 bufferId;
49361c3cd8SBen Skeggs u32 size;
50361c3cd8SBen Skeggs u8 page;
51361c3cd8SBen Skeggs u8 align;
52361c3cd8SBen Skeggs bool global;
53361c3cd8SBen Skeggs bool init;
54361c3cd8SBen Skeggs bool ro;
55361c3cd8SBen Skeggs } ctxbuf[R515_GR_MAX_CTXBUFS];
56361c3cd8SBen Skeggs int ctxbuf_nr;
57361c3cd8SBen Skeggs
58361c3cd8SBen Skeggs struct nvkm_memory *ctxbuf_mem[R515_GR_MAX_CTXBUFS];
59361c3cd8SBen Skeggs };
60361c3cd8SBen Skeggs
61361c3cd8SBen Skeggs struct r535_gr_chan {
62361c3cd8SBen Skeggs struct nvkm_object object;
63361c3cd8SBen Skeggs struct r535_gr *gr;
64361c3cd8SBen Skeggs
65361c3cd8SBen Skeggs struct nvkm_vmm *vmm;
66361c3cd8SBen Skeggs struct nvkm_chan *chan;
67361c3cd8SBen Skeggs
68361c3cd8SBen Skeggs struct nvkm_memory *mem[R515_GR_MAX_CTXBUFS];
69361c3cd8SBen Skeggs struct nvkm_vma *vma[R515_GR_MAX_CTXBUFS];
70361c3cd8SBen Skeggs };
71361c3cd8SBen Skeggs
72361c3cd8SBen Skeggs struct r535_gr_obj {
73361c3cd8SBen Skeggs struct nvkm_object object;
74361c3cd8SBen Skeggs struct nvkm_gsp_object rm;
75361c3cd8SBen Skeggs };
76361c3cd8SBen Skeggs
77361c3cd8SBen Skeggs static void *
r535_gr_obj_dtor(struct nvkm_object * object)78361c3cd8SBen Skeggs r535_gr_obj_dtor(struct nvkm_object *object)
79361c3cd8SBen Skeggs {
80361c3cd8SBen Skeggs struct r535_gr_obj *obj = container_of(object, typeof(*obj), object);
81361c3cd8SBen Skeggs
82361c3cd8SBen Skeggs nvkm_gsp_rm_free(&obj->rm);
83361c3cd8SBen Skeggs return obj;
84361c3cd8SBen Skeggs }
85361c3cd8SBen Skeggs
86361c3cd8SBen Skeggs static const struct nvkm_object_func
87361c3cd8SBen Skeggs r535_gr_obj = {
88361c3cd8SBen Skeggs .dtor = r535_gr_obj_dtor,
89361c3cd8SBen Skeggs };
90361c3cd8SBen Skeggs
91361c3cd8SBen Skeggs static int
r535_gr_obj_ctor(const struct nvkm_oclass * oclass,void * argv,u32 argc,struct nvkm_object ** pobject)92361c3cd8SBen Skeggs r535_gr_obj_ctor(const struct nvkm_oclass *oclass, void *argv, u32 argc,
93361c3cd8SBen Skeggs struct nvkm_object **pobject)
94361c3cd8SBen Skeggs {
95361c3cd8SBen Skeggs struct r535_gr_chan *chan = container_of(oclass->parent, typeof(*chan), object);
96361c3cd8SBen Skeggs struct r535_gr_obj *obj;
97361c3cd8SBen Skeggs
98361c3cd8SBen Skeggs if (!(obj = kzalloc(sizeof(*obj), GFP_KERNEL)))
99361c3cd8SBen Skeggs return -ENOMEM;
100361c3cd8SBen Skeggs
101361c3cd8SBen Skeggs nvkm_object_ctor(&r535_gr_obj, oclass, &obj->object);
102361c3cd8SBen Skeggs *pobject = &obj->object;
103361c3cd8SBen Skeggs
104361c3cd8SBen Skeggs return nvkm_gsp_rm_alloc(&chan->chan->rm.object, oclass->handle, oclass->base.oclass, 0,
105361c3cd8SBen Skeggs &obj->rm);
106361c3cd8SBen Skeggs }
107361c3cd8SBen Skeggs
108361c3cd8SBen Skeggs static void *
r535_gr_chan_dtor(struct nvkm_object * object)109361c3cd8SBen Skeggs r535_gr_chan_dtor(struct nvkm_object *object)
110361c3cd8SBen Skeggs {
111361c3cd8SBen Skeggs struct r535_gr_chan *grc = container_of(object, typeof(*grc), object);
112361c3cd8SBen Skeggs struct r535_gr *gr = grc->gr;
113361c3cd8SBen Skeggs
114361c3cd8SBen Skeggs for (int i = 0; i < gr->ctxbuf_nr; i++) {
115361c3cd8SBen Skeggs nvkm_vmm_put(grc->vmm, &grc->vma[i]);
116361c3cd8SBen Skeggs nvkm_memory_unref(&grc->mem[i]);
117361c3cd8SBen Skeggs }
118361c3cd8SBen Skeggs
119361c3cd8SBen Skeggs nvkm_vmm_unref(&grc->vmm);
120361c3cd8SBen Skeggs return grc;
121361c3cd8SBen Skeggs }
122361c3cd8SBen Skeggs
123361c3cd8SBen Skeggs static const struct nvkm_object_func
124361c3cd8SBen Skeggs r535_gr_chan = {
125361c3cd8SBen Skeggs .dtor = r535_gr_chan_dtor,
126361c3cd8SBen Skeggs };
127361c3cd8SBen Skeggs
128361c3cd8SBen Skeggs static int
r535_gr_promote_ctx(struct r535_gr * gr,bool golden,struct nvkm_vmm * vmm,struct nvkm_memory ** pmem,struct nvkm_vma ** pvma,struct nvkm_gsp_object * chan)129361c3cd8SBen Skeggs r535_gr_promote_ctx(struct r535_gr *gr, bool golden, struct nvkm_vmm *vmm,
130361c3cd8SBen Skeggs struct nvkm_memory **pmem, struct nvkm_vma **pvma,
131361c3cd8SBen Skeggs struct nvkm_gsp_object *chan)
132361c3cd8SBen Skeggs {
133361c3cd8SBen Skeggs struct nvkm_subdev *subdev = &gr->base.engine.subdev;
134361c3cd8SBen Skeggs struct nvkm_device *device = subdev->device;
135361c3cd8SBen Skeggs NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS *ctrl;
136361c3cd8SBen Skeggs
137361c3cd8SBen Skeggs ctrl = nvkm_gsp_rm_ctrl_get(&vmm->rm.device.subdevice,
138361c3cd8SBen Skeggs NV2080_CTRL_CMD_GPU_PROMOTE_CTX, sizeof(*ctrl));
139361c3cd8SBen Skeggs if (WARN_ON(IS_ERR(ctrl)))
140361c3cd8SBen Skeggs return PTR_ERR(ctrl);
141361c3cd8SBen Skeggs
142361c3cd8SBen Skeggs ctrl->engineType = 1;
143361c3cd8SBen Skeggs ctrl->hChanClient = vmm->rm.client.object.handle;
144361c3cd8SBen Skeggs ctrl->hObject = chan->handle;
145361c3cd8SBen Skeggs
146361c3cd8SBen Skeggs for (int i = 0; i < gr->ctxbuf_nr; i++) {
147361c3cd8SBen Skeggs NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY *entry =
148361c3cd8SBen Skeggs &ctrl->promoteEntry[ctrl->entryCount];
149361c3cd8SBen Skeggs const bool alloc = golden || !gr->ctxbuf[i].global;
150361c3cd8SBen Skeggs int ret;
151361c3cd8SBen Skeggs
152361c3cd8SBen Skeggs entry->bufferId = gr->ctxbuf[i].bufferId;
153361c3cd8SBen Skeggs entry->bInitialize = gr->ctxbuf[i].init && alloc;
154361c3cd8SBen Skeggs
155361c3cd8SBen Skeggs if (alloc) {
156361c3cd8SBen Skeggs ret = nvkm_memory_new(device, gr->ctxbuf[i].init ?
157361c3cd8SBen Skeggs NVKM_MEM_TARGET_INST : NVKM_MEM_TARGET_INST_SR_LOST,
158361c3cd8SBen Skeggs gr->ctxbuf[i].size, 1 << gr->ctxbuf[i].page,
159361c3cd8SBen Skeggs gr->ctxbuf[i].init, &pmem[i]);
160361c3cd8SBen Skeggs if (WARN_ON(ret))
161361c3cd8SBen Skeggs return ret;
162361c3cd8SBen Skeggs
163361c3cd8SBen Skeggs if (gr->ctxbuf[i].bufferId ==
164361c3cd8SBen Skeggs NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP)
165361c3cd8SBen Skeggs entry->bNonmapped = 1;
166361c3cd8SBen Skeggs } else {
167361c3cd8SBen Skeggs if (gr->ctxbuf[i].bufferId ==
168361c3cd8SBen Skeggs NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP)
169361c3cd8SBen Skeggs continue;
170361c3cd8SBen Skeggs
171361c3cd8SBen Skeggs pmem[i] = nvkm_memory_ref(gr->ctxbuf_mem[i]);
172361c3cd8SBen Skeggs }
173361c3cd8SBen Skeggs
174361c3cd8SBen Skeggs if (!entry->bNonmapped) {
175361c3cd8SBen Skeggs struct gf100_vmm_map_v0 args = {
176361c3cd8SBen Skeggs .priv = 1,
177361c3cd8SBen Skeggs .ro = gr->ctxbuf[i].ro,
178361c3cd8SBen Skeggs };
179361c3cd8SBen Skeggs
180361c3cd8SBen Skeggs mutex_lock(&vmm->mutex.vmm);
181361c3cd8SBen Skeggs ret = nvkm_vmm_get_locked(vmm, false, true, false, 0, gr->ctxbuf[i].align,
182361c3cd8SBen Skeggs nvkm_memory_size(pmem[i]), &pvma[i]);
183361c3cd8SBen Skeggs mutex_unlock(&vmm->mutex.vmm);
184361c3cd8SBen Skeggs if (ret)
185361c3cd8SBen Skeggs return ret;
186361c3cd8SBen Skeggs
187361c3cd8SBen Skeggs ret = nvkm_memory_map(pmem[i], 0, vmm, pvma[i], &args, sizeof(args));
188361c3cd8SBen Skeggs if (ret)
189361c3cd8SBen Skeggs return ret;
190361c3cd8SBen Skeggs
191361c3cd8SBen Skeggs entry->gpuVirtAddr = pvma[i]->addr;
192361c3cd8SBen Skeggs }
193361c3cd8SBen Skeggs
194361c3cd8SBen Skeggs if (entry->bInitialize) {
195361c3cd8SBen Skeggs entry->gpuPhysAddr = nvkm_memory_addr(pmem[i]);
196361c3cd8SBen Skeggs entry->size = gr->ctxbuf[i].size;
197361c3cd8SBen Skeggs entry->physAttr = 4;
198361c3cd8SBen Skeggs }
199361c3cd8SBen Skeggs
200361c3cd8SBen Skeggs nvkm_debug(subdev,
201361c3cd8SBen Skeggs "promote %02d: pa %016llx/%08x sz %016llx va %016llx init:%d nm:%d\n",
202361c3cd8SBen Skeggs entry->bufferId, entry->gpuPhysAddr, entry->physAttr, entry->size,
203361c3cd8SBen Skeggs entry->gpuVirtAddr, entry->bInitialize, entry->bNonmapped);
204361c3cd8SBen Skeggs
205361c3cd8SBen Skeggs ctrl->entryCount++;
206361c3cd8SBen Skeggs }
207361c3cd8SBen Skeggs
208361c3cd8SBen Skeggs return nvkm_gsp_rm_ctrl_wr(&vmm->rm.device.subdevice, ctrl);
209361c3cd8SBen Skeggs }
210361c3cd8SBen Skeggs
211361c3cd8SBen Skeggs static int
r535_gr_chan_new(struct nvkm_gr * base,struct nvkm_chan * chan,const struct nvkm_oclass * oclass,struct nvkm_object ** pobject)212361c3cd8SBen Skeggs r535_gr_chan_new(struct nvkm_gr *base, struct nvkm_chan *chan, const struct nvkm_oclass *oclass,
213361c3cd8SBen Skeggs struct nvkm_object **pobject)
214361c3cd8SBen Skeggs {
215361c3cd8SBen Skeggs struct r535_gr *gr = r535_gr(base);
216361c3cd8SBen Skeggs struct r535_gr_chan *grc;
217361c3cd8SBen Skeggs int ret;
218361c3cd8SBen Skeggs
219361c3cd8SBen Skeggs if (!(grc = kzalloc(sizeof(*grc), GFP_KERNEL)))
220361c3cd8SBen Skeggs return -ENOMEM;
221361c3cd8SBen Skeggs
222361c3cd8SBen Skeggs nvkm_object_ctor(&r535_gr_chan, oclass, &grc->object);
223361c3cd8SBen Skeggs grc->gr = gr;
224361c3cd8SBen Skeggs grc->vmm = nvkm_vmm_ref(chan->vmm);
225361c3cd8SBen Skeggs grc->chan = chan;
226361c3cd8SBen Skeggs *pobject = &grc->object;
227361c3cd8SBen Skeggs
228361c3cd8SBen Skeggs ret = r535_gr_promote_ctx(gr, false, grc->vmm, grc->mem, grc->vma, &chan->rm.object);
229361c3cd8SBen Skeggs if (ret)
230361c3cd8SBen Skeggs return ret;
231361c3cd8SBen Skeggs
232361c3cd8SBen Skeggs return 0;
233361c3cd8SBen Skeggs }
234361c3cd8SBen Skeggs
235361c3cd8SBen Skeggs static u64
r535_gr_units(struct nvkm_gr * gr)236361c3cd8SBen Skeggs r535_gr_units(struct nvkm_gr *gr)
237361c3cd8SBen Skeggs {
238361c3cd8SBen Skeggs struct nvkm_gsp *gsp = gr->engine.subdev.device->gsp;
239361c3cd8SBen Skeggs
240361c3cd8SBen Skeggs return (gsp->gr.tpcs << 8) | gsp->gr.gpcs;
241361c3cd8SBen Skeggs }
242361c3cd8SBen Skeggs
243361c3cd8SBen Skeggs static int
r535_gr_oneinit(struct nvkm_gr * base)244361c3cd8SBen Skeggs r535_gr_oneinit(struct nvkm_gr *base)
245361c3cd8SBen Skeggs {
246361c3cd8SBen Skeggs NV2080_CTRL_INTERNAL_STATIC_GR_GET_CONTEXT_BUFFERS_INFO_PARAMS *info;
247361c3cd8SBen Skeggs struct r535_gr *gr = container_of(base, typeof(*gr), base);
248361c3cd8SBen Skeggs struct nvkm_subdev *subdev = &gr->base.engine.subdev;
249361c3cd8SBen Skeggs struct nvkm_device *device = subdev->device;
250361c3cd8SBen Skeggs struct nvkm_gsp *gsp = device->gsp;
251361c3cd8SBen Skeggs struct nvkm_mmu *mmu = device->mmu;
252361c3cd8SBen Skeggs struct {
253361c3cd8SBen Skeggs struct nvkm_memory *inst;
254361c3cd8SBen Skeggs struct nvkm_vmm *vmm;
255361c3cd8SBen Skeggs struct nvkm_gsp_object chan;
256361c3cd8SBen Skeggs struct nvkm_vma *vma[R515_GR_MAX_CTXBUFS];
257361c3cd8SBen Skeggs } golden = {};
258361c3cd8SBen Skeggs int ret;
259361c3cd8SBen Skeggs
260361c3cd8SBen Skeggs /* Allocate a channel to use for golden context init. */
261361c3cd8SBen Skeggs ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x12000, 0, true, &golden.inst);
262361c3cd8SBen Skeggs if (ret)
263361c3cd8SBen Skeggs goto done;
264361c3cd8SBen Skeggs
265361c3cd8SBen Skeggs ret = nvkm_vmm_new(device, 0x1000, 0, NULL, 0, NULL, "grGoldenVmm", &golden.vmm);
266361c3cd8SBen Skeggs if (ret)
267361c3cd8SBen Skeggs goto done;
268361c3cd8SBen Skeggs
269361c3cd8SBen Skeggs ret = mmu->func->promote_vmm(golden.vmm);
270361c3cd8SBen Skeggs if (ret)
271361c3cd8SBen Skeggs goto done;
272361c3cd8SBen Skeggs
273361c3cd8SBen Skeggs {
274361c3cd8SBen Skeggs NV_CHANNELGPFIFO_ALLOCATION_PARAMETERS *args;
275361c3cd8SBen Skeggs
276361c3cd8SBen Skeggs args = nvkm_gsp_rm_alloc_get(&golden.vmm->rm.device.object, 0xf1f00000,
277361c3cd8SBen Skeggs device->fifo->func->chan.user.oclass,
278361c3cd8SBen Skeggs sizeof(*args), &golden.chan);
279361c3cd8SBen Skeggs if (IS_ERR(args)) {
280361c3cd8SBen Skeggs ret = PTR_ERR(args);
281361c3cd8SBen Skeggs goto done;
282361c3cd8SBen Skeggs }
283361c3cd8SBen Skeggs
284361c3cd8SBen Skeggs args->gpFifoOffset = 0;
285361c3cd8SBen Skeggs args->gpFifoEntries = 0x1000 / 8;
286361c3cd8SBen Skeggs args->flags =
287361c3cd8SBen Skeggs NVDEF(NVOS04, FLAGS, CHANNEL_TYPE, PHYSICAL) |
288361c3cd8SBen Skeggs NVDEF(NVOS04, FLAGS, VPR, FALSE) |
289361c3cd8SBen Skeggs NVDEF(NVOS04, FLAGS, CHANNEL_SKIP_MAP_REFCOUNTING, FALSE) |
290361c3cd8SBen Skeggs NVVAL(NVOS04, FLAGS, GROUP_CHANNEL_RUNQUEUE, 0) |
291361c3cd8SBen Skeggs NVDEF(NVOS04, FLAGS, PRIVILEGED_CHANNEL, TRUE) |
292361c3cd8SBen Skeggs NVDEF(NVOS04, FLAGS, DELAY_CHANNEL_SCHEDULING, FALSE) |
293361c3cd8SBen Skeggs NVDEF(NVOS04, FLAGS, CHANNEL_DENY_PHYSICAL_MODE_CE, FALSE) |
294361c3cd8SBen Skeggs NVVAL(NVOS04, FLAGS, CHANNEL_USERD_INDEX_VALUE, 0) |
295361c3cd8SBen Skeggs NVDEF(NVOS04, FLAGS, CHANNEL_USERD_INDEX_FIXED, FALSE) |
296361c3cd8SBen Skeggs NVVAL(NVOS04, FLAGS, CHANNEL_USERD_INDEX_PAGE_VALUE, 0) |
297361c3cd8SBen Skeggs NVDEF(NVOS04, FLAGS, CHANNEL_USERD_INDEX_PAGE_FIXED, TRUE) |
298361c3cd8SBen Skeggs NVDEF(NVOS04, FLAGS, CHANNEL_DENY_AUTH_LEVEL_PRIV, FALSE) |
299361c3cd8SBen Skeggs NVDEF(NVOS04, FLAGS, CHANNEL_SKIP_SCRUBBER, FALSE) |
300361c3cd8SBen Skeggs NVDEF(NVOS04, FLAGS, CHANNEL_CLIENT_MAP_FIFO, FALSE) |
301361c3cd8SBen Skeggs NVDEF(NVOS04, FLAGS, SET_EVICT_LAST_CE_PREFETCH_CHANNEL, FALSE) |
302361c3cd8SBen Skeggs NVDEF(NVOS04, FLAGS, CHANNEL_VGPU_PLUGIN_CONTEXT, FALSE) |
303361c3cd8SBen Skeggs NVDEF(NVOS04, FLAGS, CHANNEL_PBDMA_ACQUIRE_TIMEOUT, FALSE) |
304361c3cd8SBen Skeggs NVDEF(NVOS04, FLAGS, GROUP_CHANNEL_THREAD, DEFAULT) |
305361c3cd8SBen Skeggs NVDEF(NVOS04, FLAGS, MAP_CHANNEL, FALSE) |
306361c3cd8SBen Skeggs NVDEF(NVOS04, FLAGS, SKIP_CTXBUFFER_ALLOC, FALSE);
307361c3cd8SBen Skeggs args->hVASpace = golden.vmm->rm.object.handle;
308361c3cd8SBen Skeggs args->engineType = 1;
309361c3cd8SBen Skeggs args->instanceMem.base = nvkm_memory_addr(golden.inst);
310361c3cd8SBen Skeggs args->instanceMem.size = 0x1000;
311361c3cd8SBen Skeggs args->instanceMem.addressSpace = 2;
312361c3cd8SBen Skeggs args->instanceMem.cacheAttrib = 1;
313361c3cd8SBen Skeggs args->ramfcMem.base = nvkm_memory_addr(golden.inst);
314361c3cd8SBen Skeggs args->ramfcMem.size = 0x200;
315361c3cd8SBen Skeggs args->ramfcMem.addressSpace = 2;
316361c3cd8SBen Skeggs args->ramfcMem.cacheAttrib = 1;
317361c3cd8SBen Skeggs args->userdMem.base = nvkm_memory_addr(golden.inst) + 0x1000;
318361c3cd8SBen Skeggs args->userdMem.size = 0x200;
319361c3cd8SBen Skeggs args->userdMem.addressSpace = 2;
320361c3cd8SBen Skeggs args->userdMem.cacheAttrib = 1;
321361c3cd8SBen Skeggs args->mthdbufMem.base = nvkm_memory_addr(golden.inst) + 0x2000;
322361c3cd8SBen Skeggs args->mthdbufMem.size = 0x5000;
323361c3cd8SBen Skeggs args->mthdbufMem.addressSpace = 2;
324361c3cd8SBen Skeggs args->mthdbufMem.cacheAttrib = 1;
325361c3cd8SBen Skeggs args->internalFlags =
326361c3cd8SBen Skeggs NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, PRIVILEGE, ADMIN) |
327361c3cd8SBen Skeggs NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, ERROR_NOTIFIER_TYPE, NONE) |
328361c3cd8SBen Skeggs NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, ECC_ERROR_NOTIFIER_TYPE, NONE);
329361c3cd8SBen Skeggs
330361c3cd8SBen Skeggs ret = nvkm_gsp_rm_alloc_wr(&golden.chan, args);
331361c3cd8SBen Skeggs if (ret)
332361c3cd8SBen Skeggs goto done;
333361c3cd8SBen Skeggs }
334361c3cd8SBen Skeggs
335361c3cd8SBen Skeggs /* Fetch context buffer info from RM and allocate each of them here to use
336361c3cd8SBen Skeggs * during golden context init (or later as a global context buffer).
337361c3cd8SBen Skeggs *
338361c3cd8SBen Skeggs * Also build the information that'll be used to create channel contexts.
339361c3cd8SBen Skeggs */
340361c3cd8SBen Skeggs info = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice,
341361c3cd8SBen Skeggs NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_CONTEXT_BUFFERS_INFO,
342361c3cd8SBen Skeggs sizeof(*info));
343361c3cd8SBen Skeggs if (WARN_ON(IS_ERR(info))) {
344361c3cd8SBen Skeggs ret = PTR_ERR(info);
345361c3cd8SBen Skeggs goto done;
346361c3cd8SBen Skeggs }
347361c3cd8SBen Skeggs
348361c3cd8SBen Skeggs for (int i = 0; i < ARRAY_SIZE(info->engineContextBuffersInfo[0].engine); i++) {
349361c3cd8SBen Skeggs static const struct {
350361c3cd8SBen Skeggs u32 id0; /* NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID */
351361c3cd8SBen Skeggs u32 id1; /* NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID */
352361c3cd8SBen Skeggs bool global;
353361c3cd8SBen Skeggs bool init;
354361c3cd8SBen Skeggs bool ro;
355361c3cd8SBen Skeggs } map[] = {
356361c3cd8SBen Skeggs #define _A(n,N,G,I,R) { .id0 = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_##n, \
357361c3cd8SBen Skeggs .id1 = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_##N, \
358361c3cd8SBen Skeggs .global = (G), .init = (I), .ro = (R) }
359361c3cd8SBen Skeggs #define _B(N,G,I,R) _A(GRAPHICS_##N, N, (G), (I), (R))
360361c3cd8SBen Skeggs /* global init ro */
361361c3cd8SBen Skeggs _A( GRAPHICS, MAIN, false, true, false),
362361c3cd8SBen Skeggs _B( PATCH, false, true, false),
363361c3cd8SBen Skeggs _A( GRAPHICS_BUNDLE_CB, BUFFER_BUNDLE_CB, true, false, false),
364361c3cd8SBen Skeggs _B( PAGEPOOL, true, false, false),
365361c3cd8SBen Skeggs _B( ATTRIBUTE_CB, true, false, false),
366361c3cd8SBen Skeggs _B( RTV_CB_GLOBAL, true, false, false),
367361c3cd8SBen Skeggs _B( FECS_EVENT, true, true, false),
368361c3cd8SBen Skeggs _B( PRIV_ACCESS_MAP, true, true, true),
369361c3cd8SBen Skeggs #undef _B
370361c3cd8SBen Skeggs #undef _A
371361c3cd8SBen Skeggs };
372361c3cd8SBen Skeggs u32 size = info->engineContextBuffersInfo[0].engine[i].size;
373361c3cd8SBen Skeggs u8 align, page;
374361c3cd8SBen Skeggs int id;
375361c3cd8SBen Skeggs
376361c3cd8SBen Skeggs for (id = 0; id < ARRAY_SIZE(map); id++) {
377361c3cd8SBen Skeggs if (map[id].id0 == i)
378361c3cd8SBen Skeggs break;
379361c3cd8SBen Skeggs }
380361c3cd8SBen Skeggs
381361c3cd8SBen Skeggs nvkm_debug(subdev, "%02x: size:0x%08x %s\n", i,
382361c3cd8SBen Skeggs size, (id < ARRAY_SIZE(map)) ? "*" : "");
383361c3cd8SBen Skeggs if (id >= ARRAY_SIZE(map))
384361c3cd8SBen Skeggs continue;
385361c3cd8SBen Skeggs
386361c3cd8SBen Skeggs if (map[id].id1 == NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN)
387361c3cd8SBen Skeggs size = ALIGN(size, 0x1000) + 64 * 0x1000; /* per-subctx headers */
388361c3cd8SBen Skeggs
389361c3cd8SBen Skeggs if (size >= 1 << 21) page = 21;
390361c3cd8SBen Skeggs else if (size >= 1 << 16) page = 16;
391361c3cd8SBen Skeggs else page = 12;
392361c3cd8SBen Skeggs
393361c3cd8SBen Skeggs if (map[id].id1 == NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB)
394361c3cd8SBen Skeggs align = order_base_2(size);
395361c3cd8SBen Skeggs else
396361c3cd8SBen Skeggs align = page;
397361c3cd8SBen Skeggs
398361c3cd8SBen Skeggs if (WARN_ON(gr->ctxbuf_nr == ARRAY_SIZE(gr->ctxbuf)))
399361c3cd8SBen Skeggs continue;
400361c3cd8SBen Skeggs
401361c3cd8SBen Skeggs gr->ctxbuf[gr->ctxbuf_nr].bufferId = map[id].id1;
402361c3cd8SBen Skeggs gr->ctxbuf[gr->ctxbuf_nr].size = size;
403361c3cd8SBen Skeggs gr->ctxbuf[gr->ctxbuf_nr].page = page;
404361c3cd8SBen Skeggs gr->ctxbuf[gr->ctxbuf_nr].align = align;
405361c3cd8SBen Skeggs gr->ctxbuf[gr->ctxbuf_nr].global = map[id].global;
406361c3cd8SBen Skeggs gr->ctxbuf[gr->ctxbuf_nr].init = map[id].init;
407361c3cd8SBen Skeggs gr->ctxbuf[gr->ctxbuf_nr].ro = map[id].ro;
408361c3cd8SBen Skeggs gr->ctxbuf_nr++;
409361c3cd8SBen Skeggs
410361c3cd8SBen Skeggs if (map[id].id1 == NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP) {
411361c3cd8SBen Skeggs if (WARN_ON(gr->ctxbuf_nr == ARRAY_SIZE(gr->ctxbuf)))
412361c3cd8SBen Skeggs continue;
413361c3cd8SBen Skeggs
414361c3cd8SBen Skeggs gr->ctxbuf[gr->ctxbuf_nr] = gr->ctxbuf[gr->ctxbuf_nr - 1];
415361c3cd8SBen Skeggs gr->ctxbuf[gr->ctxbuf_nr].bufferId =
416361c3cd8SBen Skeggs NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP;
417361c3cd8SBen Skeggs gr->ctxbuf_nr++;
418361c3cd8SBen Skeggs }
419361c3cd8SBen Skeggs }
420361c3cd8SBen Skeggs
421361c3cd8SBen Skeggs nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, info);
422361c3cd8SBen Skeggs
423361c3cd8SBen Skeggs /* Promote golden context to RM. */
424361c3cd8SBen Skeggs ret = r535_gr_promote_ctx(gr, true, golden.vmm, gr->ctxbuf_mem, golden.vma, &golden.chan);
425361c3cd8SBen Skeggs if (ret)
426361c3cd8SBen Skeggs goto done;
427361c3cd8SBen Skeggs
428361c3cd8SBen Skeggs /* Allocate 3D class on channel to trigger golden context init in RM. */
429361c3cd8SBen Skeggs {
430361c3cd8SBen Skeggs int i;
431361c3cd8SBen Skeggs
432361c3cd8SBen Skeggs for (i = 0; gr->base.func->sclass[i].ctor; i++) {
433361c3cd8SBen Skeggs if ((gr->base.func->sclass[i].oclass & 0xff) == 0x97) {
434361c3cd8SBen Skeggs struct nvkm_gsp_object threed;
435361c3cd8SBen Skeggs
436361c3cd8SBen Skeggs ret = nvkm_gsp_rm_alloc(&golden.chan, 0x97000000,
437361c3cd8SBen Skeggs gr->base.func->sclass[i].oclass, 0,
438361c3cd8SBen Skeggs &threed);
439361c3cd8SBen Skeggs if (ret)
440361c3cd8SBen Skeggs goto done;
441361c3cd8SBen Skeggs
442361c3cd8SBen Skeggs nvkm_gsp_rm_free(&threed);
443361c3cd8SBen Skeggs break;
444361c3cd8SBen Skeggs }
445361c3cd8SBen Skeggs }
446361c3cd8SBen Skeggs
447361c3cd8SBen Skeggs if (WARN_ON(!gr->base.func->sclass[i].ctor)) {
448361c3cd8SBen Skeggs ret = -EINVAL;
449361c3cd8SBen Skeggs goto done;
450361c3cd8SBen Skeggs }
451361c3cd8SBen Skeggs }
452361c3cd8SBen Skeggs
453361c3cd8SBen Skeggs done:
454361c3cd8SBen Skeggs nvkm_gsp_rm_free(&golden.chan);
455361c3cd8SBen Skeggs for (int i = gr->ctxbuf_nr - 1; i >= 0; i--)
456361c3cd8SBen Skeggs nvkm_vmm_put(golden.vmm, &golden.vma[i]);
457361c3cd8SBen Skeggs nvkm_vmm_unref(&golden.vmm);
458361c3cd8SBen Skeggs nvkm_memory_unref(&golden.inst);
459361c3cd8SBen Skeggs return ret;
460361c3cd8SBen Skeggs
461361c3cd8SBen Skeggs }
462361c3cd8SBen Skeggs
463361c3cd8SBen Skeggs static void *
r535_gr_dtor(struct nvkm_gr * base)464361c3cd8SBen Skeggs r535_gr_dtor(struct nvkm_gr *base)
465361c3cd8SBen Skeggs {
466361c3cd8SBen Skeggs struct r535_gr *gr = r535_gr(base);
467361c3cd8SBen Skeggs
468361c3cd8SBen Skeggs while (gr->ctxbuf_nr)
469361c3cd8SBen Skeggs nvkm_memory_unref(&gr->ctxbuf_mem[--gr->ctxbuf_nr]);
470361c3cd8SBen Skeggs
471361c3cd8SBen Skeggs kfree(gr->base.func);
472361c3cd8SBen Skeggs return gr;
473361c3cd8SBen Skeggs }
474361c3cd8SBen Skeggs
475361c3cd8SBen Skeggs int
r535_gr_new(const struct gf100_gr_func * hw,struct nvkm_device * device,enum nvkm_subdev_type type,int inst,struct nvkm_gr ** pgr)476361c3cd8SBen Skeggs r535_gr_new(const struct gf100_gr_func *hw,
477361c3cd8SBen Skeggs struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_gr **pgr)
478361c3cd8SBen Skeggs {
479361c3cd8SBen Skeggs struct nvkm_gr_func *rm;
480361c3cd8SBen Skeggs struct r535_gr *gr;
481361c3cd8SBen Skeggs int nclass;
482361c3cd8SBen Skeggs
483361c3cd8SBen Skeggs for (nclass = 0; hw->sclass[nclass].oclass; nclass++);
484361c3cd8SBen Skeggs
485361c3cd8SBen Skeggs if (!(rm = kzalloc(sizeof(*rm) + (nclass + 1) * sizeof(rm->sclass[0]), GFP_KERNEL)))
486361c3cd8SBen Skeggs return -ENOMEM;
487361c3cd8SBen Skeggs
488361c3cd8SBen Skeggs rm->dtor = r535_gr_dtor;
489361c3cd8SBen Skeggs rm->oneinit = r535_gr_oneinit;
490361c3cd8SBen Skeggs rm->units = r535_gr_units;
491361c3cd8SBen Skeggs rm->chan_new = r535_gr_chan_new;
492361c3cd8SBen Skeggs
493361c3cd8SBen Skeggs for (int i = 0; i < nclass; i++) {
494361c3cd8SBen Skeggs rm->sclass[i].minver = hw->sclass[i].minver;
495361c3cd8SBen Skeggs rm->sclass[i].maxver = hw->sclass[i].maxver;
496361c3cd8SBen Skeggs rm->sclass[i].oclass = hw->sclass[i].oclass;
497361c3cd8SBen Skeggs rm->sclass[i].ctor = r535_gr_obj_ctor;
498361c3cd8SBen Skeggs }
499361c3cd8SBen Skeggs
500361c3cd8SBen Skeggs if (!(gr = kzalloc(sizeof(*gr), GFP_KERNEL))) {
501361c3cd8SBen Skeggs kfree(rm);
502361c3cd8SBen Skeggs return -ENOMEM;
503361c3cd8SBen Skeggs }
504361c3cd8SBen Skeggs
505361c3cd8SBen Skeggs *pgr = &gr->base;
506361c3cd8SBen Skeggs
507361c3cd8SBen Skeggs return nvkm_gr_ctor(rm, device, type, inst, true, &gr->base);
508361c3cd8SBen Skeggs }
509