1 /*
2 * Copyright 2023 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22 #include "gf100.h"
23
24 #include <core/memory.h>
25 #include <subdev/gsp.h>
26 #include <subdev/mmu/vmm.h>
27 #include <engine/fifo/priv.h>
28
29 #include <nvif/if900d.h>
30
31 #include <nvhw/drf.h>
32
33 #include <nvrm/nvtypes.h>
34 #include <nvrm/535.113.01/common/sdk/nvidia/inc/alloc/alloc_channel.h>
35 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080fifo.h>
36 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h>
37 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080internal.h>
38 #include <nvrm/535.113.01/nvidia/generated/g_kernel_channel_nvoc.h>
39
40 #define r535_gr(p) container_of((p), struct r535_gr, base)
41
42 #define R515_GR_MAX_CTXBUFS 9
43
44 struct r535_gr {
45 struct nvkm_gr base;
46
47 struct {
48 u16 bufferId;
49 u32 size;
50 u8 page;
51 u8 align;
52 bool global;
53 bool init;
54 bool ro;
55 } ctxbuf[R515_GR_MAX_CTXBUFS];
56 int ctxbuf_nr;
57
58 struct nvkm_memory *ctxbuf_mem[R515_GR_MAX_CTXBUFS];
59 };
60
61 struct r535_gr_chan {
62 struct nvkm_object object;
63 struct r535_gr *gr;
64
65 struct nvkm_vmm *vmm;
66 struct nvkm_chan *chan;
67
68 struct nvkm_memory *mem[R515_GR_MAX_CTXBUFS];
69 struct nvkm_vma *vma[R515_GR_MAX_CTXBUFS];
70 };
71
72 struct r535_gr_obj {
73 struct nvkm_object object;
74 struct nvkm_gsp_object rm;
75 };
76
77 static void *
r535_gr_obj_dtor(struct nvkm_object * object)78 r535_gr_obj_dtor(struct nvkm_object *object)
79 {
80 struct r535_gr_obj *obj = container_of(object, typeof(*obj), object);
81
82 nvkm_gsp_rm_free(&obj->rm);
83 return obj;
84 }
85
86 static const struct nvkm_object_func
87 r535_gr_obj = {
88 .dtor = r535_gr_obj_dtor,
89 };
90
91 static int
r535_gr_obj_ctor(const struct nvkm_oclass * oclass,void * argv,u32 argc,struct nvkm_object ** pobject)92 r535_gr_obj_ctor(const struct nvkm_oclass *oclass, void *argv, u32 argc,
93 struct nvkm_object **pobject)
94 {
95 struct r535_gr_chan *chan = container_of(oclass->parent, typeof(*chan), object);
96 struct r535_gr_obj *obj;
97
98 if (!(obj = kzalloc(sizeof(*obj), GFP_KERNEL)))
99 return -ENOMEM;
100
101 nvkm_object_ctor(&r535_gr_obj, oclass, &obj->object);
102 *pobject = &obj->object;
103
104 return nvkm_gsp_rm_alloc(&chan->chan->rm.object, oclass->handle, oclass->base.oclass, 0,
105 &obj->rm);
106 }
107
108 static void *
r535_gr_chan_dtor(struct nvkm_object * object)109 r535_gr_chan_dtor(struct nvkm_object *object)
110 {
111 struct r535_gr_chan *grc = container_of(object, typeof(*grc), object);
112 struct r535_gr *gr = grc->gr;
113
114 for (int i = 0; i < gr->ctxbuf_nr; i++) {
115 nvkm_vmm_put(grc->vmm, &grc->vma[i]);
116 nvkm_memory_unref(&grc->mem[i]);
117 }
118
119 nvkm_vmm_unref(&grc->vmm);
120 return grc;
121 }
122
123 static const struct nvkm_object_func
124 r535_gr_chan = {
125 .dtor = r535_gr_chan_dtor,
126 };
127
128 static int
r535_gr_promote_ctx(struct r535_gr * gr,bool golden,struct nvkm_vmm * vmm,struct nvkm_memory ** pmem,struct nvkm_vma ** pvma,struct nvkm_gsp_object * chan)129 r535_gr_promote_ctx(struct r535_gr *gr, bool golden, struct nvkm_vmm *vmm,
130 struct nvkm_memory **pmem, struct nvkm_vma **pvma,
131 struct nvkm_gsp_object *chan)
132 {
133 struct nvkm_subdev *subdev = &gr->base.engine.subdev;
134 struct nvkm_device *device = subdev->device;
135 NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS *ctrl;
136
137 ctrl = nvkm_gsp_rm_ctrl_get(&vmm->rm.device.subdevice,
138 NV2080_CTRL_CMD_GPU_PROMOTE_CTX, sizeof(*ctrl));
139 if (WARN_ON(IS_ERR(ctrl)))
140 return PTR_ERR(ctrl);
141
142 ctrl->engineType = 1;
143 ctrl->hChanClient = vmm->rm.client.object.handle;
144 ctrl->hObject = chan->handle;
145
146 for (int i = 0; i < gr->ctxbuf_nr; i++) {
147 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY *entry =
148 &ctrl->promoteEntry[ctrl->entryCount];
149 const bool alloc = golden || !gr->ctxbuf[i].global;
150 int ret;
151
152 entry->bufferId = gr->ctxbuf[i].bufferId;
153 entry->bInitialize = gr->ctxbuf[i].init && alloc;
154
155 if (alloc) {
156 ret = nvkm_memory_new(device, gr->ctxbuf[i].init ?
157 NVKM_MEM_TARGET_INST : NVKM_MEM_TARGET_INST_SR_LOST,
158 gr->ctxbuf[i].size, 1 << gr->ctxbuf[i].page,
159 gr->ctxbuf[i].init, &pmem[i]);
160 if (WARN_ON(ret))
161 return ret;
162
163 if (gr->ctxbuf[i].bufferId ==
164 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP)
165 entry->bNonmapped = 1;
166 } else {
167 if (gr->ctxbuf[i].bufferId ==
168 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP)
169 continue;
170
171 pmem[i] = nvkm_memory_ref(gr->ctxbuf_mem[i]);
172 }
173
174 if (!entry->bNonmapped) {
175 struct gf100_vmm_map_v0 args = {
176 .priv = 1,
177 .ro = gr->ctxbuf[i].ro,
178 };
179
180 mutex_lock(&vmm->mutex.vmm);
181 ret = nvkm_vmm_get_locked(vmm, false, true, false, 0, gr->ctxbuf[i].align,
182 nvkm_memory_size(pmem[i]), &pvma[i]);
183 mutex_unlock(&vmm->mutex.vmm);
184 if (ret)
185 return ret;
186
187 ret = nvkm_memory_map(pmem[i], 0, vmm, pvma[i], &args, sizeof(args));
188 if (ret)
189 return ret;
190
191 entry->gpuVirtAddr = pvma[i]->addr;
192 }
193
194 if (entry->bInitialize) {
195 entry->gpuPhysAddr = nvkm_memory_addr(pmem[i]);
196 entry->size = gr->ctxbuf[i].size;
197 entry->physAttr = 4;
198 }
199
200 nvkm_debug(subdev,
201 "promote %02d: pa %016llx/%08x sz %016llx va %016llx init:%d nm:%d\n",
202 entry->bufferId, entry->gpuPhysAddr, entry->physAttr, entry->size,
203 entry->gpuVirtAddr, entry->bInitialize, entry->bNonmapped);
204
205 ctrl->entryCount++;
206 }
207
208 return nvkm_gsp_rm_ctrl_wr(&vmm->rm.device.subdevice, ctrl);
209 }
210
211 static int
r535_gr_chan_new(struct nvkm_gr * base,struct nvkm_chan * chan,const struct nvkm_oclass * oclass,struct nvkm_object ** pobject)212 r535_gr_chan_new(struct nvkm_gr *base, struct nvkm_chan *chan, const struct nvkm_oclass *oclass,
213 struct nvkm_object **pobject)
214 {
215 struct r535_gr *gr = r535_gr(base);
216 struct r535_gr_chan *grc;
217 int ret;
218
219 if (!(grc = kzalloc(sizeof(*grc), GFP_KERNEL)))
220 return -ENOMEM;
221
222 nvkm_object_ctor(&r535_gr_chan, oclass, &grc->object);
223 grc->gr = gr;
224 grc->vmm = nvkm_vmm_ref(chan->vmm);
225 grc->chan = chan;
226 *pobject = &grc->object;
227
228 ret = r535_gr_promote_ctx(gr, false, grc->vmm, grc->mem, grc->vma, &chan->rm.object);
229 if (ret)
230 return ret;
231
232 return 0;
233 }
234
235 static u64
r535_gr_units(struct nvkm_gr * gr)236 r535_gr_units(struct nvkm_gr *gr)
237 {
238 struct nvkm_gsp *gsp = gr->engine.subdev.device->gsp;
239
240 return (gsp->gr.tpcs << 8) | gsp->gr.gpcs;
241 }
242
243 static int
r535_gr_oneinit(struct nvkm_gr * base)244 r535_gr_oneinit(struct nvkm_gr *base)
245 {
246 NV2080_CTRL_INTERNAL_STATIC_GR_GET_CONTEXT_BUFFERS_INFO_PARAMS *info;
247 struct r535_gr *gr = container_of(base, typeof(*gr), base);
248 struct nvkm_subdev *subdev = &gr->base.engine.subdev;
249 struct nvkm_device *device = subdev->device;
250 struct nvkm_gsp *gsp = device->gsp;
251 struct nvkm_mmu *mmu = device->mmu;
252 struct {
253 struct nvkm_memory *inst;
254 struct nvkm_vmm *vmm;
255 struct nvkm_gsp_object chan;
256 struct nvkm_vma *vma[R515_GR_MAX_CTXBUFS];
257 } golden = {};
258 int ret;
259
260 /* Allocate a channel to use for golden context init. */
261 ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x12000, 0, true, &golden.inst);
262 if (ret)
263 goto done;
264
265 ret = nvkm_vmm_new(device, 0x1000, 0, NULL, 0, NULL, "grGoldenVmm", &golden.vmm);
266 if (ret)
267 goto done;
268
269 ret = mmu->func->promote_vmm(golden.vmm);
270 if (ret)
271 goto done;
272
273 {
274 NV_CHANNELGPFIFO_ALLOCATION_PARAMETERS *args;
275
276 args = nvkm_gsp_rm_alloc_get(&golden.vmm->rm.device.object, 0xf1f00000,
277 device->fifo->func->chan.user.oclass,
278 sizeof(*args), &golden.chan);
279 if (IS_ERR(args)) {
280 ret = PTR_ERR(args);
281 goto done;
282 }
283
284 args->gpFifoOffset = 0;
285 args->gpFifoEntries = 0x1000 / 8;
286 args->flags =
287 NVDEF(NVOS04, FLAGS, CHANNEL_TYPE, PHYSICAL) |
288 NVDEF(NVOS04, FLAGS, VPR, FALSE) |
289 NVDEF(NVOS04, FLAGS, CHANNEL_SKIP_MAP_REFCOUNTING, FALSE) |
290 NVVAL(NVOS04, FLAGS, GROUP_CHANNEL_RUNQUEUE, 0) |
291 NVDEF(NVOS04, FLAGS, PRIVILEGED_CHANNEL, TRUE) |
292 NVDEF(NVOS04, FLAGS, DELAY_CHANNEL_SCHEDULING, FALSE) |
293 NVDEF(NVOS04, FLAGS, CHANNEL_DENY_PHYSICAL_MODE_CE, FALSE) |
294 NVVAL(NVOS04, FLAGS, CHANNEL_USERD_INDEX_VALUE, 0) |
295 NVDEF(NVOS04, FLAGS, CHANNEL_USERD_INDEX_FIXED, FALSE) |
296 NVVAL(NVOS04, FLAGS, CHANNEL_USERD_INDEX_PAGE_VALUE, 0) |
297 NVDEF(NVOS04, FLAGS, CHANNEL_USERD_INDEX_PAGE_FIXED, TRUE) |
298 NVDEF(NVOS04, FLAGS, CHANNEL_DENY_AUTH_LEVEL_PRIV, FALSE) |
299 NVDEF(NVOS04, FLAGS, CHANNEL_SKIP_SCRUBBER, FALSE) |
300 NVDEF(NVOS04, FLAGS, CHANNEL_CLIENT_MAP_FIFO, FALSE) |
301 NVDEF(NVOS04, FLAGS, SET_EVICT_LAST_CE_PREFETCH_CHANNEL, FALSE) |
302 NVDEF(NVOS04, FLAGS, CHANNEL_VGPU_PLUGIN_CONTEXT, FALSE) |
303 NVDEF(NVOS04, FLAGS, CHANNEL_PBDMA_ACQUIRE_TIMEOUT, FALSE) |
304 NVDEF(NVOS04, FLAGS, GROUP_CHANNEL_THREAD, DEFAULT) |
305 NVDEF(NVOS04, FLAGS, MAP_CHANNEL, FALSE) |
306 NVDEF(NVOS04, FLAGS, SKIP_CTXBUFFER_ALLOC, FALSE);
307 args->hVASpace = golden.vmm->rm.object.handle;
308 args->engineType = 1;
309 args->instanceMem.base = nvkm_memory_addr(golden.inst);
310 args->instanceMem.size = 0x1000;
311 args->instanceMem.addressSpace = 2;
312 args->instanceMem.cacheAttrib = 1;
313 args->ramfcMem.base = nvkm_memory_addr(golden.inst);
314 args->ramfcMem.size = 0x200;
315 args->ramfcMem.addressSpace = 2;
316 args->ramfcMem.cacheAttrib = 1;
317 args->userdMem.base = nvkm_memory_addr(golden.inst) + 0x1000;
318 args->userdMem.size = 0x200;
319 args->userdMem.addressSpace = 2;
320 args->userdMem.cacheAttrib = 1;
321 args->mthdbufMem.base = nvkm_memory_addr(golden.inst) + 0x2000;
322 args->mthdbufMem.size = 0x5000;
323 args->mthdbufMem.addressSpace = 2;
324 args->mthdbufMem.cacheAttrib = 1;
325 args->internalFlags =
326 NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, PRIVILEGE, ADMIN) |
327 NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, ERROR_NOTIFIER_TYPE, NONE) |
328 NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, ECC_ERROR_NOTIFIER_TYPE, NONE);
329
330 ret = nvkm_gsp_rm_alloc_wr(&golden.chan, args);
331 if (ret)
332 goto done;
333 }
334
335 /* Fetch context buffer info from RM and allocate each of them here to use
336 * during golden context init (or later as a global context buffer).
337 *
338 * Also build the information that'll be used to create channel contexts.
339 */
340 info = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice,
341 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_CONTEXT_BUFFERS_INFO,
342 sizeof(*info));
343 if (WARN_ON(IS_ERR(info))) {
344 ret = PTR_ERR(info);
345 goto done;
346 }
347
348 for (int i = 0; i < ARRAY_SIZE(info->engineContextBuffersInfo[0].engine); i++) {
349 static const struct {
350 u32 id0; /* NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID */
351 u32 id1; /* NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID */
352 bool global;
353 bool init;
354 bool ro;
355 } map[] = {
356 #define _A(n,N,G,I,R) { .id0 = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_##n, \
357 .id1 = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_##N, \
358 .global = (G), .init = (I), .ro = (R) }
359 #define _B(N,G,I,R) _A(GRAPHICS_##N, N, (G), (I), (R))
360 /* global init ro */
361 _A( GRAPHICS, MAIN, false, true, false),
362 _B( PATCH, false, true, false),
363 _A( GRAPHICS_BUNDLE_CB, BUFFER_BUNDLE_CB, true, false, false),
364 _B( PAGEPOOL, true, false, false),
365 _B( ATTRIBUTE_CB, true, false, false),
366 _B( RTV_CB_GLOBAL, true, false, false),
367 _B( FECS_EVENT, true, true, false),
368 _B( PRIV_ACCESS_MAP, true, true, true),
369 #undef _B
370 #undef _A
371 };
372 u32 size = info->engineContextBuffersInfo[0].engine[i].size;
373 u8 align, page;
374 int id;
375
376 for (id = 0; id < ARRAY_SIZE(map); id++) {
377 if (map[id].id0 == i)
378 break;
379 }
380
381 nvkm_debug(subdev, "%02x: size:0x%08x %s\n", i,
382 size, (id < ARRAY_SIZE(map)) ? "*" : "");
383 if (id >= ARRAY_SIZE(map))
384 continue;
385
386 if (map[id].id1 == NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN)
387 size = ALIGN(size, 0x1000) + 64 * 0x1000; /* per-subctx headers */
388
389 if (size >= 1 << 21) page = 21;
390 else if (size >= 1 << 16) page = 16;
391 else page = 12;
392
393 if (map[id].id1 == NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB)
394 align = order_base_2(size);
395 else
396 align = page;
397
398 if (WARN_ON(gr->ctxbuf_nr == ARRAY_SIZE(gr->ctxbuf)))
399 continue;
400
401 gr->ctxbuf[gr->ctxbuf_nr].bufferId = map[id].id1;
402 gr->ctxbuf[gr->ctxbuf_nr].size = size;
403 gr->ctxbuf[gr->ctxbuf_nr].page = page;
404 gr->ctxbuf[gr->ctxbuf_nr].align = align;
405 gr->ctxbuf[gr->ctxbuf_nr].global = map[id].global;
406 gr->ctxbuf[gr->ctxbuf_nr].init = map[id].init;
407 gr->ctxbuf[gr->ctxbuf_nr].ro = map[id].ro;
408 gr->ctxbuf_nr++;
409
410 if (map[id].id1 == NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP) {
411 if (WARN_ON(gr->ctxbuf_nr == ARRAY_SIZE(gr->ctxbuf)))
412 continue;
413
414 gr->ctxbuf[gr->ctxbuf_nr] = gr->ctxbuf[gr->ctxbuf_nr - 1];
415 gr->ctxbuf[gr->ctxbuf_nr].bufferId =
416 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP;
417 gr->ctxbuf_nr++;
418 }
419 }
420
421 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, info);
422
423 /* Promote golden context to RM. */
424 ret = r535_gr_promote_ctx(gr, true, golden.vmm, gr->ctxbuf_mem, golden.vma, &golden.chan);
425 if (ret)
426 goto done;
427
428 /* Allocate 3D class on channel to trigger golden context init in RM. */
429 {
430 int i;
431
432 for (i = 0; gr->base.func->sclass[i].ctor; i++) {
433 if ((gr->base.func->sclass[i].oclass & 0xff) == 0x97) {
434 struct nvkm_gsp_object threed;
435
436 ret = nvkm_gsp_rm_alloc(&golden.chan, 0x97000000,
437 gr->base.func->sclass[i].oclass, 0,
438 &threed);
439 if (ret)
440 goto done;
441
442 nvkm_gsp_rm_free(&threed);
443 break;
444 }
445 }
446
447 if (WARN_ON(!gr->base.func->sclass[i].ctor)) {
448 ret = -EINVAL;
449 goto done;
450 }
451 }
452
453 done:
454 nvkm_gsp_rm_free(&golden.chan);
455 for (int i = gr->ctxbuf_nr - 1; i >= 0; i--)
456 nvkm_vmm_put(golden.vmm, &golden.vma[i]);
457 nvkm_vmm_unref(&golden.vmm);
458 nvkm_memory_unref(&golden.inst);
459 return ret;
460
461 }
462
463 static void *
r535_gr_dtor(struct nvkm_gr * base)464 r535_gr_dtor(struct nvkm_gr *base)
465 {
466 struct r535_gr *gr = r535_gr(base);
467
468 while (gr->ctxbuf_nr)
469 nvkm_memory_unref(&gr->ctxbuf_mem[--gr->ctxbuf_nr]);
470
471 kfree(gr->base.func);
472 return gr;
473 }
474
475 int
r535_gr_new(const struct gf100_gr_func * hw,struct nvkm_device * device,enum nvkm_subdev_type type,int inst,struct nvkm_gr ** pgr)476 r535_gr_new(const struct gf100_gr_func *hw,
477 struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_gr **pgr)
478 {
479 struct nvkm_gr_func *rm;
480 struct r535_gr *gr;
481 int nclass;
482
483 for (nclass = 0; hw->sclass[nclass].oclass; nclass++);
484
485 if (!(rm = kzalloc(sizeof(*rm) + (nclass + 1) * sizeof(rm->sclass[0]), GFP_KERNEL)))
486 return -ENOMEM;
487
488 rm->dtor = r535_gr_dtor;
489 rm->oneinit = r535_gr_oneinit;
490 rm->units = r535_gr_units;
491 rm->chan_new = r535_gr_chan_new;
492
493 for (int i = 0; i < nclass; i++) {
494 rm->sclass[i].minver = hw->sclass[i].minver;
495 rm->sclass[i].maxver = hw->sclass[i].maxver;
496 rm->sclass[i].oclass = hw->sclass[i].oclass;
497 rm->sclass[i].ctor = r535_gr_obj_ctor;
498 }
499
500 if (!(gr = kzalloc(sizeof(*gr), GFP_KERNEL))) {
501 kfree(rm);
502 return -ENOMEM;
503 }
504
505 *pgr = &gr->base;
506
507 return nvkm_gr_ctor(rm, device, type, inst, true, &gr->base);
508 }
509