1 /*
2 * Copyright 2023 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22 #include "priv.h"
23 #include "cgrp.h"
24 #include "chan.h"
25 #include "chid.h"
26 #include "runl.h"
27
28 #include <core/gpuobj.h>
29 #include <subdev/gsp.h>
30 #include <subdev/mmu.h>
31 #include <subdev/vfn.h>
32 #include <engine/gr.h>
33
34 #include <nvhw/drf.h>
35
36 #include <nvrm/nvtypes.h>
37 #include <nvrm/535.113.01/common/sdk/nvidia/inc/alloc/alloc_channel.h>
38 #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl2080_notification.h>
39 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080ce.h>
40 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080fifo.h>
41 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h>
42 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080internal.h>
43 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrla06f/ctrla06fgpfifo.h>
44 #include <nvrm/535.113.01/nvidia/generated/g_kernel_channel_nvoc.h>
45 #include <nvrm/535.113.01/nvidia/generated/g_kernel_fifo_nvoc.h>
46 #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gpu_engine_type.h>
47
48 static u32
r535_chan_doorbell_handle(struct nvkm_chan * chan)49 r535_chan_doorbell_handle(struct nvkm_chan *chan)
50 {
51 return (chan->cgrp->runl->id << 16) | chan->id;
52 }
53
54 static void
r535_chan_stop(struct nvkm_chan * chan)55 r535_chan_stop(struct nvkm_chan *chan)
56 {
57 }
58
59 static void
r535_chan_start(struct nvkm_chan * chan)60 r535_chan_start(struct nvkm_chan *chan)
61 {
62 }
63
64 static void
r535_chan_ramfc_clear(struct nvkm_chan * chan)65 r535_chan_ramfc_clear(struct nvkm_chan *chan)
66 {
67 struct nvkm_fifo *fifo = chan->cgrp->runl->fifo;
68
69 nvkm_gsp_rm_free(&chan->rm.object);
70
71 dma_free_coherent(fifo->engine.subdev.device->dev, fifo->rm.mthdbuf_size,
72 chan->rm.mthdbuf.ptr, chan->rm.mthdbuf.addr);
73
74 nvkm_cgrp_vctx_put(chan->cgrp, &chan->rm.grctx);
75 }
76
77 #define CHID_PER_USERD 8
78
79 static int
r535_chan_ramfc_write(struct nvkm_chan * chan,u64 offset,u64 length,u32 devm,bool priv)80 r535_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv)
81 {
82 struct nvkm_fifo *fifo = chan->cgrp->runl->fifo;
83 struct nvkm_engn *engn;
84 struct nvkm_device *device = fifo->engine.subdev.device;
85 NV_CHANNELGPFIFO_ALLOCATION_PARAMETERS *args;
86 const int userd_p = chan->id / CHID_PER_USERD;
87 const int userd_i = chan->id % CHID_PER_USERD;
88 u32 eT = ~0;
89 int ret;
90
91 if (unlikely(device->gr && !device->gr->engine.subdev.oneinit)) {
92 ret = nvkm_subdev_oneinit(&device->gr->engine.subdev);
93 if (ret)
94 return ret;
95 }
96
97 nvkm_runl_foreach_engn(engn, chan->cgrp->runl) {
98 eT = engn->id;
99 break;
100 }
101
102 if (WARN_ON(eT == ~0))
103 return -EINVAL;
104
105 chan->rm.mthdbuf.ptr = dma_alloc_coherent(fifo->engine.subdev.device->dev,
106 fifo->rm.mthdbuf_size,
107 &chan->rm.mthdbuf.addr, GFP_KERNEL);
108 if (!chan->rm.mthdbuf.ptr)
109 return -ENOMEM;
110
111 args = nvkm_gsp_rm_alloc_get(&chan->vmm->rm.device.object, 0xf1f00000 | chan->id,
112 fifo->func->chan.user.oclass, sizeof(*args),
113 &chan->rm.object);
114 if (WARN_ON(IS_ERR(args)))
115 return PTR_ERR(args);
116
117 args->gpFifoOffset = offset;
118 args->gpFifoEntries = length / 8;
119
120 args->flags = NVDEF(NVOS04, FLAGS, CHANNEL_TYPE, PHYSICAL);
121 args->flags |= NVDEF(NVOS04, FLAGS, VPR, FALSE);
122 args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_SKIP_MAP_REFCOUNTING, FALSE);
123 args->flags |= NVVAL(NVOS04, FLAGS, GROUP_CHANNEL_RUNQUEUE, chan->runq);
124 if (!priv)
125 args->flags |= NVDEF(NVOS04, FLAGS, PRIVILEGED_CHANNEL, FALSE);
126 else
127 args->flags |= NVDEF(NVOS04, FLAGS, PRIVILEGED_CHANNEL, TRUE);
128 args->flags |= NVDEF(NVOS04, FLAGS, DELAY_CHANNEL_SCHEDULING, FALSE);
129 args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_DENY_PHYSICAL_MODE_CE, FALSE);
130
131 args->flags |= NVVAL(NVOS04, FLAGS, CHANNEL_USERD_INDEX_VALUE, userd_i);
132 args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_USERD_INDEX_FIXED, FALSE);
133 args->flags |= NVVAL(NVOS04, FLAGS, CHANNEL_USERD_INDEX_PAGE_VALUE, userd_p);
134 args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_USERD_INDEX_PAGE_FIXED, TRUE);
135
136 args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_DENY_AUTH_LEVEL_PRIV, FALSE);
137 args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_SKIP_SCRUBBER, FALSE);
138 args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_CLIENT_MAP_FIFO, FALSE);
139 args->flags |= NVDEF(NVOS04, FLAGS, SET_EVICT_LAST_CE_PREFETCH_CHANNEL, FALSE);
140 args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_VGPU_PLUGIN_CONTEXT, FALSE);
141 args->flags |= NVDEF(NVOS04, FLAGS, CHANNEL_PBDMA_ACQUIRE_TIMEOUT, FALSE);
142 args->flags |= NVDEF(NVOS04, FLAGS, GROUP_CHANNEL_THREAD, DEFAULT);
143 args->flags |= NVDEF(NVOS04, FLAGS, MAP_CHANNEL, FALSE);
144 args->flags |= NVDEF(NVOS04, FLAGS, SKIP_CTXBUFFER_ALLOC, FALSE);
145
146 args->hVASpace = chan->vmm->rm.object.handle;
147 args->engineType = eT;
148
149 args->instanceMem.base = chan->inst->addr;
150 args->instanceMem.size = chan->inst->size;
151 args->instanceMem.addressSpace = 2;
152 args->instanceMem.cacheAttrib = 1;
153
154 args->userdMem.base = nvkm_memory_addr(chan->userd.mem) + chan->userd.base;
155 args->userdMem.size = fifo->func->chan.func->userd->size;
156 args->userdMem.addressSpace = 2;
157 args->userdMem.cacheAttrib = 1;
158
159 args->ramfcMem.base = chan->inst->addr + 0;
160 args->ramfcMem.size = 0x200;
161 args->ramfcMem.addressSpace = 2;
162 args->ramfcMem.cacheAttrib = 1;
163
164 args->mthdbufMem.base = chan->rm.mthdbuf.addr;
165 args->mthdbufMem.size = fifo->rm.mthdbuf_size;
166 args->mthdbufMem.addressSpace = 1;
167 args->mthdbufMem.cacheAttrib = 0;
168
169 if (!priv)
170 args->internalFlags = NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, PRIVILEGE, USER);
171 else
172 args->internalFlags = NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, PRIVILEGE, ADMIN);
173 args->internalFlags |= NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, ERROR_NOTIFIER_TYPE, NONE);
174 args->internalFlags |= NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, ECC_ERROR_NOTIFIER_TYPE, NONE);
175
176 ret = nvkm_gsp_rm_alloc_wr(&chan->rm.object, args);
177 if (ret)
178 return ret;
179
180 if (1) {
181 NVA06F_CTRL_GPFIFO_SCHEDULE_PARAMS *ctrl;
182
183 if (1) {
184 NVA06F_CTRL_BIND_PARAMS *ctrl;
185
186 ctrl = nvkm_gsp_rm_ctrl_get(&chan->rm.object,
187 NVA06F_CTRL_CMD_BIND, sizeof(*ctrl));
188 if (WARN_ON(IS_ERR(ctrl)))
189 return PTR_ERR(ctrl);
190
191 ctrl->engineType = eT;
192
193 ret = nvkm_gsp_rm_ctrl_wr(&chan->rm.object, ctrl);
194 if (ret)
195 return ret;
196 }
197
198 ctrl = nvkm_gsp_rm_ctrl_get(&chan->rm.object,
199 NVA06F_CTRL_CMD_GPFIFO_SCHEDULE, sizeof(*ctrl));
200 if (WARN_ON(IS_ERR(ctrl)))
201 return PTR_ERR(ctrl);
202
203 ctrl->bEnable = 1;
204 ret = nvkm_gsp_rm_ctrl_wr(&chan->rm.object, ctrl);
205 }
206
207 return ret;
208 }
209
210 static const struct nvkm_chan_func_ramfc
211 r535_chan_ramfc = {
212 .write = r535_chan_ramfc_write,
213 .clear = r535_chan_ramfc_clear,
214 .devm = 0xfff,
215 .priv = true,
216 };
217
218 struct r535_chan_userd {
219 struct nvkm_memory *mem;
220 struct nvkm_memory *map;
221 int chid;
222 u32 used;
223
224 struct list_head head;
225 } *userd;
226
227 static void
r535_chan_id_put(struct nvkm_chan * chan)228 r535_chan_id_put(struct nvkm_chan *chan)
229 {
230 struct nvkm_runl *runl = chan->cgrp->runl;
231 struct nvkm_fifo *fifo = runl->fifo;
232 struct r535_chan_userd *userd;
233
234 mutex_lock(&fifo->userd.mutex);
235 list_for_each_entry(userd, &fifo->userd.list, head) {
236 if (userd->map == chan->userd.mem) {
237 u32 chid = chan->userd.base / chan->func->userd->size;
238
239 userd->used &= ~BIT(chid);
240 if (!userd->used) {
241 nvkm_memory_unref(&userd->map);
242 nvkm_memory_unref(&userd->mem);
243 nvkm_chid_put(runl->chid, userd->chid, &chan->cgrp->lock);
244 list_del(&userd->head);
245 kfree(userd);
246 }
247
248 break;
249 }
250 }
251 mutex_unlock(&fifo->userd.mutex);
252
253 }
254
255 static int
r535_chan_id_get_locked(struct nvkm_chan * chan,struct nvkm_memory * muserd,u64 ouserd)256 r535_chan_id_get_locked(struct nvkm_chan *chan, struct nvkm_memory *muserd, u64 ouserd)
257 {
258 const u32 userd_size = CHID_PER_USERD * chan->func->userd->size;
259 struct nvkm_runl *runl = chan->cgrp->runl;
260 struct nvkm_fifo *fifo = runl->fifo;
261 struct r535_chan_userd *userd;
262 u32 chid;
263 int ret;
264
265 if (ouserd + chan->func->userd->size >= userd_size ||
266 (ouserd & (chan->func->userd->size - 1))) {
267 RUNL_DEBUG(runl, "ouserd %llx", ouserd);
268 return -EINVAL;
269 }
270
271 chid = div_u64(ouserd, chan->func->userd->size);
272
273 list_for_each_entry(userd, &fifo->userd.list, head) {
274 if (userd->mem == muserd) {
275 if (userd->used & BIT(chid))
276 return -EBUSY;
277 break;
278 }
279 }
280
281 if (&userd->head == &fifo->userd.list) {
282 if (nvkm_memory_size(muserd) < userd_size) {
283 RUNL_DEBUG(runl, "userd too small");
284 return -EINVAL;
285 }
286
287 userd = kzalloc(sizeof(*userd), GFP_KERNEL);
288 if (!userd)
289 return -ENOMEM;
290
291 userd->chid = nvkm_chid_get(runl->chid, chan);
292 if (userd->chid < 0) {
293 ret = userd->chid;
294 kfree(userd);
295 return ret;
296 }
297
298 userd->mem = nvkm_memory_ref(muserd);
299
300 ret = nvkm_memory_kmap(userd->mem, &userd->map);
301 if (ret) {
302 nvkm_chid_put(runl->chid, userd->chid, &chan->cgrp->lock);
303 kfree(userd);
304 return ret;
305 }
306
307
308 list_add(&userd->head, &fifo->userd.list);
309 }
310
311 userd->used |= BIT(chid);
312
313 chan->userd.mem = nvkm_memory_ref(userd->map);
314 chan->userd.base = ouserd;
315
316 return (userd->chid * CHID_PER_USERD) + chid;
317 }
318
319 static int
r535_chan_id_get(struct nvkm_chan * chan,struct nvkm_memory * muserd,u64 ouserd)320 r535_chan_id_get(struct nvkm_chan *chan, struct nvkm_memory *muserd, u64 ouserd)
321 {
322 struct nvkm_fifo *fifo = chan->cgrp->runl->fifo;
323 int ret;
324
325 mutex_lock(&fifo->userd.mutex);
326 ret = r535_chan_id_get_locked(chan, muserd, ouserd);
327 mutex_unlock(&fifo->userd.mutex);
328 return ret;
329 }
330
331 static const struct nvkm_chan_func
332 r535_chan = {
333 .id_get = r535_chan_id_get,
334 .id_put = r535_chan_id_put,
335 .inst = &gf100_chan_inst,
336 .userd = &gv100_chan_userd,
337 .ramfc = &r535_chan_ramfc,
338 .start = r535_chan_start,
339 .stop = r535_chan_stop,
340 .doorbell_handle = r535_chan_doorbell_handle,
341 };
342
343 static const struct nvkm_cgrp_func
344 r535_cgrp = {
345 };
346
347 static int
r535_engn_nonstall(struct nvkm_engn * engn)348 r535_engn_nonstall(struct nvkm_engn *engn)
349 {
350 struct nvkm_subdev *subdev = &engn->engine->subdev;
351 int ret;
352
353 ret = nvkm_gsp_intr_nonstall(subdev->device->gsp, subdev->type, subdev->inst);
354 WARN_ON(ret == -ENOENT);
355 return ret;
356 }
357
358 static const struct nvkm_engn_func
359 r535_ce = {
360 .nonstall = r535_engn_nonstall,
361 };
362
363 static int
r535_gr_ctor(struct nvkm_engn * engn,struct nvkm_vctx * vctx,struct nvkm_chan * chan)364 r535_gr_ctor(struct nvkm_engn *engn, struct nvkm_vctx *vctx, struct nvkm_chan *chan)
365 {
366 /* RM requires GR context buffers to remain mapped until after the
367 * channel has been destroyed (as opposed to after the last gr obj
368 * has been deleted).
369 *
370 * Take an extra ref here, which will be released once the channel
371 * object has been deleted.
372 */
373 refcount_inc(&vctx->refs);
374 chan->rm.grctx = vctx;
375 return 0;
376 }
377
378 static const struct nvkm_engn_func
379 r535_gr = {
380 .nonstall = r535_engn_nonstall,
381 .ctor2 = r535_gr_ctor,
382 };
383
384 static int
r535_flcn_bind(struct nvkm_engn * engn,struct nvkm_vctx * vctx,struct nvkm_chan * chan)385 r535_flcn_bind(struct nvkm_engn *engn, struct nvkm_vctx *vctx, struct nvkm_chan *chan)
386 {
387 struct nvkm_gsp_client *client = &chan->vmm->rm.client;
388 NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS *ctrl;
389
390 ctrl = nvkm_gsp_rm_ctrl_get(&chan->vmm->rm.device.subdevice,
391 NV2080_CTRL_CMD_GPU_PROMOTE_CTX, sizeof(*ctrl));
392 if (IS_ERR(ctrl))
393 return PTR_ERR(ctrl);
394
395 ctrl->hClient = client->object.handle;
396 ctrl->hObject = chan->rm.object.handle;
397 ctrl->hChanClient = client->object.handle;
398 ctrl->virtAddress = vctx->vma->addr;
399 ctrl->size = vctx->inst->size;
400 ctrl->engineType = engn->id;
401 ctrl->ChID = chan->id;
402
403 return nvkm_gsp_rm_ctrl_wr(&chan->vmm->rm.device.subdevice, ctrl);
404 }
405
406 static int
r535_flcn_ctor(struct nvkm_engn * engn,struct nvkm_vctx * vctx,struct nvkm_chan * chan)407 r535_flcn_ctor(struct nvkm_engn *engn, struct nvkm_vctx *vctx, struct nvkm_chan *chan)
408 {
409 int ret;
410
411 if (WARN_ON(!engn->rm.size))
412 return -EINVAL;
413
414 ret = nvkm_gpuobj_new(engn->engine->subdev.device, engn->rm.size, 0, true, NULL,
415 &vctx->inst);
416 if (ret)
417 return ret;
418
419 ret = nvkm_vmm_get(vctx->vmm, 12, vctx->inst->size, &vctx->vma);
420 if (ret)
421 return ret;
422
423 ret = nvkm_memory_map(vctx->inst, 0, vctx->vmm, vctx->vma, NULL, 0);
424 if (ret)
425 return ret;
426
427 return r535_flcn_bind(engn, vctx, chan);
428 }
429
430 static const struct nvkm_engn_func
431 r535_flcn = {
432 .nonstall = r535_engn_nonstall,
433 .ctor2 = r535_flcn_ctor,
434 };
435
436 static void
r535_runl_allow(struct nvkm_runl * runl,u32 engm)437 r535_runl_allow(struct nvkm_runl *runl, u32 engm)
438 {
439 }
440
441 static void
r535_runl_block(struct nvkm_runl * runl,u32 engm)442 r535_runl_block(struct nvkm_runl *runl, u32 engm)
443 {
444 }
445
446 static const struct nvkm_runl_func
447 r535_runl = {
448 .block = r535_runl_block,
449 .allow = r535_runl_allow,
450 };
451
452 static int
r535_fifo_2080_type(enum nvkm_subdev_type type,int inst)453 r535_fifo_2080_type(enum nvkm_subdev_type type, int inst)
454 {
455 switch (type) {
456 case NVKM_ENGINE_GR: return NV2080_ENGINE_TYPE_GR0;
457 case NVKM_ENGINE_CE: return NV2080_ENGINE_TYPE_COPY0 + inst;
458 case NVKM_ENGINE_SEC2: return NV2080_ENGINE_TYPE_SEC2;
459 case NVKM_ENGINE_NVDEC: return NV2080_ENGINE_TYPE_NVDEC0 + inst;
460 case NVKM_ENGINE_NVENC: return NV2080_ENGINE_TYPE_NVENC0 + inst;
461 case NVKM_ENGINE_NVJPG: return NV2080_ENGINE_TYPE_NVJPEG0 + inst;
462 case NVKM_ENGINE_OFA: return NV2080_ENGINE_TYPE_OFA;
463 case NVKM_ENGINE_SW: return NV2080_ENGINE_TYPE_SW;
464 default:
465 break;
466 }
467
468 WARN_ON(1);
469 return -EINVAL;
470 }
471
472 static int
r535_fifo_engn_type(RM_ENGINE_TYPE rm,enum nvkm_subdev_type * ptype)473 r535_fifo_engn_type(RM_ENGINE_TYPE rm, enum nvkm_subdev_type *ptype)
474 {
475 switch (rm) {
476 case RM_ENGINE_TYPE_GR0:
477 *ptype = NVKM_ENGINE_GR;
478 return 0;
479 case RM_ENGINE_TYPE_COPY0...RM_ENGINE_TYPE_COPY9:
480 *ptype = NVKM_ENGINE_CE;
481 return rm - RM_ENGINE_TYPE_COPY0;
482 case RM_ENGINE_TYPE_NVDEC0...RM_ENGINE_TYPE_NVDEC7:
483 *ptype = NVKM_ENGINE_NVDEC;
484 return rm - RM_ENGINE_TYPE_NVDEC0;
485 case RM_ENGINE_TYPE_NVENC0...RM_ENGINE_TYPE_NVENC2:
486 *ptype = NVKM_ENGINE_NVENC;
487 return rm - RM_ENGINE_TYPE_NVENC0;
488 case RM_ENGINE_TYPE_SW:
489 *ptype = NVKM_ENGINE_SW;
490 return 0;
491 case RM_ENGINE_TYPE_SEC2:
492 *ptype = NVKM_ENGINE_SEC2;
493 return 0;
494 case RM_ENGINE_TYPE_NVJPEG0...RM_ENGINE_TYPE_NVJPEG7:
495 *ptype = NVKM_ENGINE_NVJPG;
496 return rm - RM_ENGINE_TYPE_NVJPEG0;
497 case RM_ENGINE_TYPE_OFA:
498 *ptype = NVKM_ENGINE_OFA;
499 return 0;
500 default:
501 return -EINVAL;
502 }
503 }
504
505 static int
r535_fifo_ectx_size(struct nvkm_fifo * fifo)506 r535_fifo_ectx_size(struct nvkm_fifo *fifo)
507 {
508 NV2080_CTRL_INTERNAL_GET_CONSTRUCTED_FALCON_INFO_PARAMS *ctrl;
509 struct nvkm_gsp *gsp = fifo->engine.subdev.device->gsp;
510 struct nvkm_runl *runl;
511 struct nvkm_engn *engn;
512
513 ctrl = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice,
514 NV2080_CTRL_CMD_INTERNAL_GET_CONSTRUCTED_FALCON_INFO,
515 sizeof(*ctrl));
516 if (WARN_ON(IS_ERR(ctrl)))
517 return PTR_ERR(ctrl);
518
519 for (int i = 0; i < ctrl->numConstructedFalcons; i++) {
520 nvkm_runl_foreach(runl, fifo) {
521 nvkm_runl_foreach_engn(engn, runl) {
522 if (engn->rm.desc == ctrl->constructedFalconsTable[i].engDesc) {
523 engn->rm.size =
524 ctrl->constructedFalconsTable[i].ctxBufferSize;
525 break;
526 }
527 }
528 }
529 }
530
531 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);
532 return 0;
533 }
534
535 static int
r535_fifo_runl_ctor(struct nvkm_fifo * fifo)536 r535_fifo_runl_ctor(struct nvkm_fifo *fifo)
537 {
538 struct nvkm_subdev *subdev = &fifo->engine.subdev;
539 struct nvkm_gsp *gsp = subdev->device->gsp;
540 struct nvkm_runl *runl;
541 struct nvkm_engn *engn;
542 u32 cgids = 2048;
543 u32 chids = 2048;
544 int ret;
545 NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS *ctrl;
546
547 if ((ret = nvkm_chid_new(&nvkm_chan_event, subdev, cgids, 0, cgids, &fifo->cgid)) ||
548 (ret = nvkm_chid_new(&nvkm_chan_event, subdev, chids, 0, chids, &fifo->chid)))
549 return ret;
550
551 ctrl = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice,
552 NV2080_CTRL_CMD_FIFO_GET_DEVICE_INFO_TABLE, sizeof(*ctrl));
553 if (WARN_ON(IS_ERR(ctrl)))
554 return PTR_ERR(ctrl);
555
556 for (int i = 0; i < ctrl->numEntries; i++) {
557 const u32 addr = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_RUNLIST_PRI_BASE];
558 const u32 id = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_RUNLIST];
559
560 runl = nvkm_runl_get(fifo, id, addr);
561 if (!runl) {
562 runl = nvkm_runl_new(fifo, id, addr, 0);
563 if (WARN_ON(IS_ERR(runl)))
564 continue;
565 }
566 }
567
568 for (int i = 0; i < ctrl->numEntries; i++) {
569 const u32 addr = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_RUNLIST_PRI_BASE];
570 const u32 rmid = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_RM_ENGINE_TYPE];
571 const u32 id = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_RUNLIST];
572 enum nvkm_subdev_type type;
573 int inst, nv2080;
574
575 runl = nvkm_runl_get(fifo, id, addr);
576 if (!runl)
577 continue;
578
579 inst = r535_fifo_engn_type(rmid, &type);
580 if (inst < 0) {
581 nvkm_warn(subdev, "RM_ENGINE_TYPE 0x%x\n", rmid);
582 nvkm_runl_del(runl);
583 continue;
584 }
585
586 nv2080 = r535_fifo_2080_type(type, inst);
587 if (nv2080 < 0) {
588 nvkm_runl_del(runl);
589 continue;
590 }
591
592 switch (type) {
593 case NVKM_ENGINE_CE:
594 engn = nvkm_runl_add(runl, nv2080, &r535_ce, type, inst);
595 break;
596 case NVKM_ENGINE_GR:
597 engn = nvkm_runl_add(runl, nv2080, &r535_gr, type, inst);
598 break;
599 case NVKM_ENGINE_NVDEC:
600 case NVKM_ENGINE_NVENC:
601 case NVKM_ENGINE_NVJPG:
602 case NVKM_ENGINE_OFA:
603 engn = nvkm_runl_add(runl, nv2080, &r535_flcn, type, inst);
604 break;
605 case NVKM_ENGINE_SW:
606 continue;
607 default:
608 engn = NULL;
609 break;
610 }
611
612 if (!engn) {
613 nvkm_runl_del(runl);
614 continue;
615 }
616
617 engn->rm.desc = ctrl->entries[i].engineData[ENGINE_INFO_TYPE_ENG_DESC];
618 }
619
620 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);
621
622 {
623 NV2080_CTRL_CE_GET_FAULT_METHOD_BUFFER_SIZE_PARAMS *ctrl;
624
625 ctrl = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice,
626 NV2080_CTRL_CMD_CE_GET_FAULT_METHOD_BUFFER_SIZE,
627 sizeof(*ctrl));
628 if (IS_ERR(ctrl))
629 return PTR_ERR(ctrl);
630
631 fifo->rm.mthdbuf_size = ctrl->size;
632
633 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);
634 }
635
636 return r535_fifo_ectx_size(fifo);
637 }
638
639 static void
r535_fifo_dtor(struct nvkm_fifo * fifo)640 r535_fifo_dtor(struct nvkm_fifo *fifo)
641 {
642 kfree(fifo->func);
643 }
644
645 int
r535_fifo_new(const struct nvkm_fifo_func * hw,struct nvkm_device * device,enum nvkm_subdev_type type,int inst,struct nvkm_fifo ** pfifo)646 r535_fifo_new(const struct nvkm_fifo_func *hw, struct nvkm_device *device,
647 enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo)
648 {
649 struct nvkm_fifo_func *rm;
650
651 if (!(rm = kzalloc(sizeof(*rm), GFP_KERNEL)))
652 return -ENOMEM;
653
654 rm->dtor = r535_fifo_dtor;
655 rm->runl_ctor = r535_fifo_runl_ctor;
656 rm->runl = &r535_runl;
657 rm->cgrp = hw->cgrp;
658 rm->cgrp.func = &r535_cgrp;
659 rm->chan = hw->chan;
660 rm->chan.func = &r535_chan;
661 rm->nonstall = &ga100_fifo_nonstall;
662 rm->nonstall_ctor = ga100_fifo_nonstall_ctor;
663
664 return nvkm_fifo_new_(rm, device, type, inst, pfifo);
665 }
666