1 /* 2 * Copyright 2023 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include "gf100.h" 23 24 #include <core/memory.h> 25 #include <subdev/gsp.h> 26 #include <subdev/mmu/vmm.h> 27 #include <engine/fifo/priv.h> 28 29 #include <nvif/if900d.h> 30 31 #include <nvhw/drf.h> 32 33 #include <nvrm/nvtypes.h> 34 #include <nvrm/535.113.01/common/sdk/nvidia/inc/alloc/alloc_channel.h> 35 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080fifo.h> 36 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h> 37 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080internal.h> 38 #include <nvrm/535.113.01/nvidia/generated/g_kernel_channel_nvoc.h> 39 40 #define r535_gr(p) container_of((p), struct r535_gr, base) 41 42 #define R515_GR_MAX_CTXBUFS 9 43 44 struct r535_gr { 45 struct nvkm_gr base; 46 47 struct { 48 u16 bufferId; 49 u32 size; 50 u8 page; 51 u8 align; 52 bool global; 53 bool init; 54 bool ro; 55 } ctxbuf[R515_GR_MAX_CTXBUFS]; 56 int ctxbuf_nr; 57 58 struct nvkm_memory *ctxbuf_mem[R515_GR_MAX_CTXBUFS]; 59 }; 60 61 struct r535_gr_chan { 62 struct nvkm_object object; 63 struct r535_gr *gr; 64 65 struct nvkm_vmm *vmm; 66 struct nvkm_chan *chan; 67 68 struct nvkm_memory *mem[R515_GR_MAX_CTXBUFS]; 69 struct nvkm_vma *vma[R515_GR_MAX_CTXBUFS]; 70 }; 71 72 struct r535_gr_obj { 73 struct nvkm_object object; 74 struct nvkm_gsp_object rm; 75 }; 76 77 static void * 78 r535_gr_obj_dtor(struct nvkm_object *object) 79 { 80 struct r535_gr_obj *obj = container_of(object, typeof(*obj), object); 81 82 nvkm_gsp_rm_free(&obj->rm); 83 return obj; 84 } 85 86 static const struct nvkm_object_func 87 r535_gr_obj = { 88 .dtor = r535_gr_obj_dtor, 89 }; 90 91 static int 92 r535_gr_obj_ctor(const struct nvkm_oclass *oclass, void *argv, u32 argc, 93 struct nvkm_object **pobject) 94 { 95 struct r535_gr_chan *chan = container_of(oclass->parent, typeof(*chan), object); 96 struct r535_gr_obj *obj; 97 98 if (!(obj = kzalloc(sizeof(*obj), GFP_KERNEL))) 99 return -ENOMEM; 100 101 nvkm_object_ctor(&r535_gr_obj, oclass, &obj->object); 102 *pobject = &obj->object; 103 104 return nvkm_gsp_rm_alloc(&chan->chan->rm.object, oclass->handle, oclass->base.oclass, 0, 105 &obj->rm); 106 } 107 108 static void * 109 r535_gr_chan_dtor(struct nvkm_object *object) 110 { 111 struct r535_gr_chan *grc = container_of(object, typeof(*grc), object); 112 struct r535_gr *gr = grc->gr; 113 114 for (int i = 0; i < gr->ctxbuf_nr; i++) { 115 nvkm_vmm_put(grc->vmm, &grc->vma[i]); 116 nvkm_memory_unref(&grc->mem[i]); 117 } 118 119 nvkm_vmm_unref(&grc->vmm); 120 return grc; 121 } 122 123 static const struct nvkm_object_func 124 r535_gr_chan = { 125 .dtor = r535_gr_chan_dtor, 126 }; 127 128 static int 129 r535_gr_promote_ctx(struct r535_gr *gr, bool golden, struct nvkm_vmm *vmm, 130 struct nvkm_memory **pmem, struct nvkm_vma **pvma, 131 struct nvkm_gsp_object *chan) 132 { 133 struct nvkm_subdev *subdev = &gr->base.engine.subdev; 134 struct nvkm_device *device = subdev->device; 135 NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS *ctrl; 136 137 ctrl = nvkm_gsp_rm_ctrl_get(&vmm->rm.device.subdevice, 138 NV2080_CTRL_CMD_GPU_PROMOTE_CTX, sizeof(*ctrl)); 139 if (WARN_ON(IS_ERR(ctrl))) 140 return PTR_ERR(ctrl); 141 142 ctrl->engineType = 1; 143 ctrl->hChanClient = vmm->rm.client.object.handle; 144 ctrl->hObject = chan->handle; 145 146 for (int i = 0; i < gr->ctxbuf_nr; i++) { 147 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY *entry = 148 &ctrl->promoteEntry[ctrl->entryCount]; 149 const bool alloc = golden || !gr->ctxbuf[i].global; 150 int ret; 151 152 entry->bufferId = gr->ctxbuf[i].bufferId; 153 entry->bInitialize = gr->ctxbuf[i].init && alloc; 154 155 if (alloc) { 156 ret = nvkm_memory_new(device, gr->ctxbuf[i].init ? 157 NVKM_MEM_TARGET_INST : NVKM_MEM_TARGET_INST_SR_LOST, 158 gr->ctxbuf[i].size, 1 << gr->ctxbuf[i].page, 159 gr->ctxbuf[i].init, &pmem[i]); 160 if (WARN_ON(ret)) 161 return ret; 162 163 if (gr->ctxbuf[i].bufferId == 164 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP) 165 entry->bNonmapped = 1; 166 } else { 167 if (gr->ctxbuf[i].bufferId == 168 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP) 169 continue; 170 171 pmem[i] = nvkm_memory_ref(gr->ctxbuf_mem[i]); 172 } 173 174 if (!entry->bNonmapped) { 175 struct gf100_vmm_map_v0 args = { 176 .priv = 1, 177 .ro = gr->ctxbuf[i].ro, 178 }; 179 180 mutex_lock(&vmm->mutex.vmm); 181 ret = nvkm_vmm_get_locked(vmm, false, true, false, 0, gr->ctxbuf[i].align, 182 nvkm_memory_size(pmem[i]), &pvma[i]); 183 mutex_unlock(&vmm->mutex.vmm); 184 if (ret) 185 return ret; 186 187 ret = nvkm_memory_map(pmem[i], 0, vmm, pvma[i], &args, sizeof(args)); 188 if (ret) 189 return ret; 190 191 entry->gpuVirtAddr = pvma[i]->addr; 192 } 193 194 if (entry->bInitialize) { 195 entry->gpuPhysAddr = nvkm_memory_addr(pmem[i]); 196 entry->size = gr->ctxbuf[i].size; 197 entry->physAttr = 4; 198 } 199 200 nvkm_debug(subdev, 201 "promote %02d: pa %016llx/%08x sz %016llx va %016llx init:%d nm:%d\n", 202 entry->bufferId, entry->gpuPhysAddr, entry->physAttr, entry->size, 203 entry->gpuVirtAddr, entry->bInitialize, entry->bNonmapped); 204 205 ctrl->entryCount++; 206 } 207 208 return nvkm_gsp_rm_ctrl_wr(&vmm->rm.device.subdevice, ctrl); 209 } 210 211 static int 212 r535_gr_chan_new(struct nvkm_gr *base, struct nvkm_chan *chan, const struct nvkm_oclass *oclass, 213 struct nvkm_object **pobject) 214 { 215 struct r535_gr *gr = r535_gr(base); 216 struct r535_gr_chan *grc; 217 int ret; 218 219 if (!(grc = kzalloc(sizeof(*grc), GFP_KERNEL))) 220 return -ENOMEM; 221 222 nvkm_object_ctor(&r535_gr_chan, oclass, &grc->object); 223 grc->gr = gr; 224 grc->vmm = nvkm_vmm_ref(chan->vmm); 225 grc->chan = chan; 226 *pobject = &grc->object; 227 228 ret = r535_gr_promote_ctx(gr, false, grc->vmm, grc->mem, grc->vma, &chan->rm.object); 229 if (ret) 230 return ret; 231 232 return 0; 233 } 234 235 static u64 236 r535_gr_units(struct nvkm_gr *gr) 237 { 238 struct nvkm_gsp *gsp = gr->engine.subdev.device->gsp; 239 240 return (gsp->gr.tpcs << 8) | gsp->gr.gpcs; 241 } 242 243 static int 244 r535_gr_oneinit(struct nvkm_gr *base) 245 { 246 NV2080_CTRL_INTERNAL_STATIC_GR_GET_CONTEXT_BUFFERS_INFO_PARAMS *info; 247 struct r535_gr *gr = container_of(base, typeof(*gr), base); 248 struct nvkm_subdev *subdev = &gr->base.engine.subdev; 249 struct nvkm_device *device = subdev->device; 250 struct nvkm_gsp *gsp = device->gsp; 251 struct nvkm_mmu *mmu = device->mmu; 252 struct { 253 struct nvkm_memory *inst; 254 struct nvkm_vmm *vmm; 255 struct nvkm_gsp_object chan; 256 struct nvkm_vma *vma[R515_GR_MAX_CTXBUFS]; 257 } golden = {}; 258 int ret; 259 260 /* Allocate a channel to use for golden context init. */ 261 ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x12000, 0, true, &golden.inst); 262 if (ret) 263 goto done; 264 265 ret = nvkm_vmm_new(device, 0x1000, 0, NULL, 0, NULL, "grGoldenVmm", &golden.vmm); 266 if (ret) 267 goto done; 268 269 ret = mmu->func->promote_vmm(golden.vmm); 270 if (ret) 271 goto done; 272 273 { 274 NV_CHANNELGPFIFO_ALLOCATION_PARAMETERS *args; 275 276 args = nvkm_gsp_rm_alloc_get(&golden.vmm->rm.device.object, 0xf1f00000, 277 device->fifo->func->chan.user.oclass, 278 sizeof(*args), &golden.chan); 279 if (IS_ERR(args)) { 280 ret = PTR_ERR(args); 281 goto done; 282 } 283 284 args->gpFifoOffset = 0; 285 args->gpFifoEntries = 0x1000 / 8; 286 args->flags = 287 NVDEF(NVOS04, FLAGS, CHANNEL_TYPE, PHYSICAL) | 288 NVDEF(NVOS04, FLAGS, VPR, FALSE) | 289 NVDEF(NVOS04, FLAGS, CHANNEL_SKIP_MAP_REFCOUNTING, FALSE) | 290 NVVAL(NVOS04, FLAGS, GROUP_CHANNEL_RUNQUEUE, 0) | 291 NVDEF(NVOS04, FLAGS, PRIVILEGED_CHANNEL, TRUE) | 292 NVDEF(NVOS04, FLAGS, DELAY_CHANNEL_SCHEDULING, FALSE) | 293 NVDEF(NVOS04, FLAGS, CHANNEL_DENY_PHYSICAL_MODE_CE, FALSE) | 294 NVVAL(NVOS04, FLAGS, CHANNEL_USERD_INDEX_VALUE, 0) | 295 NVDEF(NVOS04, FLAGS, CHANNEL_USERD_INDEX_FIXED, FALSE) | 296 NVVAL(NVOS04, FLAGS, CHANNEL_USERD_INDEX_PAGE_VALUE, 0) | 297 NVDEF(NVOS04, FLAGS, CHANNEL_USERD_INDEX_PAGE_FIXED, TRUE) | 298 NVDEF(NVOS04, FLAGS, CHANNEL_DENY_AUTH_LEVEL_PRIV, FALSE) | 299 NVDEF(NVOS04, FLAGS, CHANNEL_SKIP_SCRUBBER, FALSE) | 300 NVDEF(NVOS04, FLAGS, CHANNEL_CLIENT_MAP_FIFO, FALSE) | 301 NVDEF(NVOS04, FLAGS, SET_EVICT_LAST_CE_PREFETCH_CHANNEL, FALSE) | 302 NVDEF(NVOS04, FLAGS, CHANNEL_VGPU_PLUGIN_CONTEXT, FALSE) | 303 NVDEF(NVOS04, FLAGS, CHANNEL_PBDMA_ACQUIRE_TIMEOUT, FALSE) | 304 NVDEF(NVOS04, FLAGS, GROUP_CHANNEL_THREAD, DEFAULT) | 305 NVDEF(NVOS04, FLAGS, MAP_CHANNEL, FALSE) | 306 NVDEF(NVOS04, FLAGS, SKIP_CTXBUFFER_ALLOC, FALSE); 307 args->hVASpace = golden.vmm->rm.object.handle; 308 args->engineType = 1; 309 args->instanceMem.base = nvkm_memory_addr(golden.inst); 310 args->instanceMem.size = 0x1000; 311 args->instanceMem.addressSpace = 2; 312 args->instanceMem.cacheAttrib = 1; 313 args->ramfcMem.base = nvkm_memory_addr(golden.inst); 314 args->ramfcMem.size = 0x200; 315 args->ramfcMem.addressSpace = 2; 316 args->ramfcMem.cacheAttrib = 1; 317 args->userdMem.base = nvkm_memory_addr(golden.inst) + 0x1000; 318 args->userdMem.size = 0x200; 319 args->userdMem.addressSpace = 2; 320 args->userdMem.cacheAttrib = 1; 321 args->mthdbufMem.base = nvkm_memory_addr(golden.inst) + 0x2000; 322 args->mthdbufMem.size = 0x5000; 323 args->mthdbufMem.addressSpace = 2; 324 args->mthdbufMem.cacheAttrib = 1; 325 args->internalFlags = 326 NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, PRIVILEGE, ADMIN) | 327 NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, ERROR_NOTIFIER_TYPE, NONE) | 328 NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, ECC_ERROR_NOTIFIER_TYPE, NONE); 329 330 ret = nvkm_gsp_rm_alloc_wr(&golden.chan, args); 331 if (ret) 332 goto done; 333 } 334 335 /* Fetch context buffer info from RM and allocate each of them here to use 336 * during golden context init (or later as a global context buffer). 337 * 338 * Also build the information that'll be used to create channel contexts. 339 */ 340 info = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice, 341 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_CONTEXT_BUFFERS_INFO, 342 sizeof(*info)); 343 if (WARN_ON(IS_ERR(info))) { 344 ret = PTR_ERR(info); 345 goto done; 346 } 347 348 for (int i = 0; i < ARRAY_SIZE(info->engineContextBuffersInfo[0].engine); i++) { 349 static const struct { 350 u32 id0; /* NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID */ 351 u32 id1; /* NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID */ 352 bool global; 353 bool init; 354 bool ro; 355 } map[] = { 356 #define _A(n,N,G,I,R) { .id0 = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_##n, \ 357 .id1 = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_##N, \ 358 .global = (G), .init = (I), .ro = (R) } 359 #define _B(N,G,I,R) _A(GRAPHICS_##N, N, (G), (I), (R)) 360 /* global init ro */ 361 _A( GRAPHICS, MAIN, false, true, false), 362 _B( PATCH, false, true, false), 363 _A( GRAPHICS_BUNDLE_CB, BUFFER_BUNDLE_CB, true, false, false), 364 _B( PAGEPOOL, true, false, false), 365 _B( ATTRIBUTE_CB, true, false, false), 366 _B( RTV_CB_GLOBAL, true, false, false), 367 _B( FECS_EVENT, true, true, false), 368 _B( PRIV_ACCESS_MAP, true, true, true), 369 #undef _B 370 #undef _A 371 }; 372 u32 size = info->engineContextBuffersInfo[0].engine[i].size; 373 u8 align, page; 374 int id; 375 376 for (id = 0; id < ARRAY_SIZE(map); id++) { 377 if (map[id].id0 == i) 378 break; 379 } 380 381 nvkm_debug(subdev, "%02x: size:0x%08x %s\n", i, 382 size, (id < ARRAY_SIZE(map)) ? "*" : ""); 383 if (id >= ARRAY_SIZE(map)) 384 continue; 385 386 if (map[id].id1 == NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN) 387 size = ALIGN(size, 0x1000) + 64 * 0x1000; /* per-subctx headers */ 388 389 if (size >= 1 << 21) page = 21; 390 else if (size >= 1 << 16) page = 16; 391 else page = 12; 392 393 if (map[id].id1 == NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB) 394 align = order_base_2(size); 395 else 396 align = page; 397 398 if (WARN_ON(gr->ctxbuf_nr == ARRAY_SIZE(gr->ctxbuf))) 399 continue; 400 401 gr->ctxbuf[gr->ctxbuf_nr].bufferId = map[id].id1; 402 gr->ctxbuf[gr->ctxbuf_nr].size = size; 403 gr->ctxbuf[gr->ctxbuf_nr].page = page; 404 gr->ctxbuf[gr->ctxbuf_nr].align = align; 405 gr->ctxbuf[gr->ctxbuf_nr].global = map[id].global; 406 gr->ctxbuf[gr->ctxbuf_nr].init = map[id].init; 407 gr->ctxbuf[gr->ctxbuf_nr].ro = map[id].ro; 408 gr->ctxbuf_nr++; 409 410 if (map[id].id1 == NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP) { 411 if (WARN_ON(gr->ctxbuf_nr == ARRAY_SIZE(gr->ctxbuf))) 412 continue; 413 414 gr->ctxbuf[gr->ctxbuf_nr] = gr->ctxbuf[gr->ctxbuf_nr - 1]; 415 gr->ctxbuf[gr->ctxbuf_nr].bufferId = 416 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP; 417 gr->ctxbuf_nr++; 418 } 419 } 420 421 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, info); 422 423 /* Promote golden context to RM. */ 424 ret = r535_gr_promote_ctx(gr, true, golden.vmm, gr->ctxbuf_mem, golden.vma, &golden.chan); 425 if (ret) 426 goto done; 427 428 /* Allocate 3D class on channel to trigger golden context init in RM. */ 429 { 430 int i; 431 432 for (i = 0; gr->base.func->sclass[i].ctor; i++) { 433 if ((gr->base.func->sclass[i].oclass & 0xff) == 0x97) { 434 struct nvkm_gsp_object threed; 435 436 ret = nvkm_gsp_rm_alloc(&golden.chan, 0x97000000, 437 gr->base.func->sclass[i].oclass, 0, 438 &threed); 439 if (ret) 440 goto done; 441 442 nvkm_gsp_rm_free(&threed); 443 break; 444 } 445 } 446 447 if (WARN_ON(!gr->base.func->sclass[i].ctor)) { 448 ret = -EINVAL; 449 goto done; 450 } 451 } 452 453 done: 454 nvkm_gsp_rm_free(&golden.chan); 455 for (int i = gr->ctxbuf_nr - 1; i >= 0; i--) 456 nvkm_vmm_put(golden.vmm, &golden.vma[i]); 457 nvkm_vmm_unref(&golden.vmm); 458 nvkm_memory_unref(&golden.inst); 459 return ret; 460 461 } 462 463 static void * 464 r535_gr_dtor(struct nvkm_gr *base) 465 { 466 struct r535_gr *gr = r535_gr(base); 467 468 while (gr->ctxbuf_nr) 469 nvkm_memory_unref(&gr->ctxbuf_mem[--gr->ctxbuf_nr]); 470 471 kfree(gr->base.func); 472 return gr; 473 } 474 475 int 476 r535_gr_new(const struct gf100_gr_func *hw, 477 struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_gr **pgr) 478 { 479 struct nvkm_gr_func *rm; 480 struct r535_gr *gr; 481 int nclass; 482 483 for (nclass = 0; hw->sclass[nclass].oclass; nclass++); 484 485 if (!(rm = kzalloc(sizeof(*rm) + (nclass + 1) * sizeof(rm->sclass[0]), GFP_KERNEL))) 486 return -ENOMEM; 487 488 rm->dtor = r535_gr_dtor; 489 rm->oneinit = r535_gr_oneinit; 490 rm->units = r535_gr_units; 491 rm->chan_new = r535_gr_chan_new; 492 493 for (int i = 0; i < nclass; i++) { 494 rm->sclass[i].minver = hw->sclass[i].minver; 495 rm->sclass[i].maxver = hw->sclass[i].maxver; 496 rm->sclass[i].oclass = hw->sclass[i].oclass; 497 rm->sclass[i].ctor = r535_gr_obj_ctor; 498 } 499 500 if (!(gr = kzalloc(sizeof(*gr), GFP_KERNEL))) { 501 kfree(rm); 502 return -ENOMEM; 503 } 504 505 *pgr = &gr->base; 506 507 return nvkm_gr_ctor(rm, device, type, inst, true, &gr->base); 508 } 509