1 /* 2 * Copyright 2023 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include "priv.h" 23 24 #include <core/pci.h> 25 #include <subdev/timer.h> 26 #include <subdev/vfn.h> 27 #include <engine/fifo/chan.h> 28 #include <engine/sec2.h> 29 30 #include <nvfw/fw.h> 31 32 #include <nvrm/nvtypes.h> 33 #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0000.h> 34 #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0005.h> 35 #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0080.h> 36 #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl2080.h> 37 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080event.h> 38 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h> 39 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080internal.h> 40 #include <nvrm/535.113.01/common/sdk/nvidia/inc/nvos.h> 41 #include <nvrm/535.113.01/common/shared/msgq/inc/msgq/msgq_priv.h> 42 #include <nvrm/535.113.01/common/uproc/os/common/include/libos_init_args.h> 43 #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_sr_meta.h> 44 #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_wpr_meta.h> 45 #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/rmRiscvUcode.h> 46 #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/rmgspseq.h> 47 #include <nvrm/535.113.01/nvidia/generated/g_allclasses.h> 48 #include <nvrm/535.113.01/nvidia/generated/g_os_nvoc.h> 49 #include <nvrm/535.113.01/nvidia/generated/g_rpc-structures.h> 50 #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_fw_heap.h> 51 #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_init_args.h> 52 #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_static_config.h> 53 #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/intr/engine_idx.h> 54 #include <nvrm/535.113.01/nvidia/kernel/inc/vgpu/rpc_global_enums.h> 55 56 #include <linux/acpi.h> 57 #include <linux/ctype.h> 58 #include <linux/parser.h> 59 60 #define GSP_MSG_MIN_SIZE GSP_PAGE_SIZE 61 #define GSP_MSG_MAX_SIZE GSP_PAGE_MIN_SIZE * 16 62 63 struct r535_gsp_msg { 64 u8 auth_tag_buffer[16]; 65 u8 aad_buffer[16]; 66 u32 checksum; 67 u32 sequence; 68 u32 elem_count; 69 u32 pad; 70 u8 data[]; 71 }; 72 73 #define GSP_MSG_HDR_SIZE offsetof(struct r535_gsp_msg, data) 74 75 static int 76 r535_rpc_status_to_errno(uint32_t rpc_status) 77 { 78 switch (rpc_status) { 79 case 0x55: /* NV_ERR_NOT_READY */ 80 case 0x66: /* NV_ERR_TIMEOUT_RETRY */ 81 return -EAGAIN; 82 case 0x51: /* NV_ERR_NO_MEMORY */ 83 return -ENOMEM; 84 default: 85 return -EINVAL; 86 } 87 } 88 89 static void * 90 r535_gsp_msgq_wait(struct nvkm_gsp *gsp, u32 repc, u32 *prepc, int *ptime) 91 { 92 struct r535_gsp_msg *mqe; 93 u32 size, rptr = *gsp->msgq.rptr; 94 int used; 95 u8 *msg; 96 u32 len; 97 98 size = DIV_ROUND_UP(GSP_MSG_HDR_SIZE + repc, GSP_PAGE_SIZE); 99 if (WARN_ON(!size || size >= gsp->msgq.cnt)) 100 return ERR_PTR(-EINVAL); 101 102 do { 103 u32 wptr = *gsp->msgq.wptr; 104 105 used = wptr + gsp->msgq.cnt - rptr; 106 if (used >= gsp->msgq.cnt) 107 used -= gsp->msgq.cnt; 108 if (used >= size) 109 break; 110 111 usleep_range(1, 2); 112 } while (--(*ptime)); 113 114 if (WARN_ON(!*ptime)) 115 return ERR_PTR(-ETIMEDOUT); 116 117 mqe = (void *)((u8 *)gsp->shm.msgq.ptr + 0x1000 + rptr * 0x1000); 118 119 if (prepc) { 120 *prepc = (used * GSP_PAGE_SIZE) - sizeof(*mqe); 121 return mqe->data; 122 } 123 124 msg = kvmalloc(repc, GFP_KERNEL); 125 if (!msg) 126 return ERR_PTR(-ENOMEM); 127 128 len = ((gsp->msgq.cnt - rptr) * GSP_PAGE_SIZE) - sizeof(*mqe); 129 len = min_t(u32, repc, len); 130 memcpy(msg, mqe->data, len); 131 132 rptr += DIV_ROUND_UP(len, GSP_PAGE_SIZE); 133 if (rptr == gsp->msgq.cnt) 134 rptr = 0; 135 136 repc -= len; 137 138 if (repc) { 139 mqe = (void *)((u8 *)gsp->shm.msgq.ptr + 0x1000 + 0 * 0x1000); 140 memcpy(msg + len, mqe, repc); 141 142 rptr += DIV_ROUND_UP(repc, GSP_PAGE_SIZE); 143 } 144 145 mb(); 146 (*gsp->msgq.rptr) = rptr; 147 return msg; 148 } 149 150 static void * 151 r535_gsp_msgq_recv(struct nvkm_gsp *gsp, u32 repc, int *ptime) 152 { 153 return r535_gsp_msgq_wait(gsp, repc, NULL, ptime); 154 } 155 156 static int 157 r535_gsp_cmdq_push(struct nvkm_gsp *gsp, void *argv) 158 { 159 struct r535_gsp_msg *cmd = container_of(argv, typeof(*cmd), data); 160 struct r535_gsp_msg *cqe; 161 u32 argc = cmd->checksum; 162 u64 *ptr = (void *)cmd; 163 u64 *end; 164 u64 csum = 0; 165 int free, time = 1000000; 166 u32 wptr, size; 167 u32 off = 0; 168 169 argc = ALIGN(GSP_MSG_HDR_SIZE + argc, GSP_PAGE_SIZE); 170 171 end = (u64 *)((char *)ptr + argc); 172 cmd->pad = 0; 173 cmd->checksum = 0; 174 cmd->sequence = gsp->cmdq.seq++; 175 cmd->elem_count = DIV_ROUND_UP(argc, 0x1000); 176 177 while (ptr < end) 178 csum ^= *ptr++; 179 180 cmd->checksum = upper_32_bits(csum) ^ lower_32_bits(csum); 181 182 wptr = *gsp->cmdq.wptr; 183 do { 184 do { 185 free = *gsp->cmdq.rptr + gsp->cmdq.cnt - wptr - 1; 186 if (free >= gsp->cmdq.cnt) 187 free -= gsp->cmdq.cnt; 188 if (free >= 1) 189 break; 190 191 usleep_range(1, 2); 192 } while(--time); 193 194 if (WARN_ON(!time)) { 195 kvfree(cmd); 196 return -ETIMEDOUT; 197 } 198 199 cqe = (void *)((u8 *)gsp->shm.cmdq.ptr + 0x1000 + wptr * 0x1000); 200 size = min_t(u32, argc, (gsp->cmdq.cnt - wptr) * GSP_PAGE_SIZE); 201 memcpy(cqe, (u8 *)cmd + off, size); 202 203 wptr += DIV_ROUND_UP(size, 0x1000); 204 if (wptr == gsp->cmdq.cnt) 205 wptr = 0; 206 207 off += size; 208 argc -= size; 209 } while(argc); 210 211 nvkm_trace(&gsp->subdev, "cmdq: wptr %d\n", wptr); 212 wmb(); 213 (*gsp->cmdq.wptr) = wptr; 214 mb(); 215 216 nvkm_falcon_wr32(&gsp->falcon, 0xc00, 0x00000000); 217 218 kvfree(cmd); 219 return 0; 220 } 221 222 static void * 223 r535_gsp_cmdq_get(struct nvkm_gsp *gsp, u32 argc) 224 { 225 struct r535_gsp_msg *cmd; 226 u32 size = GSP_MSG_HDR_SIZE + argc; 227 228 size = ALIGN(size, GSP_MSG_MIN_SIZE); 229 cmd = kvzalloc(size, GFP_KERNEL); 230 if (!cmd) 231 return ERR_PTR(-ENOMEM); 232 233 cmd->checksum = argc; 234 return cmd->data; 235 } 236 237 struct nvfw_gsp_rpc { 238 u32 header_version; 239 u32 signature; 240 u32 length; 241 u32 function; 242 u32 rpc_result; 243 u32 rpc_result_private; 244 u32 sequence; 245 union { 246 u32 spare; 247 u32 cpuRmGfid; 248 }; 249 u8 data[]; 250 }; 251 252 static void 253 r535_gsp_msg_done(struct nvkm_gsp *gsp, struct nvfw_gsp_rpc *msg) 254 { 255 kvfree(msg); 256 } 257 258 static void 259 r535_gsp_msg_dump(struct nvkm_gsp *gsp, struct nvfw_gsp_rpc *msg, int lvl) 260 { 261 if (gsp->subdev.debug >= lvl) { 262 nvkm_printk__(&gsp->subdev, lvl, info, 263 "msg fn:%d len:0x%x/0x%zx res:0x%x resp:0x%x\n", 264 msg->function, msg->length, msg->length - sizeof(*msg), 265 msg->rpc_result, msg->rpc_result_private); 266 print_hex_dump(KERN_INFO, "msg: ", DUMP_PREFIX_OFFSET, 16, 1, 267 msg->data, msg->length - sizeof(*msg), true); 268 } 269 } 270 271 static struct nvfw_gsp_rpc * 272 r535_gsp_msg_recv(struct nvkm_gsp *gsp, int fn, u32 repc) 273 { 274 struct nvkm_subdev *subdev = &gsp->subdev; 275 struct nvfw_gsp_rpc *msg; 276 int time = 4000000, i; 277 u32 size; 278 279 retry: 280 msg = r535_gsp_msgq_wait(gsp, sizeof(*msg), &size, &time); 281 if (IS_ERR_OR_NULL(msg)) 282 return msg; 283 284 msg = r535_gsp_msgq_recv(gsp, msg->length, &time); 285 if (IS_ERR_OR_NULL(msg)) 286 return msg; 287 288 if (msg->rpc_result) { 289 r535_gsp_msg_dump(gsp, msg, NV_DBG_ERROR); 290 r535_gsp_msg_done(gsp, msg); 291 return ERR_PTR(-EINVAL); 292 } 293 294 r535_gsp_msg_dump(gsp, msg, NV_DBG_TRACE); 295 296 if (fn && msg->function == fn) { 297 if (repc) { 298 if (msg->length < sizeof(*msg) + repc) { 299 nvkm_error(subdev, "msg len %d < %zd\n", 300 msg->length, sizeof(*msg) + repc); 301 r535_gsp_msg_dump(gsp, msg, NV_DBG_ERROR); 302 r535_gsp_msg_done(gsp, msg); 303 return ERR_PTR(-EIO); 304 } 305 306 return msg; 307 } 308 309 r535_gsp_msg_done(gsp, msg); 310 return NULL; 311 } 312 313 for (i = 0; i < gsp->msgq.ntfy_nr; i++) { 314 struct nvkm_gsp_msgq_ntfy *ntfy = &gsp->msgq.ntfy[i]; 315 316 if (ntfy->fn == msg->function) { 317 if (ntfy->func) 318 ntfy->func(ntfy->priv, ntfy->fn, msg->data, msg->length - sizeof(*msg)); 319 break; 320 } 321 } 322 323 if (i == gsp->msgq.ntfy_nr) 324 r535_gsp_msg_dump(gsp, msg, NV_DBG_WARN); 325 326 r535_gsp_msg_done(gsp, msg); 327 if (fn) 328 goto retry; 329 330 if (*gsp->msgq.rptr != *gsp->msgq.wptr) 331 goto retry; 332 333 return NULL; 334 } 335 336 static int 337 r535_gsp_msg_ntfy_add(struct nvkm_gsp *gsp, u32 fn, nvkm_gsp_msg_ntfy_func func, void *priv) 338 { 339 int ret = 0; 340 341 mutex_lock(&gsp->msgq.mutex); 342 if (WARN_ON(gsp->msgq.ntfy_nr >= ARRAY_SIZE(gsp->msgq.ntfy))) { 343 ret = -ENOSPC; 344 } else { 345 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].fn = fn; 346 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].func = func; 347 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].priv = priv; 348 gsp->msgq.ntfy_nr++; 349 } 350 mutex_unlock(&gsp->msgq.mutex); 351 return ret; 352 } 353 354 static int 355 r535_gsp_rpc_poll(struct nvkm_gsp *gsp, u32 fn) 356 { 357 void *repv; 358 359 mutex_lock(&gsp->cmdq.mutex); 360 repv = r535_gsp_msg_recv(gsp, fn, 0); 361 mutex_unlock(&gsp->cmdq.mutex); 362 if (IS_ERR(repv)) 363 return PTR_ERR(repv); 364 365 return 0; 366 } 367 368 static void * 369 r535_gsp_rpc_send(struct nvkm_gsp *gsp, void *argv, bool wait, u32 repc) 370 { 371 struct nvfw_gsp_rpc *rpc = container_of(argv, typeof(*rpc), data); 372 struct nvfw_gsp_rpc *msg; 373 u32 fn = rpc->function; 374 void *repv = NULL; 375 int ret; 376 377 if (gsp->subdev.debug >= NV_DBG_TRACE) { 378 nvkm_trace(&gsp->subdev, "rpc fn:%d len:0x%x/0x%zx\n", rpc->function, 379 rpc->length, rpc->length - sizeof(*rpc)); 380 print_hex_dump(KERN_INFO, "rpc: ", DUMP_PREFIX_OFFSET, 16, 1, 381 rpc->data, rpc->length - sizeof(*rpc), true); 382 } 383 384 ret = r535_gsp_cmdq_push(gsp, rpc); 385 if (ret) 386 return ERR_PTR(ret); 387 388 if (wait) { 389 msg = r535_gsp_msg_recv(gsp, fn, repc); 390 if (!IS_ERR_OR_NULL(msg)) 391 repv = msg->data; 392 else 393 repv = msg; 394 } 395 396 return repv; 397 } 398 399 static void 400 r535_gsp_event_dtor(struct nvkm_gsp_event *event) 401 { 402 struct nvkm_gsp_device *device = event->device; 403 struct nvkm_gsp_client *client = device->object.client; 404 struct nvkm_gsp *gsp = client->gsp; 405 406 mutex_lock(&gsp->client_id.mutex); 407 if (event->func) { 408 list_del(&event->head); 409 event->func = NULL; 410 } 411 mutex_unlock(&gsp->client_id.mutex); 412 413 nvkm_gsp_rm_free(&event->object); 414 event->device = NULL; 415 } 416 417 static int 418 r535_gsp_device_event_get(struct nvkm_gsp_event *event) 419 { 420 struct nvkm_gsp_device *device = event->device; 421 NV2080_CTRL_EVENT_SET_NOTIFICATION_PARAMS *ctrl; 422 423 ctrl = nvkm_gsp_rm_ctrl_get(&device->subdevice, 424 NV2080_CTRL_CMD_EVENT_SET_NOTIFICATION, sizeof(*ctrl)); 425 if (IS_ERR(ctrl)) 426 return PTR_ERR(ctrl); 427 428 ctrl->event = event->id; 429 ctrl->action = NV2080_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT; 430 return nvkm_gsp_rm_ctrl_wr(&device->subdevice, ctrl); 431 } 432 433 static int 434 r535_gsp_device_event_ctor(struct nvkm_gsp_device *device, u32 handle, u32 id, 435 nvkm_gsp_event_func func, struct nvkm_gsp_event *event) 436 { 437 struct nvkm_gsp_client *client = device->object.client; 438 struct nvkm_gsp *gsp = client->gsp; 439 NV0005_ALLOC_PARAMETERS *args; 440 int ret; 441 442 args = nvkm_gsp_rm_alloc_get(&device->subdevice, handle, 443 NV01_EVENT_KERNEL_CALLBACK_EX, sizeof(*args), 444 &event->object); 445 if (IS_ERR(args)) 446 return PTR_ERR(args); 447 448 args->hParentClient = client->object.handle; 449 args->hSrcResource = 0; 450 args->hClass = NV01_EVENT_KERNEL_CALLBACK_EX; 451 args->notifyIndex = NV01_EVENT_CLIENT_RM | id; 452 args->data = NULL; 453 454 ret = nvkm_gsp_rm_alloc_wr(&event->object, args); 455 if (ret) 456 return ret; 457 458 event->device = device; 459 event->id = id; 460 461 ret = r535_gsp_device_event_get(event); 462 if (ret) { 463 nvkm_gsp_event_dtor(event); 464 return ret; 465 } 466 467 mutex_lock(&gsp->client_id.mutex); 468 event->func = func; 469 list_add(&event->head, &client->events); 470 mutex_unlock(&gsp->client_id.mutex); 471 return 0; 472 } 473 474 static void 475 r535_gsp_device_dtor(struct nvkm_gsp_device *device) 476 { 477 nvkm_gsp_rm_free(&device->subdevice); 478 nvkm_gsp_rm_free(&device->object); 479 } 480 481 static int 482 r535_gsp_subdevice_ctor(struct nvkm_gsp_device *device) 483 { 484 NV2080_ALLOC_PARAMETERS *args; 485 486 return nvkm_gsp_rm_alloc(&device->object, 0x5d1d0000, NV20_SUBDEVICE_0, sizeof(*args), 487 &device->subdevice); 488 } 489 490 static int 491 r535_gsp_device_ctor(struct nvkm_gsp_client *client, struct nvkm_gsp_device *device) 492 { 493 NV0080_ALLOC_PARAMETERS *args; 494 int ret; 495 496 args = nvkm_gsp_rm_alloc_get(&client->object, 0xde1d0000, NV01_DEVICE_0, sizeof(*args), 497 &device->object); 498 if (IS_ERR(args)) 499 return PTR_ERR(args); 500 501 args->hClientShare = client->object.handle; 502 503 ret = nvkm_gsp_rm_alloc_wr(&device->object, args); 504 if (ret) 505 return ret; 506 507 ret = r535_gsp_subdevice_ctor(device); 508 if (ret) 509 nvkm_gsp_rm_free(&device->object); 510 511 return ret; 512 } 513 514 static void 515 r535_gsp_client_dtor(struct nvkm_gsp_client *client) 516 { 517 struct nvkm_gsp *gsp = client->gsp; 518 519 nvkm_gsp_rm_free(&client->object); 520 521 mutex_lock(&gsp->client_id.mutex); 522 idr_remove(&gsp->client_id.idr, client->object.handle & 0xffff); 523 mutex_unlock(&gsp->client_id.mutex); 524 525 client->gsp = NULL; 526 } 527 528 static int 529 r535_gsp_client_ctor(struct nvkm_gsp *gsp, struct nvkm_gsp_client *client) 530 { 531 NV0000_ALLOC_PARAMETERS *args; 532 int ret; 533 534 mutex_lock(&gsp->client_id.mutex); 535 ret = idr_alloc(&gsp->client_id.idr, client, 0, 0xffff + 1, GFP_KERNEL); 536 mutex_unlock(&gsp->client_id.mutex); 537 if (ret < 0) 538 return ret; 539 540 client->gsp = gsp; 541 client->object.client = client; 542 INIT_LIST_HEAD(&client->events); 543 544 args = nvkm_gsp_rm_alloc_get(&client->object, 0xc1d00000 | ret, NV01_ROOT, sizeof(*args), 545 &client->object); 546 if (IS_ERR(args)) { 547 r535_gsp_client_dtor(client); 548 return ret; 549 } 550 551 args->hClient = client->object.handle; 552 args->processID = ~0; 553 554 ret = nvkm_gsp_rm_alloc_wr(&client->object, args); 555 if (ret) { 556 r535_gsp_client_dtor(client); 557 return ret; 558 } 559 560 return 0; 561 } 562 563 static int 564 r535_gsp_rpc_rm_free(struct nvkm_gsp_object *object) 565 { 566 struct nvkm_gsp_client *client = object->client; 567 struct nvkm_gsp *gsp = client->gsp; 568 rpc_free_v03_00 *rpc; 569 570 nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x free\n", 571 client->object.handle, object->handle); 572 573 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_FREE, sizeof(*rpc)); 574 if (WARN_ON(IS_ERR_OR_NULL(rpc))) 575 return -EIO; 576 577 rpc->params.hRoot = client->object.handle; 578 rpc->params.hObjectParent = 0; 579 rpc->params.hObjectOld = object->handle; 580 return nvkm_gsp_rpc_wr(gsp, rpc, true); 581 } 582 583 static void 584 r535_gsp_rpc_rm_alloc_done(struct nvkm_gsp_object *object, void *repv) 585 { 586 rpc_gsp_rm_alloc_v03_00 *rpc = container_of(repv, typeof(*rpc), params); 587 588 nvkm_gsp_rpc_done(object->client->gsp, rpc); 589 } 590 591 static void * 592 r535_gsp_rpc_rm_alloc_push(struct nvkm_gsp_object *object, void *argv, u32 repc) 593 { 594 rpc_gsp_rm_alloc_v03_00 *rpc = container_of(argv, typeof(*rpc), params); 595 struct nvkm_gsp *gsp = object->client->gsp; 596 void *ret; 597 598 rpc = nvkm_gsp_rpc_push(gsp, rpc, true, sizeof(*rpc) + repc); 599 if (IS_ERR_OR_NULL(rpc)) 600 return rpc; 601 602 if (rpc->status) { 603 ret = ERR_PTR(r535_rpc_status_to_errno(rpc->status)); 604 if (PTR_ERR(ret) != -EAGAIN) 605 nvkm_error(&gsp->subdev, "RM_ALLOC: 0x%x\n", rpc->status); 606 } else { 607 ret = repc ? rpc->params : NULL; 608 } 609 610 nvkm_gsp_rpc_done(gsp, rpc); 611 612 return ret; 613 } 614 615 static void * 616 r535_gsp_rpc_rm_alloc_get(struct nvkm_gsp_object *object, u32 oclass, u32 argc) 617 { 618 struct nvkm_gsp_client *client = object->client; 619 struct nvkm_gsp *gsp = client->gsp; 620 rpc_gsp_rm_alloc_v03_00 *rpc; 621 622 nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x new obj:0x%08x cls:0x%08x argc:%d\n", 623 client->object.handle, object->parent->handle, object->handle, oclass, argc); 624 625 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_RM_ALLOC, sizeof(*rpc) + argc); 626 if (IS_ERR(rpc)) 627 return rpc; 628 629 rpc->hClient = client->object.handle; 630 rpc->hParent = object->parent->handle; 631 rpc->hObject = object->handle; 632 rpc->hClass = oclass; 633 rpc->status = 0; 634 rpc->paramsSize = argc; 635 return rpc->params; 636 } 637 638 static void 639 r535_gsp_rpc_rm_ctrl_done(struct nvkm_gsp_object *object, void *repv) 640 { 641 rpc_gsp_rm_control_v03_00 *rpc = container_of(repv, typeof(*rpc), params); 642 643 if (!repv) 644 return; 645 nvkm_gsp_rpc_done(object->client->gsp, rpc); 646 } 647 648 static int 649 r535_gsp_rpc_rm_ctrl_push(struct nvkm_gsp_object *object, void **argv, u32 repc) 650 { 651 rpc_gsp_rm_control_v03_00 *rpc = container_of((*argv), typeof(*rpc), params); 652 struct nvkm_gsp *gsp = object->client->gsp; 653 int ret = 0; 654 655 rpc = nvkm_gsp_rpc_push(gsp, rpc, true, repc); 656 if (IS_ERR_OR_NULL(rpc)) { 657 *argv = NULL; 658 return PTR_ERR(rpc); 659 } 660 661 if (rpc->status) { 662 ret = r535_rpc_status_to_errno(rpc->status); 663 if (ret != -EAGAIN) 664 nvkm_error(&gsp->subdev, "cli:0x%08x obj:0x%08x ctrl cmd:0x%08x failed: 0x%08x\n", 665 object->client->object.handle, object->handle, rpc->cmd, rpc->status); 666 } 667 668 if (repc) 669 *argv = rpc->params; 670 else 671 nvkm_gsp_rpc_done(gsp, rpc); 672 673 return ret; 674 } 675 676 static void * 677 r535_gsp_rpc_rm_ctrl_get(struct nvkm_gsp_object *object, u32 cmd, u32 argc) 678 { 679 struct nvkm_gsp_client *client = object->client; 680 struct nvkm_gsp *gsp = client->gsp; 681 rpc_gsp_rm_control_v03_00 *rpc; 682 683 nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x ctrl cmd:0x%08x argc:%d\n", 684 client->object.handle, object->handle, cmd, argc); 685 686 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL, sizeof(*rpc) + argc); 687 if (IS_ERR(rpc)) 688 return rpc; 689 690 rpc->hClient = client->object.handle; 691 rpc->hObject = object->handle; 692 rpc->cmd = cmd; 693 rpc->status = 0; 694 rpc->paramsSize = argc; 695 return rpc->params; 696 } 697 698 static void 699 r535_gsp_rpc_done(struct nvkm_gsp *gsp, void *repv) 700 { 701 struct nvfw_gsp_rpc *rpc = container_of(repv, typeof(*rpc), data); 702 703 r535_gsp_msg_done(gsp, rpc); 704 } 705 706 static void * 707 r535_gsp_rpc_get(struct nvkm_gsp *gsp, u32 fn, u32 argc) 708 { 709 struct nvfw_gsp_rpc *rpc; 710 711 rpc = r535_gsp_cmdq_get(gsp, ALIGN(sizeof(*rpc) + argc, sizeof(u64))); 712 if (IS_ERR(rpc)) 713 return ERR_CAST(rpc); 714 715 rpc->header_version = 0x03000000; 716 rpc->signature = ('C' << 24) | ('P' << 16) | ('R' << 8) | 'V'; 717 rpc->function = fn; 718 rpc->rpc_result = 0xffffffff; 719 rpc->rpc_result_private = 0xffffffff; 720 rpc->length = sizeof(*rpc) + argc; 721 return rpc->data; 722 } 723 724 static void * 725 r535_gsp_rpc_push(struct nvkm_gsp *gsp, void *argv, bool wait, u32 repc) 726 { 727 struct nvfw_gsp_rpc *rpc = container_of(argv, typeof(*rpc), data); 728 struct r535_gsp_msg *cmd = container_of((void *)rpc, typeof(*cmd), data); 729 const u32 max_msg_size = (16 * 0x1000) - sizeof(struct r535_gsp_msg); 730 const u32 max_rpc_size = max_msg_size - sizeof(*rpc); 731 u32 rpc_size = rpc->length - sizeof(*rpc); 732 void *repv; 733 734 mutex_lock(&gsp->cmdq.mutex); 735 if (rpc_size > max_rpc_size) { 736 const u32 fn = rpc->function; 737 738 /* Adjust length, and send initial RPC. */ 739 rpc->length = sizeof(*rpc) + max_rpc_size; 740 cmd->checksum = rpc->length; 741 742 repv = r535_gsp_rpc_send(gsp, argv, false, 0); 743 if (IS_ERR(repv)) 744 goto done; 745 746 argv += max_rpc_size; 747 rpc_size -= max_rpc_size; 748 749 /* Remaining chunks sent as CONTINUATION_RECORD RPCs. */ 750 while (rpc_size) { 751 u32 size = min(rpc_size, max_rpc_size); 752 void *next; 753 754 next = r535_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD, size); 755 if (IS_ERR(next)) { 756 repv = next; 757 goto done; 758 } 759 760 memcpy(next, argv, size); 761 762 repv = r535_gsp_rpc_send(gsp, next, false, 0); 763 if (IS_ERR(repv)) 764 goto done; 765 766 argv += size; 767 rpc_size -= size; 768 } 769 770 /* Wait for reply. */ 771 if (wait) { 772 rpc = r535_gsp_msg_recv(gsp, fn, repc); 773 if (!IS_ERR_OR_NULL(rpc)) 774 repv = rpc->data; 775 else 776 repv = rpc; 777 } else { 778 repv = NULL; 779 } 780 } else { 781 repv = r535_gsp_rpc_send(gsp, argv, wait, repc); 782 } 783 784 done: 785 mutex_unlock(&gsp->cmdq.mutex); 786 return repv; 787 } 788 789 const struct nvkm_gsp_rm 790 r535_gsp_rm = { 791 .rpc_get = r535_gsp_rpc_get, 792 .rpc_push = r535_gsp_rpc_push, 793 .rpc_done = r535_gsp_rpc_done, 794 795 .rm_ctrl_get = r535_gsp_rpc_rm_ctrl_get, 796 .rm_ctrl_push = r535_gsp_rpc_rm_ctrl_push, 797 .rm_ctrl_done = r535_gsp_rpc_rm_ctrl_done, 798 799 .rm_alloc_get = r535_gsp_rpc_rm_alloc_get, 800 .rm_alloc_push = r535_gsp_rpc_rm_alloc_push, 801 .rm_alloc_done = r535_gsp_rpc_rm_alloc_done, 802 803 .rm_free = r535_gsp_rpc_rm_free, 804 805 .client_ctor = r535_gsp_client_ctor, 806 .client_dtor = r535_gsp_client_dtor, 807 808 .device_ctor = r535_gsp_device_ctor, 809 .device_dtor = r535_gsp_device_dtor, 810 811 .event_ctor = r535_gsp_device_event_ctor, 812 .event_dtor = r535_gsp_event_dtor, 813 }; 814 815 static void 816 r535_gsp_msgq_work(struct work_struct *work) 817 { 818 struct nvkm_gsp *gsp = container_of(work, typeof(*gsp), msgq.work); 819 820 mutex_lock(&gsp->cmdq.mutex); 821 if (*gsp->msgq.rptr != *gsp->msgq.wptr) 822 r535_gsp_msg_recv(gsp, 0, 0); 823 mutex_unlock(&gsp->cmdq.mutex); 824 } 825 826 static irqreturn_t 827 r535_gsp_intr(struct nvkm_inth *inth) 828 { 829 struct nvkm_gsp *gsp = container_of(inth, typeof(*gsp), subdev.inth); 830 struct nvkm_subdev *subdev = &gsp->subdev; 831 u32 intr = nvkm_falcon_rd32(&gsp->falcon, 0x0008); 832 u32 inte = nvkm_falcon_rd32(&gsp->falcon, gsp->falcon.func->addr2 + 833 gsp->falcon.func->riscv_irqmask); 834 u32 stat = intr & inte; 835 836 if (!stat) { 837 nvkm_debug(subdev, "inte %08x %08x\n", intr, inte); 838 return IRQ_NONE; 839 } 840 841 if (stat & 0x00000040) { 842 nvkm_falcon_wr32(&gsp->falcon, 0x004, 0x00000040); 843 schedule_work(&gsp->msgq.work); 844 stat &= ~0x00000040; 845 } 846 847 if (stat) { 848 nvkm_error(subdev, "intr %08x\n", stat); 849 nvkm_falcon_wr32(&gsp->falcon, 0x014, stat); 850 nvkm_falcon_wr32(&gsp->falcon, 0x004, stat); 851 } 852 853 nvkm_falcon_intr_retrigger(&gsp->falcon); 854 return IRQ_HANDLED; 855 } 856 857 static int 858 r535_gsp_intr_get_table(struct nvkm_gsp *gsp) 859 { 860 NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_PARAMS *ctrl; 861 int ret = 0; 862 863 ctrl = nvkm_gsp_rm_ctrl_get(&gsp->internal.device.subdevice, 864 NV2080_CTRL_CMD_INTERNAL_INTR_GET_KERNEL_TABLE, sizeof(*ctrl)); 865 if (IS_ERR(ctrl)) 866 return PTR_ERR(ctrl); 867 868 ret = nvkm_gsp_rm_ctrl_push(&gsp->internal.device.subdevice, &ctrl, sizeof(*ctrl)); 869 if (WARN_ON(ret)) { 870 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl); 871 return ret; 872 } 873 874 for (unsigned i = 0; i < ctrl->tableLen; i++) { 875 enum nvkm_subdev_type type; 876 int inst; 877 878 nvkm_debug(&gsp->subdev, 879 "%2d: engineIdx %3d pmcIntrMask %08x stall %08x nonStall %08x\n", i, 880 ctrl->table[i].engineIdx, ctrl->table[i].pmcIntrMask, 881 ctrl->table[i].vectorStall, ctrl->table[i].vectorNonStall); 882 883 switch (ctrl->table[i].engineIdx) { 884 case MC_ENGINE_IDX_GSP: 885 type = NVKM_SUBDEV_GSP; 886 inst = 0; 887 break; 888 case MC_ENGINE_IDX_DISP: 889 type = NVKM_ENGINE_DISP; 890 inst = 0; 891 break; 892 case MC_ENGINE_IDX_CE0 ... MC_ENGINE_IDX_CE9: 893 type = NVKM_ENGINE_CE; 894 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_CE0; 895 break; 896 case MC_ENGINE_IDX_GR0: 897 type = NVKM_ENGINE_GR; 898 inst = 0; 899 break; 900 case MC_ENGINE_IDX_NVDEC0 ... MC_ENGINE_IDX_NVDEC7: 901 type = NVKM_ENGINE_NVDEC; 902 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_NVDEC0; 903 break; 904 case MC_ENGINE_IDX_MSENC ... MC_ENGINE_IDX_MSENC2: 905 type = NVKM_ENGINE_NVENC; 906 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_MSENC; 907 break; 908 case MC_ENGINE_IDX_NVJPEG0 ... MC_ENGINE_IDX_NVJPEG7: 909 type = NVKM_ENGINE_NVJPG; 910 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_NVJPEG0; 911 break; 912 case MC_ENGINE_IDX_OFA0: 913 type = NVKM_ENGINE_OFA; 914 inst = 0; 915 break; 916 default: 917 continue; 918 } 919 920 if (WARN_ON(gsp->intr_nr == ARRAY_SIZE(gsp->intr))) { 921 ret = -ENOSPC; 922 break; 923 } 924 925 gsp->intr[gsp->intr_nr].type = type; 926 gsp->intr[gsp->intr_nr].inst = inst; 927 gsp->intr[gsp->intr_nr].stall = ctrl->table[i].vectorStall; 928 gsp->intr[gsp->intr_nr].nonstall = ctrl->table[i].vectorNonStall; 929 gsp->intr_nr++; 930 } 931 932 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl); 933 return ret; 934 } 935 936 static int 937 r535_gsp_rpc_get_gsp_static_info(struct nvkm_gsp *gsp) 938 { 939 GspStaticConfigInfo *rpc; 940 int last_usable = -1; 941 942 rpc = nvkm_gsp_rpc_rd(gsp, NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO, sizeof(*rpc)); 943 if (IS_ERR(rpc)) 944 return PTR_ERR(rpc); 945 946 gsp->internal.client.object.client = &gsp->internal.client; 947 gsp->internal.client.object.parent = NULL; 948 gsp->internal.client.object.handle = rpc->hInternalClient; 949 gsp->internal.client.gsp = gsp; 950 951 gsp->internal.device.object.client = &gsp->internal.client; 952 gsp->internal.device.object.parent = &gsp->internal.client.object; 953 gsp->internal.device.object.handle = rpc->hInternalDevice; 954 955 gsp->internal.device.subdevice.client = &gsp->internal.client; 956 gsp->internal.device.subdevice.parent = &gsp->internal.device.object; 957 gsp->internal.device.subdevice.handle = rpc->hInternalSubdevice; 958 959 gsp->bar.rm_bar1_pdb = rpc->bar1PdeBase; 960 gsp->bar.rm_bar2_pdb = rpc->bar2PdeBase; 961 962 for (int i = 0; i < rpc->fbRegionInfoParams.numFBRegions; i++) { 963 NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO *reg = 964 &rpc->fbRegionInfoParams.fbRegion[i]; 965 966 nvkm_debug(&gsp->subdev, "fb region %d: " 967 "%016llx-%016llx rsvd:%016llx perf:%08x comp:%d iso:%d prot:%d\n", i, 968 reg->base, reg->limit, reg->reserved, reg->performance, 969 reg->supportCompressed, reg->supportISO, reg->bProtected); 970 971 if (!reg->reserved && !reg->bProtected) { 972 if (reg->supportCompressed && reg->supportISO && 973 !WARN_ON_ONCE(gsp->fb.region_nr >= ARRAY_SIZE(gsp->fb.region))) { 974 const u64 size = (reg->limit + 1) - reg->base; 975 976 gsp->fb.region[gsp->fb.region_nr].addr = reg->base; 977 gsp->fb.region[gsp->fb.region_nr].size = size; 978 gsp->fb.region_nr++; 979 } 980 981 last_usable = i; 982 } 983 } 984 985 if (last_usable >= 0) { 986 u32 rsvd_base = rpc->fbRegionInfoParams.fbRegion[last_usable].limit + 1; 987 988 gsp->fb.rsvd_size = gsp->fb.heap.addr - rsvd_base; 989 } 990 991 for (int gpc = 0; gpc < ARRAY_SIZE(rpc->tpcInfo); gpc++) { 992 if (rpc->gpcInfo.gpcMask & BIT(gpc)) { 993 gsp->gr.tpcs += hweight32(rpc->tpcInfo[gpc].tpcMask); 994 gsp->gr.gpcs++; 995 } 996 } 997 998 nvkm_gsp_rpc_done(gsp, rpc); 999 return 0; 1000 } 1001 1002 static void 1003 nvkm_gsp_mem_dtor(struct nvkm_gsp *gsp, struct nvkm_gsp_mem *mem) 1004 { 1005 if (mem->data) { 1006 /* 1007 * Poison the buffer to catch any unexpected access from 1008 * GSP-RM if the buffer was prematurely freed. 1009 */ 1010 memset(mem->data, 0xFF, mem->size); 1011 1012 dma_free_coherent(gsp->subdev.device->dev, mem->size, mem->data, mem->addr); 1013 memset(mem, 0, sizeof(*mem)); 1014 } 1015 } 1016 1017 static int 1018 nvkm_gsp_mem_ctor(struct nvkm_gsp *gsp, size_t size, struct nvkm_gsp_mem *mem) 1019 { 1020 mem->size = size; 1021 mem->data = dma_alloc_coherent(gsp->subdev.device->dev, size, &mem->addr, GFP_KERNEL); 1022 if (WARN_ON(!mem->data)) 1023 return -ENOMEM; 1024 1025 return 0; 1026 } 1027 1028 static int 1029 r535_gsp_postinit(struct nvkm_gsp *gsp) 1030 { 1031 struct nvkm_device *device = gsp->subdev.device; 1032 int ret; 1033 1034 ret = r535_gsp_rpc_get_gsp_static_info(gsp); 1035 if (WARN_ON(ret)) 1036 return ret; 1037 1038 INIT_WORK(&gsp->msgq.work, r535_gsp_msgq_work); 1039 1040 ret = r535_gsp_intr_get_table(gsp); 1041 if (WARN_ON(ret)) 1042 return ret; 1043 1044 ret = nvkm_gsp_intr_stall(gsp, gsp->subdev.type, gsp->subdev.inst); 1045 if (WARN_ON(ret < 0)) 1046 return ret; 1047 1048 ret = nvkm_inth_add(&device->vfn->intr, ret, NVKM_INTR_PRIO_NORMAL, &gsp->subdev, 1049 r535_gsp_intr, &gsp->subdev.inth); 1050 if (WARN_ON(ret)) 1051 return ret; 1052 1053 nvkm_inth_allow(&gsp->subdev.inth); 1054 nvkm_wr32(device, 0x110004, 0x00000040); 1055 1056 /* Release the DMA buffers that were needed only for boot and init */ 1057 nvkm_gsp_mem_dtor(gsp, &gsp->boot.fw); 1058 nvkm_gsp_mem_dtor(gsp, &gsp->libos); 1059 1060 return ret; 1061 } 1062 1063 static int 1064 r535_gsp_rpc_unloading_guest_driver(struct nvkm_gsp *gsp, bool suspend) 1065 { 1066 rpc_unloading_guest_driver_v1F_07 *rpc; 1067 1068 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER, sizeof(*rpc)); 1069 if (IS_ERR(rpc)) 1070 return PTR_ERR(rpc); 1071 1072 if (suspend) { 1073 rpc->bInPMTransition = 1; 1074 rpc->bGc6Entering = 0; 1075 rpc->newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3; 1076 } else { 1077 rpc->bInPMTransition = 0; 1078 rpc->bGc6Entering = 0; 1079 rpc->newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0; 1080 } 1081 1082 return nvkm_gsp_rpc_wr(gsp, rpc, true); 1083 } 1084 1085 enum registry_type { 1086 REGISTRY_TABLE_ENTRY_TYPE_DWORD = 1, /* 32-bit unsigned integer */ 1087 REGISTRY_TABLE_ENTRY_TYPE_BINARY = 2, /* Binary blob */ 1088 REGISTRY_TABLE_ENTRY_TYPE_STRING = 3, /* Null-terminated string */ 1089 }; 1090 1091 /* An arbitrary limit to the length of a registry key */ 1092 #define REGISTRY_MAX_KEY_LENGTH 64 1093 1094 /** 1095 * registry_list_entry - linked list member for a registry key/value 1096 * @head: list_head struct 1097 * @type: dword, binary, or string 1098 * @klen: the length of name of the key 1099 * @vlen: the length of the value 1100 * @key: the key name 1101 * @dword: the data, if REGISTRY_TABLE_ENTRY_TYPE_DWORD 1102 * @binary: the data, if TYPE_BINARY or TYPE_STRING 1103 * 1104 * Every registry key/value is represented internally by this struct. 1105 * 1106 * Type DWORD is a simple 32-bit unsigned integer, and its value is stored in 1107 * @dword. 1108 * 1109 * Types BINARY and STRING are variable-length binary blobs. The only real 1110 * difference between BINARY and STRING is that STRING is null-terminated and 1111 * is expected to contain only printable characters. 1112 * 1113 * Note: it is technically possible to have multiple keys with the same name 1114 * but different types, but this is not useful since GSP-RM expects keys to 1115 * have only one specific type. 1116 */ 1117 struct registry_list_entry { 1118 struct list_head head; 1119 enum registry_type type; 1120 size_t klen; 1121 char key[REGISTRY_MAX_KEY_LENGTH]; 1122 size_t vlen; 1123 u32 dword; /* TYPE_DWORD */ 1124 u8 binary[] __counted_by(vlen); /* TYPE_BINARY or TYPE_STRING */ 1125 }; 1126 1127 /** 1128 * add_registry -- adds a registry entry 1129 * @gsp: gsp pointer 1130 * @key: name of the registry key 1131 * @type: type of data 1132 * @data: pointer to value 1133 * @length: size of data, in bytes 1134 * 1135 * Adds a registry key/value pair to the registry database. 1136 * 1137 * This function collects the registry information in a linked list. After 1138 * all registry keys have been added, build_registry() is used to create the 1139 * RPC data structure. 1140 * 1141 * registry_rpc_size is a running total of the size of all registry keys. 1142 * It's used to avoid an O(n) calculation of the size when the RPC is built. 1143 * 1144 * Returns 0 on success, or negative error code on error. 1145 */ 1146 static int add_registry(struct nvkm_gsp *gsp, const char *key, 1147 enum registry_type type, const void *data, size_t length) 1148 { 1149 struct registry_list_entry *reg; 1150 const size_t nlen = strnlen(key, REGISTRY_MAX_KEY_LENGTH) + 1; 1151 size_t alloc_size; /* extra bytes to alloc for binary or string value */ 1152 1153 if (nlen > REGISTRY_MAX_KEY_LENGTH) 1154 return -EINVAL; 1155 1156 alloc_size = (type == REGISTRY_TABLE_ENTRY_TYPE_DWORD) ? 0 : length; 1157 1158 reg = kmalloc(sizeof(*reg) + alloc_size, GFP_KERNEL); 1159 if (!reg) 1160 return -ENOMEM; 1161 1162 switch (type) { 1163 case REGISTRY_TABLE_ENTRY_TYPE_DWORD: 1164 reg->dword = *(const u32 *)(data); 1165 break; 1166 case REGISTRY_TABLE_ENTRY_TYPE_BINARY: 1167 case REGISTRY_TABLE_ENTRY_TYPE_STRING: 1168 memcpy(reg->binary, data, alloc_size); 1169 break; 1170 default: 1171 nvkm_error(&gsp->subdev, "unrecognized registry type %u for '%s'\n", 1172 type, key); 1173 kfree(reg); 1174 return -EINVAL; 1175 } 1176 1177 memcpy(reg->key, key, nlen); 1178 reg->klen = nlen; 1179 reg->vlen = length; 1180 reg->type = type; 1181 1182 list_add_tail(®->head, &gsp->registry_list); 1183 gsp->registry_rpc_size += sizeof(PACKED_REGISTRY_ENTRY) + nlen + alloc_size; 1184 1185 return 0; 1186 } 1187 1188 static int add_registry_num(struct nvkm_gsp *gsp, const char *key, u32 value) 1189 { 1190 return add_registry(gsp, key, REGISTRY_TABLE_ENTRY_TYPE_DWORD, 1191 &value, sizeof(u32)); 1192 } 1193 1194 static int add_registry_string(struct nvkm_gsp *gsp, const char *key, const char *value) 1195 { 1196 return add_registry(gsp, key, REGISTRY_TABLE_ENTRY_TYPE_STRING, 1197 value, strlen(value) + 1); 1198 } 1199 1200 /** 1201 * build_registry -- create the registry RPC data 1202 * @gsp: gsp pointer 1203 * @registry: pointer to the RPC payload to fill 1204 * 1205 * After all registry key/value pairs have been added, call this function to 1206 * build the RPC. 1207 * 1208 * The registry RPC looks like this: 1209 * 1210 * +-----------------+ 1211 * |NvU32 size; | 1212 * |NvU32 numEntries;| 1213 * +-----------------+ 1214 * +----------------------------------------+ 1215 * |PACKED_REGISTRY_ENTRY | 1216 * +----------------------------------------+ 1217 * |Null-terminated key (string) for entry 0| 1218 * +----------------------------------------+ 1219 * |Binary/string data value for entry 0 | (only if necessary) 1220 * +----------------------------------------+ 1221 * 1222 * +----------------------------------------+ 1223 * |PACKED_REGISTRY_ENTRY | 1224 * +----------------------------------------+ 1225 * |Null-terminated key (string) for entry 1| 1226 * +----------------------------------------+ 1227 * |Binary/string data value for entry 1 | (only if necessary) 1228 * +----------------------------------------+ 1229 * ... (and so on, one copy for each entry) 1230 * 1231 * 1232 * The 'data' field of an entry is either a 32-bit integer (for type DWORD) 1233 * or an offset into the PACKED_REGISTRY_TABLE (for types BINARY and STRING). 1234 * 1235 * All memory allocated by add_registry() is released. 1236 */ 1237 static void build_registry(struct nvkm_gsp *gsp, PACKED_REGISTRY_TABLE *registry) 1238 { 1239 struct registry_list_entry *reg, *n; 1240 size_t str_offset; 1241 unsigned int i = 0; 1242 1243 registry->numEntries = list_count_nodes(&gsp->registry_list); 1244 str_offset = struct_size(registry, entries, registry->numEntries); 1245 1246 list_for_each_entry_safe(reg, n, &gsp->registry_list, head) { 1247 registry->entries[i].type = reg->type; 1248 registry->entries[i].length = reg->vlen; 1249 1250 /* Append the key name to the table */ 1251 registry->entries[i].nameOffset = str_offset; 1252 memcpy((void *)registry + str_offset, reg->key, reg->klen); 1253 str_offset += reg->klen; 1254 1255 switch (reg->type) { 1256 case REGISTRY_TABLE_ENTRY_TYPE_DWORD: 1257 registry->entries[i].data = reg->dword; 1258 break; 1259 case REGISTRY_TABLE_ENTRY_TYPE_BINARY: 1260 case REGISTRY_TABLE_ENTRY_TYPE_STRING: 1261 /* If the type is binary or string, also append the value */ 1262 memcpy((void *)registry + str_offset, reg->binary, reg->vlen); 1263 registry->entries[i].data = str_offset; 1264 str_offset += reg->vlen; 1265 break; 1266 default: 1267 break; 1268 } 1269 1270 i++; 1271 list_del(®->head); 1272 kfree(reg); 1273 } 1274 1275 /* Double-check that we calculated the sizes correctly */ 1276 WARN_ON(gsp->registry_rpc_size != str_offset); 1277 1278 registry->size = gsp->registry_rpc_size; 1279 } 1280 1281 /** 1282 * clean_registry -- clean up registry memory in case of error 1283 * @gsp: gsp pointer 1284 * 1285 * Call this function to clean up all memory allocated by add_registry() 1286 * in case of error and build_registry() is not called. 1287 */ 1288 static void clean_registry(struct nvkm_gsp *gsp) 1289 { 1290 struct registry_list_entry *reg, *n; 1291 1292 list_for_each_entry_safe(reg, n, &gsp->registry_list, head) { 1293 list_del(®->head); 1294 kfree(reg); 1295 } 1296 1297 gsp->registry_rpc_size = sizeof(PACKED_REGISTRY_TABLE); 1298 } 1299 1300 MODULE_PARM_DESC(NVreg_RegistryDwords, 1301 "A semicolon-separated list of key=integer pairs of GSP-RM registry keys"); 1302 static char *NVreg_RegistryDwords; 1303 module_param(NVreg_RegistryDwords, charp, 0400); 1304 1305 /* dword only */ 1306 struct nv_gsp_registry_entries { 1307 const char *name; 1308 u32 value; 1309 }; 1310 1311 /** 1312 * r535_registry_entries - required registry entries for GSP-RM 1313 * 1314 * This array lists registry entries that are required for GSP-RM to 1315 * function correctly. 1316 * 1317 * RMSecBusResetEnable - enables PCI secondary bus reset 1318 * RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration 1319 * registers on any PCI reset. 1320 */ 1321 static const struct nv_gsp_registry_entries r535_registry_entries[] = { 1322 { "RMSecBusResetEnable", 1 }, 1323 { "RMForcePcieConfigSave", 1 }, 1324 }; 1325 #define NV_GSP_REG_NUM_ENTRIES ARRAY_SIZE(r535_registry_entries) 1326 1327 /** 1328 * strip - strips all characters in 'reject' from 's' 1329 * @s: string to strip 1330 * @reject: string of characters to remove 1331 * 1332 * 's' is modified. 1333 * 1334 * Returns the length of the new string. 1335 */ 1336 static size_t strip(char *s, const char *reject) 1337 { 1338 char *p = s, *p2 = s; 1339 size_t length = 0; 1340 char c; 1341 1342 do { 1343 while ((c = *p2) && strchr(reject, c)) 1344 p2++; 1345 1346 *p++ = c = *p2++; 1347 length++; 1348 } while (c); 1349 1350 return length; 1351 } 1352 1353 /** 1354 * r535_gsp_rpc_set_registry - build registry RPC and call GSP-RM 1355 * @gsp: gsp pointer 1356 * 1357 * The GSP-RM registry is a set of key/value pairs that configure some aspects 1358 * of GSP-RM. The keys are strings, and the values are 32-bit integers. 1359 * 1360 * The registry is built from a combination of a static hard-coded list (see 1361 * above) and entries passed on the driver's command line. 1362 */ 1363 static int 1364 r535_gsp_rpc_set_registry(struct nvkm_gsp *gsp) 1365 { 1366 PACKED_REGISTRY_TABLE *rpc; 1367 unsigned int i; 1368 int ret; 1369 1370 INIT_LIST_HEAD(&gsp->registry_list); 1371 gsp->registry_rpc_size = sizeof(PACKED_REGISTRY_TABLE); 1372 1373 for (i = 0; i < NV_GSP_REG_NUM_ENTRIES; i++) { 1374 ret = add_registry_num(gsp, r535_registry_entries[i].name, 1375 r535_registry_entries[i].value); 1376 if (ret) 1377 goto fail; 1378 } 1379 1380 /* 1381 * The NVreg_RegistryDwords parameter is a string of key=value 1382 * pairs separated by semicolons. We need to extract and trim each 1383 * substring, and then parse the substring to extract the key and 1384 * value. 1385 */ 1386 if (NVreg_RegistryDwords) { 1387 char *p = kstrdup(NVreg_RegistryDwords, GFP_KERNEL); 1388 char *start, *next = p, *equal; 1389 1390 if (!p) { 1391 ret = -ENOMEM; 1392 goto fail; 1393 } 1394 1395 /* Remove any whitespace from the parameter string */ 1396 strip(p, " \t\n"); 1397 1398 while ((start = strsep(&next, ";"))) { 1399 long value; 1400 1401 equal = strchr(start, '='); 1402 if (!equal || equal == start || equal[1] == 0) { 1403 nvkm_error(&gsp->subdev, 1404 "ignoring invalid registry string '%s'\n", 1405 start); 1406 continue; 1407 } 1408 1409 /* Truncate the key=value string to just key */ 1410 *equal = 0; 1411 1412 ret = kstrtol(equal + 1, 0, &value); 1413 if (!ret) { 1414 ret = add_registry_num(gsp, start, value); 1415 } else { 1416 /* Not a number, so treat it as a string */ 1417 ret = add_registry_string(gsp, start, equal + 1); 1418 } 1419 1420 if (ret) { 1421 nvkm_error(&gsp->subdev, 1422 "ignoring invalid registry key/value '%s=%s'\n", 1423 start, equal + 1); 1424 continue; 1425 } 1426 } 1427 1428 kfree(p); 1429 } 1430 1431 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_SET_REGISTRY, gsp->registry_rpc_size); 1432 if (IS_ERR(rpc)) { 1433 ret = PTR_ERR(rpc); 1434 goto fail; 1435 } 1436 1437 build_registry(gsp, rpc); 1438 1439 return nvkm_gsp_rpc_wr(gsp, rpc, false); 1440 1441 fail: 1442 clean_registry(gsp); 1443 return ret; 1444 } 1445 1446 #if defined(CONFIG_ACPI) && defined(CONFIG_X86) 1447 static void 1448 r535_gsp_acpi_caps(acpi_handle handle, CAPS_METHOD_DATA *caps) 1449 { 1450 const guid_t NVOP_DSM_GUID = 1451 GUID_INIT(0xA486D8F8, 0x0BDA, 0x471B, 1452 0xA7, 0x2B, 0x60, 0x42, 0xA6, 0xB5, 0xBE, 0xE0); 1453 u64 NVOP_DSM_REV = 0x00000100; 1454 union acpi_object argv4 = { 1455 .buffer.type = ACPI_TYPE_BUFFER, 1456 .buffer.length = 4, 1457 .buffer.pointer = kmalloc(argv4.buffer.length, GFP_KERNEL), 1458 }, *obj; 1459 1460 caps->status = 0xffff; 1461 1462 if (!acpi_check_dsm(handle, &NVOP_DSM_GUID, NVOP_DSM_REV, BIT_ULL(0x1a))) 1463 return; 1464 1465 obj = acpi_evaluate_dsm(handle, &NVOP_DSM_GUID, NVOP_DSM_REV, 0x1a, &argv4); 1466 if (!obj) 1467 return; 1468 1469 if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) || 1470 WARN_ON(obj->buffer.length != 4)) 1471 return; 1472 1473 caps->status = 0; 1474 caps->optimusCaps = *(u32 *)obj->buffer.pointer; 1475 1476 ACPI_FREE(obj); 1477 1478 kfree(argv4.buffer.pointer); 1479 } 1480 1481 static void 1482 r535_gsp_acpi_jt(acpi_handle handle, JT_METHOD_DATA *jt) 1483 { 1484 const guid_t JT_DSM_GUID = 1485 GUID_INIT(0xCBECA351L, 0x067B, 0x4924, 1486 0x9C, 0xBD, 0xB4, 0x6B, 0x00, 0xB8, 0x6F, 0x34); 1487 u64 JT_DSM_REV = 0x00000103; 1488 u32 caps; 1489 union acpi_object argv4 = { 1490 .buffer.type = ACPI_TYPE_BUFFER, 1491 .buffer.length = sizeof(caps), 1492 .buffer.pointer = kmalloc(argv4.buffer.length, GFP_KERNEL), 1493 }, *obj; 1494 1495 jt->status = 0xffff; 1496 1497 obj = acpi_evaluate_dsm(handle, &JT_DSM_GUID, JT_DSM_REV, 0x1, &argv4); 1498 if (!obj) 1499 return; 1500 1501 if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) || 1502 WARN_ON(obj->buffer.length != 4)) 1503 return; 1504 1505 jt->status = 0; 1506 jt->jtCaps = *(u32 *)obj->buffer.pointer; 1507 jt->jtRevId = (jt->jtCaps & 0xfff00000) >> 20; 1508 jt->bSBIOSCaps = 0; 1509 1510 ACPI_FREE(obj); 1511 1512 kfree(argv4.buffer.pointer); 1513 } 1514 1515 static void 1516 r535_gsp_acpi_mux_id(acpi_handle handle, u32 id, MUX_METHOD_DATA_ELEMENT *mode, 1517 MUX_METHOD_DATA_ELEMENT *part) 1518 { 1519 union acpi_object mux_arg = { ACPI_TYPE_INTEGER }; 1520 struct acpi_object_list input = { 1, &mux_arg }; 1521 acpi_handle iter = NULL, handle_mux = NULL; 1522 acpi_status status; 1523 unsigned long long value; 1524 1525 mode->status = 0xffff; 1526 part->status = 0xffff; 1527 1528 do { 1529 status = acpi_get_next_object(ACPI_TYPE_DEVICE, handle, iter, &iter); 1530 if (ACPI_FAILURE(status) || !iter) 1531 return; 1532 1533 status = acpi_evaluate_integer(iter, "_ADR", NULL, &value); 1534 if (ACPI_FAILURE(status) || value != id) 1535 continue; 1536 1537 handle_mux = iter; 1538 } while (!handle_mux); 1539 1540 if (!handle_mux) 1541 return; 1542 1543 /* I -think- 0 means "acquire" according to nvidia's driver source */ 1544 input.pointer->integer.type = ACPI_TYPE_INTEGER; 1545 input.pointer->integer.value = 0; 1546 1547 status = acpi_evaluate_integer(handle_mux, "MXDM", &input, &value); 1548 if (ACPI_SUCCESS(status)) { 1549 mode->acpiId = id; 1550 mode->mode = value; 1551 mode->status = 0; 1552 } 1553 1554 status = acpi_evaluate_integer(handle_mux, "MXDS", &input, &value); 1555 if (ACPI_SUCCESS(status)) { 1556 part->acpiId = id; 1557 part->mode = value; 1558 part->status = 0; 1559 } 1560 } 1561 1562 static void 1563 r535_gsp_acpi_mux(acpi_handle handle, DOD_METHOD_DATA *dod, MUX_METHOD_DATA *mux) 1564 { 1565 mux->tableLen = dod->acpiIdListLen / sizeof(dod->acpiIdList[0]); 1566 1567 for (int i = 0; i < mux->tableLen; i++) { 1568 r535_gsp_acpi_mux_id(handle, dod->acpiIdList[i], &mux->acpiIdMuxModeTable[i], 1569 &mux->acpiIdMuxPartTable[i]); 1570 } 1571 } 1572 1573 static void 1574 r535_gsp_acpi_dod(acpi_handle handle, DOD_METHOD_DATA *dod) 1575 { 1576 acpi_status status; 1577 struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL }; 1578 union acpi_object *_DOD; 1579 1580 dod->status = 0xffff; 1581 1582 status = acpi_evaluate_object(handle, "_DOD", NULL, &output); 1583 if (ACPI_FAILURE(status)) 1584 return; 1585 1586 _DOD = output.pointer; 1587 1588 if (WARN_ON(_DOD->type != ACPI_TYPE_PACKAGE) || 1589 WARN_ON(_DOD->package.count > ARRAY_SIZE(dod->acpiIdList))) 1590 return; 1591 1592 for (int i = 0; i < _DOD->package.count; i++) { 1593 if (WARN_ON(_DOD->package.elements[i].type != ACPI_TYPE_INTEGER)) 1594 return; 1595 1596 dod->acpiIdList[i] = _DOD->package.elements[i].integer.value; 1597 dod->acpiIdListLen += sizeof(dod->acpiIdList[0]); 1598 } 1599 1600 dod->status = 0; 1601 kfree(output.pointer); 1602 } 1603 #endif 1604 1605 static void 1606 r535_gsp_acpi_info(struct nvkm_gsp *gsp, ACPI_METHOD_DATA *acpi) 1607 { 1608 #if defined(CONFIG_ACPI) && defined(CONFIG_X86) 1609 acpi_handle handle = ACPI_HANDLE(gsp->subdev.device->dev); 1610 1611 if (!handle) 1612 return; 1613 1614 acpi->bValid = 1; 1615 1616 r535_gsp_acpi_dod(handle, &acpi->dodMethodData); 1617 if (acpi->dodMethodData.status == 0) 1618 r535_gsp_acpi_mux(handle, &acpi->dodMethodData, &acpi->muxMethodData); 1619 1620 r535_gsp_acpi_jt(handle, &acpi->jtMethodData); 1621 r535_gsp_acpi_caps(handle, &acpi->capsMethodData); 1622 #endif 1623 } 1624 1625 static int 1626 r535_gsp_rpc_set_system_info(struct nvkm_gsp *gsp) 1627 { 1628 struct nvkm_device *device = gsp->subdev.device; 1629 struct nvkm_device_pci *pdev = container_of(device, typeof(*pdev), device); 1630 GspSystemInfo *info; 1631 1632 if (WARN_ON(device->type == NVKM_DEVICE_TEGRA)) 1633 return -ENOSYS; 1634 1635 info = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO, sizeof(*info)); 1636 if (IS_ERR(info)) 1637 return PTR_ERR(info); 1638 1639 info->gpuPhysAddr = device->func->resource_addr(device, 0); 1640 info->gpuPhysFbAddr = device->func->resource_addr(device, 1); 1641 info->gpuPhysInstAddr = device->func->resource_addr(device, 3); 1642 info->nvDomainBusDeviceFunc = pci_dev_id(pdev->pdev); 1643 info->maxUserVa = TASK_SIZE; 1644 info->pciConfigMirrorBase = 0x088000; 1645 info->pciConfigMirrorSize = 0x001000; 1646 r535_gsp_acpi_info(gsp, &info->acpiMethodData); 1647 1648 return nvkm_gsp_rpc_wr(gsp, info, false); 1649 } 1650 1651 static int 1652 r535_gsp_msg_os_error_log(void *priv, u32 fn, void *repv, u32 repc) 1653 { 1654 struct nvkm_gsp *gsp = priv; 1655 struct nvkm_subdev *subdev = &gsp->subdev; 1656 rpc_os_error_log_v17_00 *msg = repv; 1657 1658 if (WARN_ON(repc < sizeof(*msg))) 1659 return -EINVAL; 1660 1661 nvkm_error(subdev, "Xid:%d %s\n", msg->exceptType, msg->errString); 1662 return 0; 1663 } 1664 1665 static int 1666 r535_gsp_msg_rc_triggered(void *priv, u32 fn, void *repv, u32 repc) 1667 { 1668 rpc_rc_triggered_v17_02 *msg = repv; 1669 struct nvkm_gsp *gsp = priv; 1670 struct nvkm_subdev *subdev = &gsp->subdev; 1671 struct nvkm_chan *chan; 1672 unsigned long flags; 1673 1674 if (WARN_ON(repc < sizeof(*msg))) 1675 return -EINVAL; 1676 1677 nvkm_error(subdev, "rc engn:%08x chid:%d type:%d scope:%d part:%d\n", 1678 msg->nv2080EngineType, msg->chid, msg->exceptType, msg->scope, 1679 msg->partitionAttributionId); 1680 1681 chan = nvkm_chan_get_chid(&subdev->device->fifo->engine, msg->chid / 8, &flags); 1682 if (!chan) { 1683 nvkm_error(subdev, "rc chid:%d not found!\n", msg->chid); 1684 return 0; 1685 } 1686 1687 nvkm_chan_error(chan, false); 1688 nvkm_chan_put(&chan, flags); 1689 return 0; 1690 } 1691 1692 static int 1693 r535_gsp_msg_mmu_fault_queued(void *priv, u32 fn, void *repv, u32 repc) 1694 { 1695 struct nvkm_gsp *gsp = priv; 1696 struct nvkm_subdev *subdev = &gsp->subdev; 1697 1698 WARN_ON(repc != 0); 1699 1700 nvkm_error(subdev, "mmu fault queued\n"); 1701 return 0; 1702 } 1703 1704 static int 1705 r535_gsp_msg_post_event(void *priv, u32 fn, void *repv, u32 repc) 1706 { 1707 struct nvkm_gsp *gsp = priv; 1708 struct nvkm_gsp_client *client; 1709 struct nvkm_subdev *subdev = &gsp->subdev; 1710 rpc_post_event_v17_00 *msg = repv; 1711 1712 if (WARN_ON(repc < sizeof(*msg))) 1713 return -EINVAL; 1714 if (WARN_ON(repc != sizeof(*msg) + msg->eventDataSize)) 1715 return -EINVAL; 1716 1717 nvkm_debug(subdev, "event: %08x %08x %d %08x %08x %d %d\n", 1718 msg->hClient, msg->hEvent, msg->notifyIndex, msg->data, 1719 msg->status, msg->eventDataSize, msg->bNotifyList); 1720 1721 mutex_lock(&gsp->client_id.mutex); 1722 client = idr_find(&gsp->client_id.idr, msg->hClient & 0xffff); 1723 if (client) { 1724 struct nvkm_gsp_event *event; 1725 bool handled = false; 1726 1727 list_for_each_entry(event, &client->events, head) { 1728 if (event->object.handle == msg->hEvent) { 1729 event->func(event, msg->eventData, msg->eventDataSize); 1730 handled = true; 1731 } 1732 } 1733 1734 if (!handled) { 1735 nvkm_error(subdev, "event: cid 0x%08x event 0x%08x not found!\n", 1736 msg->hClient, msg->hEvent); 1737 } 1738 } else { 1739 nvkm_error(subdev, "event: cid 0x%08x not found!\n", msg->hClient); 1740 } 1741 mutex_unlock(&gsp->client_id.mutex); 1742 return 0; 1743 } 1744 1745 /** 1746 * r535_gsp_msg_run_cpu_sequencer() -- process I/O commands from the GSP 1747 * @priv: gsp pointer 1748 * @fn: function number (ignored) 1749 * @repv: pointer to libos print RPC 1750 * @repc: message size 1751 * 1752 * The GSP sequencer is a list of I/O commands that the GSP can send to 1753 * the driver to perform for various purposes. The most common usage is to 1754 * perform a special mid-initialization reset. 1755 */ 1756 static int 1757 r535_gsp_msg_run_cpu_sequencer(void *priv, u32 fn, void *repv, u32 repc) 1758 { 1759 struct nvkm_gsp *gsp = priv; 1760 struct nvkm_subdev *subdev = &gsp->subdev; 1761 struct nvkm_device *device = subdev->device; 1762 rpc_run_cpu_sequencer_v17_00 *seq = repv; 1763 int ptr = 0, ret; 1764 1765 nvkm_debug(subdev, "seq: %08x %08x\n", seq->bufferSizeDWord, seq->cmdIndex); 1766 1767 while (ptr < seq->cmdIndex) { 1768 GSP_SEQUENCER_BUFFER_CMD *cmd = (void *)&seq->commandBuffer[ptr]; 1769 1770 ptr += 1; 1771 ptr += GSP_SEQUENCER_PAYLOAD_SIZE_DWORDS(cmd->opCode); 1772 1773 switch (cmd->opCode) { 1774 case GSP_SEQ_BUF_OPCODE_REG_WRITE: { 1775 u32 addr = cmd->payload.regWrite.addr; 1776 u32 data = cmd->payload.regWrite.val; 1777 1778 nvkm_trace(subdev, "seq wr32 %06x %08x\n", addr, data); 1779 nvkm_wr32(device, addr, data); 1780 } 1781 break; 1782 case GSP_SEQ_BUF_OPCODE_REG_MODIFY: { 1783 u32 addr = cmd->payload.regModify.addr; 1784 u32 mask = cmd->payload.regModify.mask; 1785 u32 data = cmd->payload.regModify.val; 1786 1787 nvkm_trace(subdev, "seq mask %06x %08x %08x\n", addr, mask, data); 1788 nvkm_mask(device, addr, mask, data); 1789 } 1790 break; 1791 case GSP_SEQ_BUF_OPCODE_REG_POLL: { 1792 u32 addr = cmd->payload.regPoll.addr; 1793 u32 mask = cmd->payload.regPoll.mask; 1794 u32 data = cmd->payload.regPoll.val; 1795 u32 usec = cmd->payload.regPoll.timeout ?: 4000000; 1796 //u32 error = cmd->payload.regPoll.error; 1797 1798 nvkm_trace(subdev, "seq poll %06x %08x %08x %d\n", addr, mask, data, usec); 1799 nvkm_rd32(device, addr); 1800 nvkm_usec(device, usec, 1801 if ((nvkm_rd32(device, addr) & mask) == data) 1802 break; 1803 ); 1804 } 1805 break; 1806 case GSP_SEQ_BUF_OPCODE_DELAY_US: { 1807 u32 usec = cmd->payload.delayUs.val; 1808 1809 nvkm_trace(subdev, "seq usec %d\n", usec); 1810 udelay(usec); 1811 } 1812 break; 1813 case GSP_SEQ_BUF_OPCODE_REG_STORE: { 1814 u32 addr = cmd->payload.regStore.addr; 1815 u32 slot = cmd->payload.regStore.index; 1816 1817 seq->regSaveArea[slot] = nvkm_rd32(device, addr); 1818 nvkm_trace(subdev, "seq save %08x -> %d: %08x\n", addr, slot, 1819 seq->regSaveArea[slot]); 1820 } 1821 break; 1822 case GSP_SEQ_BUF_OPCODE_CORE_RESET: 1823 nvkm_trace(subdev, "seq core reset\n"); 1824 nvkm_falcon_reset(&gsp->falcon); 1825 nvkm_falcon_mask(&gsp->falcon, 0x624, 0x00000080, 0x00000080); 1826 nvkm_falcon_wr32(&gsp->falcon, 0x10c, 0x00000000); 1827 break; 1828 case GSP_SEQ_BUF_OPCODE_CORE_START: 1829 nvkm_trace(subdev, "seq core start\n"); 1830 if (nvkm_falcon_rd32(&gsp->falcon, 0x100) & 0x00000040) 1831 nvkm_falcon_wr32(&gsp->falcon, 0x130, 0x00000002); 1832 else 1833 nvkm_falcon_wr32(&gsp->falcon, 0x100, 0x00000002); 1834 break; 1835 case GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT: 1836 nvkm_trace(subdev, "seq core wait halt\n"); 1837 nvkm_msec(device, 2000, 1838 if (nvkm_falcon_rd32(&gsp->falcon, 0x100) & 0x00000010) 1839 break; 1840 ); 1841 break; 1842 case GSP_SEQ_BUF_OPCODE_CORE_RESUME: { 1843 struct nvkm_sec2 *sec2 = device->sec2; 1844 u32 mbox0; 1845 1846 nvkm_trace(subdev, "seq core resume\n"); 1847 1848 ret = gsp->func->reset(gsp); 1849 if (WARN_ON(ret)) 1850 return ret; 1851 1852 nvkm_falcon_wr32(&gsp->falcon, 0x040, lower_32_bits(gsp->libos.addr)); 1853 nvkm_falcon_wr32(&gsp->falcon, 0x044, upper_32_bits(gsp->libos.addr)); 1854 1855 nvkm_falcon_start(&sec2->falcon); 1856 1857 if (nvkm_msec(device, 2000, 1858 if (nvkm_rd32(device, 0x1180f8) & 0x04000000) 1859 break; 1860 ) < 0) 1861 return -ETIMEDOUT; 1862 1863 mbox0 = nvkm_falcon_rd32(&sec2->falcon, 0x040); 1864 if (WARN_ON(mbox0)) { 1865 nvkm_error(&gsp->subdev, "seq core resume sec2: 0x%x\n", mbox0); 1866 return -EIO; 1867 } 1868 1869 nvkm_falcon_wr32(&gsp->falcon, 0x080, gsp->boot.app_version); 1870 1871 if (WARN_ON(!nvkm_falcon_riscv_active(&gsp->falcon))) 1872 return -EIO; 1873 } 1874 break; 1875 default: 1876 nvkm_error(subdev, "unknown sequencer opcode %08x\n", cmd->opCode); 1877 return -EINVAL; 1878 } 1879 } 1880 1881 return 0; 1882 } 1883 1884 static int 1885 r535_gsp_booter_unload(struct nvkm_gsp *gsp, u32 mbox0, u32 mbox1) 1886 { 1887 struct nvkm_subdev *subdev = &gsp->subdev; 1888 struct nvkm_device *device = subdev->device; 1889 u32 wpr2_hi; 1890 int ret; 1891 1892 wpr2_hi = nvkm_rd32(device, 0x1fa828); 1893 if (!wpr2_hi) { 1894 nvkm_debug(subdev, "WPR2 not set - skipping booter unload\n"); 1895 return 0; 1896 } 1897 1898 ret = nvkm_falcon_fw_boot(&gsp->booter.unload, &gsp->subdev, true, &mbox0, &mbox1, 0, 0); 1899 if (WARN_ON(ret)) 1900 return ret; 1901 1902 wpr2_hi = nvkm_rd32(device, 0x1fa828); 1903 if (WARN_ON(wpr2_hi)) 1904 return -EIO; 1905 1906 return 0; 1907 } 1908 1909 static int 1910 r535_gsp_booter_load(struct nvkm_gsp *gsp, u32 mbox0, u32 mbox1) 1911 { 1912 int ret; 1913 1914 ret = nvkm_falcon_fw_boot(&gsp->booter.load, &gsp->subdev, true, &mbox0, &mbox1, 0, 0); 1915 if (ret) 1916 return ret; 1917 1918 nvkm_falcon_wr32(&gsp->falcon, 0x080, gsp->boot.app_version); 1919 1920 if (WARN_ON(!nvkm_falcon_riscv_active(&gsp->falcon))) 1921 return -EIO; 1922 1923 return 0; 1924 } 1925 1926 static int 1927 r535_gsp_wpr_meta_init(struct nvkm_gsp *gsp) 1928 { 1929 GspFwWprMeta *meta; 1930 int ret; 1931 1932 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->wpr_meta); 1933 if (ret) 1934 return ret; 1935 1936 meta = gsp->wpr_meta.data; 1937 1938 meta->magic = GSP_FW_WPR_META_MAGIC; 1939 meta->revision = GSP_FW_WPR_META_REVISION; 1940 1941 meta->sysmemAddrOfRadix3Elf = gsp->radix3.lvl0.addr; 1942 meta->sizeOfRadix3Elf = gsp->fb.wpr2.elf.size; 1943 1944 meta->sysmemAddrOfBootloader = gsp->boot.fw.addr; 1945 meta->sizeOfBootloader = gsp->boot.fw.size; 1946 meta->bootloaderCodeOffset = gsp->boot.code_offset; 1947 meta->bootloaderDataOffset = gsp->boot.data_offset; 1948 meta->bootloaderManifestOffset = gsp->boot.manifest_offset; 1949 1950 meta->sysmemAddrOfSignature = gsp->sig.addr; 1951 meta->sizeOfSignature = gsp->sig.size; 1952 1953 meta->gspFwRsvdStart = gsp->fb.heap.addr; 1954 meta->nonWprHeapOffset = gsp->fb.heap.addr; 1955 meta->nonWprHeapSize = gsp->fb.heap.size; 1956 meta->gspFwWprStart = gsp->fb.wpr2.addr; 1957 meta->gspFwHeapOffset = gsp->fb.wpr2.heap.addr; 1958 meta->gspFwHeapSize = gsp->fb.wpr2.heap.size; 1959 meta->gspFwOffset = gsp->fb.wpr2.elf.addr; 1960 meta->bootBinOffset = gsp->fb.wpr2.boot.addr; 1961 meta->frtsOffset = gsp->fb.wpr2.frts.addr; 1962 meta->frtsSize = gsp->fb.wpr2.frts.size; 1963 meta->gspFwWprEnd = ALIGN_DOWN(gsp->fb.bios.vga_workspace.addr, 0x20000); 1964 meta->fbSize = gsp->fb.size; 1965 meta->vgaWorkspaceOffset = gsp->fb.bios.vga_workspace.addr; 1966 meta->vgaWorkspaceSize = gsp->fb.bios.vga_workspace.size; 1967 meta->bootCount = 0; 1968 meta->partitionRpcAddr = 0; 1969 meta->partitionRpcRequestOffset = 0; 1970 meta->partitionRpcReplyOffset = 0; 1971 meta->verified = 0; 1972 return 0; 1973 } 1974 1975 static int 1976 r535_gsp_shared_init(struct nvkm_gsp *gsp) 1977 { 1978 struct { 1979 msgqTxHeader tx; 1980 msgqRxHeader rx; 1981 } *cmdq, *msgq; 1982 int ret, i; 1983 1984 gsp->shm.cmdq.size = 0x40000; 1985 gsp->shm.msgq.size = 0x40000; 1986 1987 gsp->shm.ptes.nr = (gsp->shm.cmdq.size + gsp->shm.msgq.size) >> GSP_PAGE_SHIFT; 1988 gsp->shm.ptes.nr += DIV_ROUND_UP(gsp->shm.ptes.nr * sizeof(u64), GSP_PAGE_SIZE); 1989 gsp->shm.ptes.size = ALIGN(gsp->shm.ptes.nr * sizeof(u64), GSP_PAGE_SIZE); 1990 1991 ret = nvkm_gsp_mem_ctor(gsp, gsp->shm.ptes.size + 1992 gsp->shm.cmdq.size + 1993 gsp->shm.msgq.size, 1994 &gsp->shm.mem); 1995 if (ret) 1996 return ret; 1997 1998 gsp->shm.ptes.ptr = gsp->shm.mem.data; 1999 gsp->shm.cmdq.ptr = (u8 *)gsp->shm.ptes.ptr + gsp->shm.ptes.size; 2000 gsp->shm.msgq.ptr = (u8 *)gsp->shm.cmdq.ptr + gsp->shm.cmdq.size; 2001 2002 for (i = 0; i < gsp->shm.ptes.nr; i++) 2003 gsp->shm.ptes.ptr[i] = gsp->shm.mem.addr + (i << GSP_PAGE_SHIFT); 2004 2005 cmdq = gsp->shm.cmdq.ptr; 2006 cmdq->tx.version = 0; 2007 cmdq->tx.size = gsp->shm.cmdq.size; 2008 cmdq->tx.entryOff = GSP_PAGE_SIZE; 2009 cmdq->tx.msgSize = GSP_PAGE_SIZE; 2010 cmdq->tx.msgCount = (cmdq->tx.size - cmdq->tx.entryOff) / cmdq->tx.msgSize; 2011 cmdq->tx.writePtr = 0; 2012 cmdq->tx.flags = 1; 2013 cmdq->tx.rxHdrOff = offsetof(typeof(*cmdq), rx.readPtr); 2014 2015 msgq = gsp->shm.msgq.ptr; 2016 2017 gsp->cmdq.cnt = cmdq->tx.msgCount; 2018 gsp->cmdq.wptr = &cmdq->tx.writePtr; 2019 gsp->cmdq.rptr = &msgq->rx.readPtr; 2020 gsp->msgq.cnt = cmdq->tx.msgCount; 2021 gsp->msgq.wptr = &msgq->tx.writePtr; 2022 gsp->msgq.rptr = &cmdq->rx.readPtr; 2023 return 0; 2024 } 2025 2026 static int 2027 r535_gsp_rmargs_init(struct nvkm_gsp *gsp, bool resume) 2028 { 2029 GSP_ARGUMENTS_CACHED *args; 2030 int ret; 2031 2032 if (!resume) { 2033 ret = r535_gsp_shared_init(gsp); 2034 if (ret) 2035 return ret; 2036 2037 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->rmargs); 2038 if (ret) 2039 return ret; 2040 } 2041 2042 args = gsp->rmargs.data; 2043 args->messageQueueInitArguments.sharedMemPhysAddr = gsp->shm.mem.addr; 2044 args->messageQueueInitArguments.pageTableEntryCount = gsp->shm.ptes.nr; 2045 args->messageQueueInitArguments.cmdQueueOffset = 2046 (u8 *)gsp->shm.cmdq.ptr - (u8 *)gsp->shm.mem.data; 2047 args->messageQueueInitArguments.statQueueOffset = 2048 (u8 *)gsp->shm.msgq.ptr - (u8 *)gsp->shm.mem.data; 2049 2050 if (!resume) { 2051 args->srInitArguments.oldLevel = 0; 2052 args->srInitArguments.flags = 0; 2053 args->srInitArguments.bInPMTransition = 0; 2054 } else { 2055 args->srInitArguments.oldLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3; 2056 args->srInitArguments.flags = 0; 2057 args->srInitArguments.bInPMTransition = 1; 2058 } 2059 2060 return 0; 2061 } 2062 2063 static inline u64 2064 r535_gsp_libos_id8(const char *name) 2065 { 2066 u64 id = 0; 2067 2068 for (int i = 0; i < sizeof(id) && *name; i++, name++) 2069 id = (id << 8) | *name; 2070 2071 return id; 2072 } 2073 2074 /** 2075 * create_pte_array() - creates a PTE array of a physically contiguous buffer 2076 * @ptes: pointer to the array 2077 * @addr: base address of physically contiguous buffer (GSP_PAGE_SIZE aligned) 2078 * @size: size of the buffer 2079 * 2080 * GSP-RM sometimes expects physically-contiguous buffers to have an array of 2081 * "PTEs" for each page in that buffer. Although in theory that allows for 2082 * the buffer to be physically discontiguous, GSP-RM does not currently 2083 * support that. 2084 * 2085 * In this case, the PTEs are DMA addresses of each page of the buffer. Since 2086 * the buffer is physically contiguous, calculating all the PTEs is simple 2087 * math. 2088 * 2089 * See memdescGetPhysAddrsForGpu() 2090 */ 2091 static void create_pte_array(u64 *ptes, dma_addr_t addr, size_t size) 2092 { 2093 unsigned int num_pages = DIV_ROUND_UP_ULL(size, GSP_PAGE_SIZE); 2094 unsigned int i; 2095 2096 for (i = 0; i < num_pages; i++) 2097 ptes[i] = (u64)addr + (i << GSP_PAGE_SHIFT); 2098 } 2099 2100 /** 2101 * r535_gsp_libos_init() -- create the libos arguments structure 2102 * @gsp: gsp pointer 2103 * 2104 * The logging buffers are byte queues that contain encoded printf-like 2105 * messages from GSP-RM. They need to be decoded by a special application 2106 * that can parse the buffers. 2107 * 2108 * The 'loginit' buffer contains logs from early GSP-RM init and 2109 * exception dumps. The 'logrm' buffer contains the subsequent logs. Both are 2110 * written to directly by GSP-RM and can be any multiple of GSP_PAGE_SIZE. 2111 * 2112 * The physical address map for the log buffer is stored in the buffer 2113 * itself, starting with offset 1. Offset 0 contains the "put" pointer. 2114 * 2115 * The GSP only understands 4K pages (GSP_PAGE_SIZE), so even if the kernel is 2116 * configured for a larger page size (e.g. 64K pages), we need to give 2117 * the GSP an array of 4K pages. Fortunately, since the buffer is 2118 * physically contiguous, it's simple math to calculate the addresses. 2119 * 2120 * The buffers must be a multiple of GSP_PAGE_SIZE. GSP-RM also currently 2121 * ignores the @kind field for LOGINIT, LOGINTR, and LOGRM, but expects the 2122 * buffers to be physically contiguous anyway. 2123 * 2124 * The memory allocated for the arguments must remain until the GSP sends the 2125 * init_done RPC. 2126 * 2127 * See _kgspInitLibosLoggingStructures (allocates memory for buffers) 2128 * See kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array) 2129 */ 2130 static int 2131 r535_gsp_libos_init(struct nvkm_gsp *gsp) 2132 { 2133 LibosMemoryRegionInitArgument *args; 2134 int ret; 2135 2136 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->libos); 2137 if (ret) 2138 return ret; 2139 2140 args = gsp->libos.data; 2141 2142 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->loginit); 2143 if (ret) 2144 return ret; 2145 2146 args[0].id8 = r535_gsp_libos_id8("LOGINIT"); 2147 args[0].pa = gsp->loginit.addr; 2148 args[0].size = gsp->loginit.size; 2149 args[0].kind = LIBOS_MEMORY_REGION_CONTIGUOUS; 2150 args[0].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM; 2151 create_pte_array(gsp->loginit.data + sizeof(u64), gsp->loginit.addr, gsp->loginit.size); 2152 2153 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->logintr); 2154 if (ret) 2155 return ret; 2156 2157 args[1].id8 = r535_gsp_libos_id8("LOGINTR"); 2158 args[1].pa = gsp->logintr.addr; 2159 args[1].size = gsp->logintr.size; 2160 args[1].kind = LIBOS_MEMORY_REGION_CONTIGUOUS; 2161 args[1].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM; 2162 create_pte_array(gsp->logintr.data + sizeof(u64), gsp->logintr.addr, gsp->logintr.size); 2163 2164 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->logrm); 2165 if (ret) 2166 return ret; 2167 2168 args[2].id8 = r535_gsp_libos_id8("LOGRM"); 2169 args[2].pa = gsp->logrm.addr; 2170 args[2].size = gsp->logrm.size; 2171 args[2].kind = LIBOS_MEMORY_REGION_CONTIGUOUS; 2172 args[2].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM; 2173 create_pte_array(gsp->logrm.data + sizeof(u64), gsp->logrm.addr, gsp->logrm.size); 2174 2175 ret = r535_gsp_rmargs_init(gsp, false); 2176 if (ret) 2177 return ret; 2178 2179 args[3].id8 = r535_gsp_libos_id8("RMARGS"); 2180 args[3].pa = gsp->rmargs.addr; 2181 args[3].size = gsp->rmargs.size; 2182 args[3].kind = LIBOS_MEMORY_REGION_CONTIGUOUS; 2183 args[3].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM; 2184 return 0; 2185 } 2186 2187 void 2188 nvkm_gsp_sg_free(struct nvkm_device *device, struct sg_table *sgt) 2189 { 2190 struct scatterlist *sgl; 2191 int i; 2192 2193 dma_unmap_sgtable(device->dev, sgt, DMA_BIDIRECTIONAL, 0); 2194 2195 for_each_sgtable_sg(sgt, sgl, i) { 2196 struct page *page = sg_page(sgl); 2197 2198 __free_page(page); 2199 } 2200 2201 sg_free_table(sgt); 2202 } 2203 2204 int 2205 nvkm_gsp_sg(struct nvkm_device *device, u64 size, struct sg_table *sgt) 2206 { 2207 const u64 pages = DIV_ROUND_UP(size, PAGE_SIZE); 2208 struct scatterlist *sgl; 2209 int ret, i; 2210 2211 ret = sg_alloc_table(sgt, pages, GFP_KERNEL); 2212 if (ret) 2213 return ret; 2214 2215 for_each_sgtable_sg(sgt, sgl, i) { 2216 struct page *page = alloc_page(GFP_KERNEL); 2217 2218 if (!page) { 2219 nvkm_gsp_sg_free(device, sgt); 2220 return -ENOMEM; 2221 } 2222 2223 sg_set_page(sgl, page, PAGE_SIZE, 0); 2224 } 2225 2226 ret = dma_map_sgtable(device->dev, sgt, DMA_BIDIRECTIONAL, 0); 2227 if (ret) 2228 nvkm_gsp_sg_free(device, sgt); 2229 2230 return ret; 2231 } 2232 2233 static void 2234 nvkm_gsp_radix3_dtor(struct nvkm_gsp *gsp, struct nvkm_gsp_radix3 *rx3) 2235 { 2236 nvkm_gsp_sg_free(gsp->subdev.device, &rx3->lvl2); 2237 nvkm_gsp_mem_dtor(gsp, &rx3->lvl1); 2238 nvkm_gsp_mem_dtor(gsp, &rx3->lvl0); 2239 } 2240 2241 /** 2242 * nvkm_gsp_radix3_sg - build a radix3 table from a S/G list 2243 * @gsp: gsp pointer 2244 * @sgt: S/G list to traverse 2245 * @size: size of the image, in bytes 2246 * @rx3: radix3 array to update 2247 * 2248 * The GSP uses a three-level page table, called radix3, to map the firmware. 2249 * Each 64-bit "pointer" in the table is either the bus address of an entry in 2250 * the next table (for levels 0 and 1) or the bus address of the next page in 2251 * the GSP firmware image itself. 2252 * 2253 * Level 0 contains a single entry in one page that points to the first page 2254 * of level 1. 2255 * 2256 * Level 1, since it's also only one page in size, contains up to 512 entries, 2257 * one for each page in Level 2. 2258 * 2259 * Level 2 can be up to 512 pages in size, and each of those entries points to 2260 * the next page of the firmware image. Since there can be up to 512*512 2261 * pages, that limits the size of the firmware to 512*512*GSP_PAGE_SIZE = 1GB. 2262 * 2263 * Internally, the GSP has its window into system memory, but the base 2264 * physical address of the aperture is not 0. In fact, it varies depending on 2265 * the GPU architecture. Since the GPU is a PCI device, this window is 2266 * accessed via DMA and is therefore bound by IOMMU translation. The end 2267 * result is that GSP-RM must translate the bus addresses in the table to GSP 2268 * physical addresses. All this should happen transparently. 2269 * 2270 * Returns 0 on success, or negative error code 2271 * 2272 * See kgspCreateRadix3_IMPL 2273 */ 2274 static int 2275 nvkm_gsp_radix3_sg(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size, 2276 struct nvkm_gsp_radix3 *rx3) 2277 { 2278 struct sg_dma_page_iter sg_dma_iter; 2279 struct scatterlist *sg; 2280 size_t bufsize; 2281 u64 *pte; 2282 int ret, i, page_idx = 0; 2283 2284 ret = nvkm_gsp_mem_ctor(gsp, GSP_PAGE_SIZE, &rx3->lvl0); 2285 if (ret) 2286 return ret; 2287 2288 ret = nvkm_gsp_mem_ctor(gsp, GSP_PAGE_SIZE, &rx3->lvl1); 2289 if (ret) 2290 goto lvl1_fail; 2291 2292 // Allocate level 2 2293 bufsize = ALIGN((size / GSP_PAGE_SIZE) * sizeof(u64), GSP_PAGE_SIZE); 2294 ret = nvkm_gsp_sg(gsp->subdev.device, bufsize, &rx3->lvl2); 2295 if (ret) 2296 goto lvl2_fail; 2297 2298 // Write the bus address of level 1 to level 0 2299 pte = rx3->lvl0.data; 2300 *pte = rx3->lvl1.addr; 2301 2302 // Write the bus address of each page in level 2 to level 1 2303 pte = rx3->lvl1.data; 2304 for_each_sgtable_dma_page(&rx3->lvl2, &sg_dma_iter, 0) 2305 *pte++ = sg_page_iter_dma_address(&sg_dma_iter); 2306 2307 // Finally, write the bus address of each page in sgt to level 2 2308 for_each_sgtable_sg(&rx3->lvl2, sg, i) { 2309 void *sgl_end; 2310 2311 pte = sg_virt(sg); 2312 sgl_end = (void *)pte + sg->length; 2313 2314 for_each_sgtable_dma_page(sgt, &sg_dma_iter, page_idx) { 2315 *pte++ = sg_page_iter_dma_address(&sg_dma_iter); 2316 page_idx++; 2317 2318 // Go to the next scatterlist for level 2 if we've reached the end 2319 if ((void *)pte >= sgl_end) 2320 break; 2321 } 2322 } 2323 2324 if (ret) { 2325 lvl2_fail: 2326 nvkm_gsp_mem_dtor(gsp, &rx3->lvl1); 2327 lvl1_fail: 2328 nvkm_gsp_mem_dtor(gsp, &rx3->lvl0); 2329 } 2330 2331 return ret; 2332 } 2333 2334 int 2335 r535_gsp_fini(struct nvkm_gsp *gsp, bool suspend) 2336 { 2337 u32 mbox0 = 0xff, mbox1 = 0xff; 2338 int ret; 2339 2340 if (!gsp->running) 2341 return 0; 2342 2343 if (suspend) { 2344 GspFwWprMeta *meta = gsp->wpr_meta.data; 2345 u64 len = meta->gspFwWprEnd - meta->gspFwWprStart; 2346 GspFwSRMeta *sr; 2347 2348 ret = nvkm_gsp_sg(gsp->subdev.device, len, &gsp->sr.sgt); 2349 if (ret) 2350 return ret; 2351 2352 ret = nvkm_gsp_radix3_sg(gsp, &gsp->sr.sgt, len, &gsp->sr.radix3); 2353 if (ret) 2354 return ret; 2355 2356 ret = nvkm_gsp_mem_ctor(gsp, sizeof(*sr), &gsp->sr.meta); 2357 if (ret) 2358 return ret; 2359 2360 sr = gsp->sr.meta.data; 2361 sr->magic = GSP_FW_SR_META_MAGIC; 2362 sr->revision = GSP_FW_SR_META_REVISION; 2363 sr->sysmemAddrOfSuspendResumeData = gsp->sr.radix3.lvl0.addr; 2364 sr->sizeOfSuspendResumeData = len; 2365 2366 mbox0 = lower_32_bits(gsp->sr.meta.addr); 2367 mbox1 = upper_32_bits(gsp->sr.meta.addr); 2368 } 2369 2370 ret = r535_gsp_rpc_unloading_guest_driver(gsp, suspend); 2371 if (WARN_ON(ret)) 2372 return ret; 2373 2374 nvkm_msec(gsp->subdev.device, 2000, 2375 if (nvkm_falcon_rd32(&gsp->falcon, 0x040) & 0x80000000) 2376 break; 2377 ); 2378 2379 nvkm_falcon_reset(&gsp->falcon); 2380 2381 ret = nvkm_gsp_fwsec_sb(gsp); 2382 WARN_ON(ret); 2383 2384 ret = r535_gsp_booter_unload(gsp, mbox0, mbox1); 2385 WARN_ON(ret); 2386 2387 gsp->running = false; 2388 return 0; 2389 } 2390 2391 int 2392 r535_gsp_init(struct nvkm_gsp *gsp) 2393 { 2394 u32 mbox0, mbox1; 2395 int ret; 2396 2397 if (!gsp->sr.meta.data) { 2398 mbox0 = lower_32_bits(gsp->wpr_meta.addr); 2399 mbox1 = upper_32_bits(gsp->wpr_meta.addr); 2400 } else { 2401 r535_gsp_rmargs_init(gsp, true); 2402 2403 mbox0 = lower_32_bits(gsp->sr.meta.addr); 2404 mbox1 = upper_32_bits(gsp->sr.meta.addr); 2405 } 2406 2407 /* Execute booter to handle (eventually...) booting GSP-RM. */ 2408 ret = r535_gsp_booter_load(gsp, mbox0, mbox1); 2409 if (WARN_ON(ret)) 2410 goto done; 2411 2412 ret = r535_gsp_rpc_poll(gsp, NV_VGPU_MSG_EVENT_GSP_INIT_DONE); 2413 if (ret) 2414 goto done; 2415 2416 gsp->running = true; 2417 2418 done: 2419 if (gsp->sr.meta.data) { 2420 nvkm_gsp_mem_dtor(gsp, &gsp->sr.meta); 2421 nvkm_gsp_radix3_dtor(gsp, &gsp->sr.radix3); 2422 nvkm_gsp_sg_free(gsp->subdev.device, &gsp->sr.sgt); 2423 return ret; 2424 } 2425 2426 if (ret == 0) 2427 ret = r535_gsp_postinit(gsp); 2428 2429 return ret; 2430 } 2431 2432 static int 2433 r535_gsp_rm_boot_ctor(struct nvkm_gsp *gsp) 2434 { 2435 const struct firmware *fw = gsp->fws.bl; 2436 const struct nvfw_bin_hdr *hdr; 2437 RM_RISCV_UCODE_DESC *desc; 2438 int ret; 2439 2440 hdr = nvfw_bin_hdr(&gsp->subdev, fw->data); 2441 desc = (void *)fw->data + hdr->header_offset; 2442 2443 ret = nvkm_gsp_mem_ctor(gsp, hdr->data_size, &gsp->boot.fw); 2444 if (ret) 2445 return ret; 2446 2447 memcpy(gsp->boot.fw.data, fw->data + hdr->data_offset, hdr->data_size); 2448 2449 gsp->boot.code_offset = desc->monitorCodeOffset; 2450 gsp->boot.data_offset = desc->monitorDataOffset; 2451 gsp->boot.manifest_offset = desc->manifestOffset; 2452 gsp->boot.app_version = desc->appVersion; 2453 return 0; 2454 } 2455 2456 static const struct nvkm_firmware_func 2457 r535_gsp_fw = { 2458 .type = NVKM_FIRMWARE_IMG_SGT, 2459 }; 2460 2461 static int 2462 r535_gsp_elf_section(struct nvkm_gsp *gsp, const char *name, const u8 **pdata, u64 *psize) 2463 { 2464 const u8 *img = gsp->fws.rm->data; 2465 const struct elf64_hdr *ehdr = (const struct elf64_hdr *)img; 2466 const struct elf64_shdr *shdr = (const struct elf64_shdr *)&img[ehdr->e_shoff]; 2467 const char *names = &img[shdr[ehdr->e_shstrndx].sh_offset]; 2468 2469 for (int i = 0; i < ehdr->e_shnum; i++, shdr++) { 2470 if (!strcmp(&names[shdr->sh_name], name)) { 2471 *pdata = &img[shdr->sh_offset]; 2472 *psize = shdr->sh_size; 2473 return 0; 2474 } 2475 } 2476 2477 nvkm_error(&gsp->subdev, "section '%s' not found\n", name); 2478 return -ENOENT; 2479 } 2480 2481 static void 2482 r535_gsp_dtor_fws(struct nvkm_gsp *gsp) 2483 { 2484 nvkm_firmware_put(gsp->fws.bl); 2485 gsp->fws.bl = NULL; 2486 nvkm_firmware_put(gsp->fws.booter.unload); 2487 gsp->fws.booter.unload = NULL; 2488 nvkm_firmware_put(gsp->fws.booter.load); 2489 gsp->fws.booter.load = NULL; 2490 nvkm_firmware_put(gsp->fws.rm); 2491 gsp->fws.rm = NULL; 2492 } 2493 2494 void 2495 r535_gsp_dtor(struct nvkm_gsp *gsp) 2496 { 2497 idr_destroy(&gsp->client_id.idr); 2498 mutex_destroy(&gsp->client_id.mutex); 2499 2500 nvkm_gsp_radix3_dtor(gsp, &gsp->radix3); 2501 nvkm_gsp_mem_dtor(gsp, &gsp->sig); 2502 nvkm_firmware_dtor(&gsp->fw); 2503 2504 nvkm_falcon_fw_dtor(&gsp->booter.unload); 2505 nvkm_falcon_fw_dtor(&gsp->booter.load); 2506 2507 mutex_destroy(&gsp->msgq.mutex); 2508 mutex_destroy(&gsp->cmdq.mutex); 2509 2510 r535_gsp_dtor_fws(gsp); 2511 2512 nvkm_gsp_mem_dtor(gsp, &gsp->rmargs); 2513 nvkm_gsp_mem_dtor(gsp, &gsp->wpr_meta); 2514 nvkm_gsp_mem_dtor(gsp, &gsp->shm.mem); 2515 nvkm_gsp_mem_dtor(gsp, &gsp->loginit); 2516 nvkm_gsp_mem_dtor(gsp, &gsp->logintr); 2517 nvkm_gsp_mem_dtor(gsp, &gsp->logrm); 2518 } 2519 2520 int 2521 r535_gsp_oneinit(struct nvkm_gsp *gsp) 2522 { 2523 struct nvkm_device *device = gsp->subdev.device; 2524 const u8 *data; 2525 u64 size; 2526 int ret; 2527 2528 mutex_init(&gsp->cmdq.mutex); 2529 mutex_init(&gsp->msgq.mutex); 2530 2531 ret = gsp->func->booter.ctor(gsp, "booter-load", gsp->fws.booter.load, 2532 &device->sec2->falcon, &gsp->booter.load); 2533 if (ret) 2534 return ret; 2535 2536 ret = gsp->func->booter.ctor(gsp, "booter-unload", gsp->fws.booter.unload, 2537 &device->sec2->falcon, &gsp->booter.unload); 2538 if (ret) 2539 return ret; 2540 2541 /* Load GSP firmware from ELF image into DMA-accessible memory. */ 2542 ret = r535_gsp_elf_section(gsp, ".fwimage", &data, &size); 2543 if (ret) 2544 return ret; 2545 2546 ret = nvkm_firmware_ctor(&r535_gsp_fw, "gsp-rm", device, data, size, &gsp->fw); 2547 if (ret) 2548 return ret; 2549 2550 /* Load relevant signature from ELF image. */ 2551 ret = r535_gsp_elf_section(gsp, gsp->func->sig_section, &data, &size); 2552 if (ret) 2553 return ret; 2554 2555 ret = nvkm_gsp_mem_ctor(gsp, ALIGN(size, 256), &gsp->sig); 2556 if (ret) 2557 return ret; 2558 2559 memcpy(gsp->sig.data, data, size); 2560 2561 /* Build radix3 page table for ELF image. */ 2562 ret = nvkm_gsp_radix3_sg(gsp, &gsp->fw.mem.sgt, gsp->fw.len, &gsp->radix3); 2563 if (ret) 2564 return ret; 2565 2566 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER, 2567 r535_gsp_msg_run_cpu_sequencer, gsp); 2568 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_POST_EVENT, r535_gsp_msg_post_event, gsp); 2569 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_RC_TRIGGERED, 2570 r535_gsp_msg_rc_triggered, gsp); 2571 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED, 2572 r535_gsp_msg_mmu_fault_queued, gsp); 2573 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_OS_ERROR_LOG, r535_gsp_msg_os_error_log, gsp); 2574 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_PERF_BRIDGELESS_INFO_UPDATE, NULL, NULL); 2575 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT, NULL, NULL); 2576 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_GSP_SEND_USER_SHARED_DATA, NULL, NULL); 2577 ret = r535_gsp_rm_boot_ctor(gsp); 2578 if (ret) 2579 return ret; 2580 2581 /* Release FW images - we've copied them to DMA buffers now. */ 2582 r535_gsp_dtor_fws(gsp); 2583 2584 /* Calculate FB layout. */ 2585 gsp->fb.wpr2.frts.size = 0x100000; 2586 gsp->fb.wpr2.frts.addr = ALIGN_DOWN(gsp->fb.bios.addr, 0x20000) - gsp->fb.wpr2.frts.size; 2587 2588 gsp->fb.wpr2.boot.size = gsp->boot.fw.size; 2589 gsp->fb.wpr2.boot.addr = ALIGN_DOWN(gsp->fb.wpr2.frts.addr - gsp->fb.wpr2.boot.size, 0x1000); 2590 2591 gsp->fb.wpr2.elf.size = gsp->fw.len; 2592 gsp->fb.wpr2.elf.addr = ALIGN_DOWN(gsp->fb.wpr2.boot.addr - gsp->fb.wpr2.elf.size, 0x10000); 2593 2594 { 2595 u32 fb_size_gb = DIV_ROUND_UP_ULL(gsp->fb.size, 1 << 30); 2596 2597 gsp->fb.wpr2.heap.size = 2598 gsp->func->wpr_heap.os_carveout_size + 2599 gsp->func->wpr_heap.base_size + 2600 ALIGN(GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB * fb_size_gb, 1 << 20) + 2601 ALIGN(GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE, 1 << 20); 2602 2603 gsp->fb.wpr2.heap.size = max(gsp->fb.wpr2.heap.size, gsp->func->wpr_heap.min_size); 2604 } 2605 2606 gsp->fb.wpr2.heap.addr = ALIGN_DOWN(gsp->fb.wpr2.elf.addr - gsp->fb.wpr2.heap.size, 0x100000); 2607 gsp->fb.wpr2.heap.size = ALIGN_DOWN(gsp->fb.wpr2.elf.addr - gsp->fb.wpr2.heap.addr, 0x100000); 2608 2609 gsp->fb.wpr2.addr = ALIGN_DOWN(gsp->fb.wpr2.heap.addr - sizeof(GspFwWprMeta), 0x100000); 2610 gsp->fb.wpr2.size = gsp->fb.wpr2.frts.addr + gsp->fb.wpr2.frts.size - gsp->fb.wpr2.addr; 2611 2612 gsp->fb.heap.size = 0x100000; 2613 gsp->fb.heap.addr = gsp->fb.wpr2.addr - gsp->fb.heap.size; 2614 2615 ret = nvkm_gsp_fwsec_frts(gsp); 2616 if (WARN_ON(ret)) 2617 return ret; 2618 2619 ret = r535_gsp_libos_init(gsp); 2620 if (WARN_ON(ret)) 2621 return ret; 2622 2623 ret = r535_gsp_wpr_meta_init(gsp); 2624 if (WARN_ON(ret)) 2625 return ret; 2626 2627 ret = r535_gsp_rpc_set_system_info(gsp); 2628 if (WARN_ON(ret)) 2629 return ret; 2630 2631 ret = r535_gsp_rpc_set_registry(gsp); 2632 if (WARN_ON(ret)) 2633 return ret; 2634 2635 /* Reset GSP into RISC-V mode. */ 2636 ret = gsp->func->reset(gsp); 2637 if (WARN_ON(ret)) 2638 return ret; 2639 2640 nvkm_falcon_wr32(&gsp->falcon, 0x040, lower_32_bits(gsp->libos.addr)); 2641 nvkm_falcon_wr32(&gsp->falcon, 0x044, upper_32_bits(gsp->libos.addr)); 2642 2643 mutex_init(&gsp->client_id.mutex); 2644 idr_init(&gsp->client_id.idr); 2645 return 0; 2646 } 2647 2648 static int 2649 r535_gsp_load_fw(struct nvkm_gsp *gsp, const char *name, const char *ver, 2650 const struct firmware **pfw) 2651 { 2652 char fwname[64]; 2653 2654 snprintf(fwname, sizeof(fwname), "gsp/%s-%s", name, ver); 2655 return nvkm_firmware_get(&gsp->subdev, fwname, 0, pfw); 2656 } 2657 2658 int 2659 r535_gsp_load(struct nvkm_gsp *gsp, int ver, const struct nvkm_gsp_fwif *fwif) 2660 { 2661 struct nvkm_subdev *subdev = &gsp->subdev; 2662 int ret; 2663 bool enable_gsp = fwif->enable; 2664 2665 #if IS_ENABLED(CONFIG_DRM_NOUVEAU_GSP_DEFAULT) 2666 enable_gsp = true; 2667 #endif 2668 if (!nvkm_boolopt(subdev->device->cfgopt, "NvGspRm", enable_gsp)) 2669 return -EINVAL; 2670 2671 if ((ret = r535_gsp_load_fw(gsp, "gsp", fwif->ver, &gsp->fws.rm)) || 2672 (ret = r535_gsp_load_fw(gsp, "booter_load", fwif->ver, &gsp->fws.booter.load)) || 2673 (ret = r535_gsp_load_fw(gsp, "booter_unload", fwif->ver, &gsp->fws.booter.unload)) || 2674 (ret = r535_gsp_load_fw(gsp, "bootloader", fwif->ver, &gsp->fws.bl))) { 2675 r535_gsp_dtor_fws(gsp); 2676 return ret; 2677 } 2678 2679 return 0; 2680 } 2681 2682 #define NVKM_GSP_FIRMWARE(chip) \ 2683 MODULE_FIRMWARE("nvidia/"#chip"/gsp/booter_load-535.113.01.bin"); \ 2684 MODULE_FIRMWARE("nvidia/"#chip"/gsp/booter_unload-535.113.01.bin"); \ 2685 MODULE_FIRMWARE("nvidia/"#chip"/gsp/bootloader-535.113.01.bin"); \ 2686 MODULE_FIRMWARE("nvidia/"#chip"/gsp/gsp-535.113.01.bin") 2687 2688 NVKM_GSP_FIRMWARE(tu102); 2689 NVKM_GSP_FIRMWARE(tu104); 2690 NVKM_GSP_FIRMWARE(tu106); 2691 2692 NVKM_GSP_FIRMWARE(tu116); 2693 NVKM_GSP_FIRMWARE(tu117); 2694 2695 NVKM_GSP_FIRMWARE(ga100); 2696 2697 NVKM_GSP_FIRMWARE(ga102); 2698 NVKM_GSP_FIRMWARE(ga103); 2699 NVKM_GSP_FIRMWARE(ga104); 2700 NVKM_GSP_FIRMWARE(ga106); 2701 NVKM_GSP_FIRMWARE(ga107); 2702 2703 NVKM_GSP_FIRMWARE(ad102); 2704 NVKM_GSP_FIRMWARE(ad103); 2705 NVKM_GSP_FIRMWARE(ad104); 2706 NVKM_GSP_FIRMWARE(ad106); 2707 NVKM_GSP_FIRMWARE(ad107); 2708