1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2023 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/firmware.h> 26 #include <drm/drm_exec.h> 27 28 #include "amdgpu.h" 29 #include "amdgpu_umsch_mm.h" 30 #include "umsch_mm_v4_0.h" 31 32 struct umsch_mm_test_ctx_data { 33 uint8_t process_csa[PAGE_SIZE]; 34 uint8_t vpe_ctx_csa[PAGE_SIZE]; 35 uint8_t vcn_ctx_csa[PAGE_SIZE]; 36 }; 37 38 struct umsch_mm_test_mqd_data { 39 uint8_t vpe_mqd[PAGE_SIZE]; 40 uint8_t vcn_mqd[PAGE_SIZE]; 41 }; 42 43 struct umsch_mm_test_ring_data { 44 uint8_t vpe_ring[PAGE_SIZE]; 45 uint8_t vpe_ib[PAGE_SIZE]; 46 uint8_t vcn_ring[PAGE_SIZE]; 47 uint8_t vcn_ib[PAGE_SIZE]; 48 }; 49 50 struct umsch_mm_test_queue_info { 51 uint64_t mqd_addr; 52 uint64_t csa_addr; 53 uint32_t doorbell_offset_0; 54 uint32_t doorbell_offset_1; 55 enum UMSCH_SWIP_ENGINE_TYPE engine; 56 }; 57 58 struct umsch_mm_test { 59 struct amdgpu_bo *ctx_data_obj; 60 uint64_t ctx_data_gpu_addr; 61 uint32_t *ctx_data_cpu_addr; 62 63 struct amdgpu_bo *mqd_data_obj; 64 uint64_t mqd_data_gpu_addr; 65 uint32_t *mqd_data_cpu_addr; 66 67 struct amdgpu_bo *ring_data_obj; 68 uint64_t ring_data_gpu_addr; 69 uint32_t *ring_data_cpu_addr; 70 71 72 struct amdgpu_vm *vm; 73 struct amdgpu_bo_va *bo_va; 74 uint32_t pasid; 75 uint32_t vm_cntx_cntl; 76 uint32_t num_queues; 77 }; 78 79 static int map_ring_data(struct amdgpu_device *adev, struct amdgpu_vm *vm, 80 struct amdgpu_bo *bo, struct amdgpu_bo_va **bo_va, 81 uint64_t addr, uint32_t size) 82 { 83 struct amdgpu_sync sync; 84 struct drm_exec exec; 85 int r; 86 87 amdgpu_sync_create(&sync); 88 89 drm_exec_init(&exec, 0); 90 drm_exec_until_all_locked(&exec) { 91 r = drm_exec_lock_obj(&exec, &bo->tbo.base); 92 drm_exec_retry_on_contention(&exec); 93 if (unlikely(r)) 94 goto error_fini_exec; 95 96 r = amdgpu_vm_lock_pd(vm, &exec, 0); 97 drm_exec_retry_on_contention(&exec); 98 if (unlikely(r)) 99 goto error_fini_exec; 100 } 101 102 *bo_va = amdgpu_vm_bo_add(adev, vm, bo); 103 if (!*bo_va) { 104 r = -ENOMEM; 105 goto error_fini_exec; 106 } 107 108 r = amdgpu_vm_bo_map(adev, *bo_va, addr, 0, size, 109 AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE | 110 AMDGPU_PTE_EXECUTABLE); 111 112 if (r) 113 goto error_del_bo_va; 114 115 116 r = amdgpu_vm_bo_update(adev, *bo_va, false); 117 if (r) 118 goto error_del_bo_va; 119 120 amdgpu_sync_fence(&sync, (*bo_va)->last_pt_update); 121 122 r = amdgpu_vm_update_pdes(adev, vm, false); 123 if (r) 124 goto error_del_bo_va; 125 126 amdgpu_sync_fence(&sync, vm->last_update); 127 128 amdgpu_sync_wait(&sync, false); 129 drm_exec_fini(&exec); 130 131 amdgpu_sync_free(&sync); 132 133 return 0; 134 135 error_del_bo_va: 136 amdgpu_vm_bo_del(adev, *bo_va); 137 amdgpu_sync_free(&sync); 138 139 error_fini_exec: 140 drm_exec_fini(&exec); 141 amdgpu_sync_free(&sync); 142 return r; 143 } 144 145 static int unmap_ring_data(struct amdgpu_device *adev, struct amdgpu_vm *vm, 146 struct amdgpu_bo *bo, struct amdgpu_bo_va *bo_va, 147 uint64_t addr) 148 { 149 struct drm_exec exec; 150 long r; 151 152 drm_exec_init(&exec, 0); 153 drm_exec_until_all_locked(&exec) { 154 r = drm_exec_lock_obj(&exec, &bo->tbo.base); 155 drm_exec_retry_on_contention(&exec); 156 if (unlikely(r)) 157 goto out_unlock; 158 159 r = amdgpu_vm_lock_pd(vm, &exec, 0); 160 drm_exec_retry_on_contention(&exec); 161 if (unlikely(r)) 162 goto out_unlock; 163 } 164 165 166 r = amdgpu_vm_bo_unmap(adev, bo_va, addr); 167 if (r) 168 goto out_unlock; 169 170 amdgpu_vm_bo_del(adev, bo_va); 171 172 out_unlock: 173 drm_exec_fini(&exec); 174 175 return r; 176 } 177 178 static void setup_vpe_queue(struct amdgpu_device *adev, 179 struct umsch_mm_test *test, 180 struct umsch_mm_test_queue_info *qinfo) 181 { 182 struct MQD_INFO *mqd = (struct MQD_INFO *)test->mqd_data_cpu_addr; 183 uint64_t ring_gpu_addr = test->ring_data_gpu_addr; 184 185 mqd->rb_base_lo = (ring_gpu_addr >> 8); 186 mqd->rb_base_hi = (ring_gpu_addr >> 40); 187 mqd->rb_size = PAGE_SIZE / 4; 188 mqd->wptr_val = 0; 189 mqd->rptr_val = 0; 190 mqd->unmapped = 1; 191 192 qinfo->mqd_addr = test->mqd_data_gpu_addr; 193 qinfo->csa_addr = test->ctx_data_gpu_addr + 194 offsetof(struct umsch_mm_test_ctx_data, vpe_ctx_csa); 195 qinfo->doorbell_offset_0 = (adev->doorbell_index.vpe_ring + 1) << 1; 196 qinfo->doorbell_offset_1 = 0; 197 } 198 199 static void setup_vcn_queue(struct amdgpu_device *adev, 200 struct umsch_mm_test *test, 201 struct umsch_mm_test_queue_info *qinfo) 202 { 203 } 204 205 static int add_test_queue(struct amdgpu_device *adev, 206 struct umsch_mm_test *test, 207 struct umsch_mm_test_queue_info *qinfo) 208 { 209 struct umsch_mm_add_queue_input queue_input = {}; 210 int r; 211 212 queue_input.process_id = test->pasid; 213 queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(test->vm->root.bo); 214 215 queue_input.process_va_start = 0; 216 queue_input.process_va_end = (adev->vm_manager.max_pfn - 1) << AMDGPU_GPU_PAGE_SHIFT; 217 218 queue_input.process_quantum = 100000; /* 10ms */ 219 queue_input.process_csa_addr = test->ctx_data_gpu_addr + 220 offsetof(struct umsch_mm_test_ctx_data, process_csa); 221 222 queue_input.context_quantum = 10000; /* 1ms */ 223 queue_input.context_csa_addr = qinfo->csa_addr; 224 225 queue_input.inprocess_context_priority = CONTEXT_PRIORITY_LEVEL_NORMAL; 226 queue_input.context_global_priority_level = CONTEXT_PRIORITY_LEVEL_NORMAL; 227 queue_input.doorbell_offset_0 = qinfo->doorbell_offset_0; 228 queue_input.doorbell_offset_1 = qinfo->doorbell_offset_1; 229 230 queue_input.engine_type = qinfo->engine; 231 queue_input.mqd_addr = qinfo->mqd_addr; 232 queue_input.vm_context_cntl = test->vm_cntx_cntl; 233 234 amdgpu_umsch_mm_lock(&adev->umsch_mm); 235 r = adev->umsch_mm.funcs->add_queue(&adev->umsch_mm, &queue_input); 236 amdgpu_umsch_mm_unlock(&adev->umsch_mm); 237 if (r) 238 return r; 239 240 return 0; 241 } 242 243 static int remove_test_queue(struct amdgpu_device *adev, 244 struct umsch_mm_test *test, 245 struct umsch_mm_test_queue_info *qinfo) 246 { 247 struct umsch_mm_remove_queue_input queue_input = {}; 248 int r; 249 250 queue_input.doorbell_offset_0 = qinfo->doorbell_offset_0; 251 queue_input.doorbell_offset_1 = qinfo->doorbell_offset_1; 252 queue_input.context_csa_addr = qinfo->csa_addr; 253 254 amdgpu_umsch_mm_lock(&adev->umsch_mm); 255 r = adev->umsch_mm.funcs->remove_queue(&adev->umsch_mm, &queue_input); 256 amdgpu_umsch_mm_unlock(&adev->umsch_mm); 257 if (r) 258 return r; 259 260 return 0; 261 } 262 263 static int submit_vpe_queue(struct amdgpu_device *adev, struct umsch_mm_test *test) 264 { 265 struct MQD_INFO *mqd = (struct MQD_INFO *)test->mqd_data_cpu_addr; 266 uint32_t *ring = test->ring_data_cpu_addr + 267 offsetof(struct umsch_mm_test_ring_data, vpe_ring) / 4; 268 uint32_t *ib = test->ring_data_cpu_addr + 269 offsetof(struct umsch_mm_test_ring_data, vpe_ib) / 4; 270 uint64_t ib_gpu_addr = test->ring_data_gpu_addr + 271 offsetof(struct umsch_mm_test_ring_data, vpe_ib); 272 uint32_t *fence = ib + 2048 / 4; 273 uint64_t fence_gpu_addr = ib_gpu_addr + 2048; 274 const uint32_t test_pattern = 0xdeadbeef; 275 int i; 276 277 ib[0] = VPE_CMD_HEADER(VPE_CMD_OPCODE_FENCE, 0); 278 ib[1] = lower_32_bits(fence_gpu_addr); 279 ib[2] = upper_32_bits(fence_gpu_addr); 280 ib[3] = test_pattern; 281 282 ring[0] = VPE_CMD_HEADER(VPE_CMD_OPCODE_INDIRECT, 0); 283 ring[1] = (ib_gpu_addr & 0xffffffe0); 284 ring[2] = upper_32_bits(ib_gpu_addr); 285 ring[3] = 4; 286 ring[4] = 0; 287 ring[5] = 0; 288 289 mqd->wptr_val = (6 << 2); 290 // WDOORBELL32(adev->umsch_mm.agdb_index[CONTEXT_PRIORITY_LEVEL_NORMAL], mqd->wptr_val); 291 292 for (i = 0; i < adev->usec_timeout; i++) { 293 if (*fence == test_pattern) 294 return 0; 295 udelay(1); 296 } 297 298 dev_err(adev->dev, "vpe queue submission timeout\n"); 299 300 return -ETIMEDOUT; 301 } 302 303 static int submit_vcn_queue(struct amdgpu_device *adev, struct umsch_mm_test *test) 304 { 305 return 0; 306 } 307 308 static int setup_umsch_mm_test(struct amdgpu_device *adev, 309 struct umsch_mm_test *test) 310 { 311 struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)]; 312 int r; 313 314 test->vm_cntx_cntl = hub->vm_cntx_cntl; 315 316 test->vm = kzalloc(sizeof(*test->vm), GFP_KERNEL); 317 if (!test->vm) { 318 r = -ENOMEM; 319 return r; 320 } 321 322 r = amdgpu_vm_init(adev, test->vm, -1); 323 if (r) 324 goto error_free_vm; 325 326 r = amdgpu_pasid_alloc(16); 327 if (r < 0) 328 goto error_fini_vm; 329 test->pasid = r; 330 331 r = amdgpu_bo_create_kernel(adev, sizeof(struct umsch_mm_test_ctx_data), 332 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 333 &test->ctx_data_obj, 334 &test->ctx_data_gpu_addr, 335 (void **)&test->ctx_data_cpu_addr); 336 if (r) 337 goto error_free_pasid; 338 339 memset(test->ctx_data_cpu_addr, 0, sizeof(struct umsch_mm_test_ctx_data)); 340 341 r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, 342 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 343 &test->mqd_data_obj, 344 &test->mqd_data_gpu_addr, 345 (void **)&test->mqd_data_cpu_addr); 346 if (r) 347 goto error_free_ctx_data_obj; 348 349 memset(test->mqd_data_cpu_addr, 0, PAGE_SIZE); 350 351 r = amdgpu_bo_create_kernel(adev, sizeof(struct umsch_mm_test_ring_data), 352 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 353 &test->ring_data_obj, 354 NULL, 355 (void **)&test->ring_data_cpu_addr); 356 if (r) 357 goto error_free_mqd_data_obj; 358 359 memset(test->ring_data_cpu_addr, 0, sizeof(struct umsch_mm_test_ring_data)); 360 361 test->ring_data_gpu_addr = AMDGPU_VA_RESERVED_SIZE; 362 r = map_ring_data(adev, test->vm, test->ring_data_obj, &test->bo_va, 363 test->ring_data_gpu_addr, sizeof(struct umsch_mm_test_ring_data)); 364 if (r) 365 goto error_free_ring_data_obj; 366 367 return 0; 368 369 error_free_ring_data_obj: 370 amdgpu_bo_free_kernel(&test->ring_data_obj, NULL, 371 (void **)&test->ring_data_cpu_addr); 372 error_free_mqd_data_obj: 373 amdgpu_bo_free_kernel(&test->mqd_data_obj, &test->mqd_data_gpu_addr, 374 (void **)&test->mqd_data_cpu_addr); 375 error_free_ctx_data_obj: 376 amdgpu_bo_free_kernel(&test->ctx_data_obj, &test->ctx_data_gpu_addr, 377 (void **)&test->ctx_data_cpu_addr); 378 error_free_pasid: 379 amdgpu_pasid_free(test->pasid); 380 error_fini_vm: 381 amdgpu_vm_fini(adev, test->vm); 382 error_free_vm: 383 kfree(test->vm); 384 385 return r; 386 } 387 388 static void cleanup_umsch_mm_test(struct amdgpu_device *adev, 389 struct umsch_mm_test *test) 390 { 391 unmap_ring_data(adev, test->vm, test->ring_data_obj, 392 test->bo_va, test->ring_data_gpu_addr); 393 amdgpu_bo_free_kernel(&test->mqd_data_obj, &test->mqd_data_gpu_addr, 394 (void **)&test->mqd_data_cpu_addr); 395 amdgpu_bo_free_kernel(&test->ring_data_obj, NULL, 396 (void **)&test->ring_data_cpu_addr); 397 amdgpu_bo_free_kernel(&test->ctx_data_obj, &test->ctx_data_gpu_addr, 398 (void **)&test->ctx_data_cpu_addr); 399 amdgpu_pasid_free(test->pasid); 400 amdgpu_vm_fini(adev, test->vm); 401 kfree(test->vm); 402 } 403 404 static int setup_test_queues(struct amdgpu_device *adev, 405 struct umsch_mm_test *test, 406 struct umsch_mm_test_queue_info *qinfo) 407 { 408 int i, r; 409 410 for (i = 0; i < test->num_queues; i++) { 411 if (qinfo[i].engine == UMSCH_SWIP_ENGINE_TYPE_VPE) 412 setup_vpe_queue(adev, test, &qinfo[i]); 413 else 414 setup_vcn_queue(adev, test, &qinfo[i]); 415 416 r = add_test_queue(adev, test, &qinfo[i]); 417 if (r) 418 return r; 419 } 420 421 return 0; 422 } 423 424 static int submit_test_queues(struct amdgpu_device *adev, 425 struct umsch_mm_test *test, 426 struct umsch_mm_test_queue_info *qinfo) 427 { 428 int i, r; 429 430 for (i = 0; i < test->num_queues; i++) { 431 if (qinfo[i].engine == UMSCH_SWIP_ENGINE_TYPE_VPE) 432 r = submit_vpe_queue(adev, test); 433 else 434 r = submit_vcn_queue(adev, test); 435 if (r) 436 return r; 437 } 438 439 return 0; 440 } 441 442 static void cleanup_test_queues(struct amdgpu_device *adev, 443 struct umsch_mm_test *test, 444 struct umsch_mm_test_queue_info *qinfo) 445 { 446 int i; 447 448 for (i = 0; i < test->num_queues; i++) 449 remove_test_queue(adev, test, &qinfo[i]); 450 } 451 452 static int umsch_mm_test(struct amdgpu_device *adev) 453 { 454 struct umsch_mm_test_queue_info qinfo[] = { 455 { .engine = UMSCH_SWIP_ENGINE_TYPE_VPE }, 456 }; 457 struct umsch_mm_test test = { .num_queues = ARRAY_SIZE(qinfo) }; 458 int r; 459 460 r = setup_umsch_mm_test(adev, &test); 461 if (r) 462 return r; 463 464 r = setup_test_queues(adev, &test, qinfo); 465 if (r) 466 goto cleanup; 467 468 r = submit_test_queues(adev, &test, qinfo); 469 if (r) 470 goto cleanup; 471 472 cleanup_test_queues(adev, &test, qinfo); 473 cleanup_umsch_mm_test(adev, &test); 474 475 return 0; 476 477 cleanup: 478 cleanup_test_queues(adev, &test, qinfo); 479 cleanup_umsch_mm_test(adev, &test); 480 return r; 481 } 482 483 int amdgpu_umsch_mm_submit_pkt(struct amdgpu_umsch_mm *umsch, void *pkt, int ndws) 484 { 485 struct amdgpu_ring *ring = &umsch->ring; 486 487 if (amdgpu_ring_alloc(ring, ndws)) 488 return -ENOMEM; 489 490 amdgpu_ring_write_multiple(ring, pkt, ndws); 491 amdgpu_ring_commit(ring); 492 493 return 0; 494 } 495 496 int amdgpu_umsch_mm_query_fence(struct amdgpu_umsch_mm *umsch) 497 { 498 struct amdgpu_ring *ring = &umsch->ring; 499 struct amdgpu_device *adev = ring->adev; 500 int r; 501 502 r = amdgpu_fence_wait_polling(ring, ring->fence_drv.sync_seq, adev->usec_timeout); 503 if (r < 1) { 504 dev_err(adev->dev, "ring umsch timeout, emitted fence %u\n", 505 ring->fence_drv.sync_seq); 506 return -ETIMEDOUT; 507 } 508 509 return 0; 510 } 511 512 static void umsch_mm_ring_set_wptr(struct amdgpu_ring *ring) 513 { 514 struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring; 515 struct amdgpu_device *adev = ring->adev; 516 517 if (ring->use_doorbell) 518 WDOORBELL32(ring->doorbell_index, ring->wptr << 2); 519 else 520 WREG32(umsch->rb_wptr, ring->wptr << 2); 521 } 522 523 static u64 umsch_mm_ring_get_rptr(struct amdgpu_ring *ring) 524 { 525 struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring; 526 struct amdgpu_device *adev = ring->adev; 527 528 return RREG32(umsch->rb_rptr); 529 } 530 531 static u64 umsch_mm_ring_get_wptr(struct amdgpu_ring *ring) 532 { 533 struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring; 534 struct amdgpu_device *adev = ring->adev; 535 536 return RREG32(umsch->rb_wptr); 537 } 538 539 static const struct amdgpu_ring_funcs umsch_v4_0_ring_funcs = { 540 .type = AMDGPU_RING_TYPE_UMSCH_MM, 541 .align_mask = 0, 542 .nop = 0, 543 .support_64bit_ptrs = false, 544 .get_rptr = umsch_mm_ring_get_rptr, 545 .get_wptr = umsch_mm_ring_get_wptr, 546 .set_wptr = umsch_mm_ring_set_wptr, 547 .insert_nop = amdgpu_ring_insert_nop, 548 }; 549 550 int amdgpu_umsch_mm_ring_init(struct amdgpu_umsch_mm *umsch) 551 { 552 struct amdgpu_device *adev = container_of(umsch, struct amdgpu_device, umsch_mm); 553 struct amdgpu_ring *ring = &umsch->ring; 554 555 ring->vm_hub = AMDGPU_MMHUB0(0); 556 ring->use_doorbell = true; 557 ring->no_scheduler = true; 558 ring->doorbell_index = (AMDGPU_NAVI10_DOORBELL64_VCN0_1 << 1) + 6; 559 560 snprintf(ring->name, sizeof(ring->name), "umsch"); 561 562 return amdgpu_ring_init(adev, ring, 1024, NULL, 0, AMDGPU_RING_PRIO_DEFAULT, NULL); 563 } 564 565 int amdgpu_umsch_mm_init_microcode(struct amdgpu_umsch_mm *umsch) 566 { 567 const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr; 568 struct amdgpu_device *adev = umsch->ring.adev; 569 const char *fw_name = NULL; 570 int r; 571 572 switch (amdgpu_ip_version(adev, VCN_HWIP, 0)) { 573 case IP_VERSION(4, 0, 5): 574 fw_name = "amdgpu/umsch_mm_4_0_0.bin"; 575 break; 576 default: 577 break; 578 } 579 580 r = amdgpu_ucode_request(adev, &adev->umsch_mm.fw, fw_name); 581 if (r) { 582 release_firmware(adev->umsch_mm.fw); 583 adev->umsch_mm.fw = NULL; 584 return r; 585 } 586 587 umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *)adev->umsch_mm.fw->data; 588 589 adev->umsch_mm.ucode_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes); 590 adev->umsch_mm.data_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes); 591 592 adev->umsch_mm.irq_start_addr = 593 le32_to_cpu(umsch_mm_hdr->umsch_mm_irq_start_addr_lo) | 594 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_irq_start_addr_hi)) << 32); 595 adev->umsch_mm.uc_start_addr = 596 le32_to_cpu(umsch_mm_hdr->umsch_mm_uc_start_addr_lo) | 597 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_uc_start_addr_hi)) << 32); 598 adev->umsch_mm.data_start_addr = 599 le32_to_cpu(umsch_mm_hdr->umsch_mm_data_start_addr_lo) | 600 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_data_start_addr_hi)) << 32); 601 602 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 603 struct amdgpu_firmware_info *info; 604 605 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_UMSCH_MM_UCODE]; 606 info->ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_UCODE; 607 info->fw = adev->umsch_mm.fw; 608 adev->firmware.fw_size += 609 ALIGN(le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes), PAGE_SIZE); 610 611 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_UMSCH_MM_DATA]; 612 info->ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_DATA; 613 info->fw = adev->umsch_mm.fw; 614 adev->firmware.fw_size += 615 ALIGN(le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes), PAGE_SIZE); 616 } 617 618 return 0; 619 } 620 621 int amdgpu_umsch_mm_allocate_ucode_buffer(struct amdgpu_umsch_mm *umsch) 622 { 623 const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr; 624 struct amdgpu_device *adev = umsch->ring.adev; 625 const __le32 *fw_data; 626 uint32_t fw_size; 627 int r; 628 629 umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *) 630 adev->umsch_mm.fw->data; 631 632 fw_data = (const __le32 *)(adev->umsch_mm.fw->data + 633 le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_offset_bytes)); 634 fw_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes); 635 636 r = amdgpu_bo_create_reserved(adev, fw_size, 637 4 * 1024, AMDGPU_GEM_DOMAIN_VRAM, 638 &adev->umsch_mm.ucode_fw_obj, 639 &adev->umsch_mm.ucode_fw_gpu_addr, 640 (void **)&adev->umsch_mm.ucode_fw_ptr); 641 if (r) { 642 dev_err(adev->dev, "(%d) failed to create umsch_mm fw ucode bo\n", r); 643 return r; 644 } 645 646 memcpy(adev->umsch_mm.ucode_fw_ptr, fw_data, fw_size); 647 648 amdgpu_bo_kunmap(adev->umsch_mm.ucode_fw_obj); 649 amdgpu_bo_unreserve(adev->umsch_mm.ucode_fw_obj); 650 return 0; 651 } 652 653 int amdgpu_umsch_mm_allocate_ucode_data_buffer(struct amdgpu_umsch_mm *umsch) 654 { 655 const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr; 656 struct amdgpu_device *adev = umsch->ring.adev; 657 const __le32 *fw_data; 658 uint32_t fw_size; 659 int r; 660 661 umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *) 662 adev->umsch_mm.fw->data; 663 664 fw_data = (const __le32 *)(adev->umsch_mm.fw->data + 665 le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_offset_bytes)); 666 fw_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes); 667 668 r = amdgpu_bo_create_reserved(adev, fw_size, 669 64 * 1024, AMDGPU_GEM_DOMAIN_VRAM, 670 &adev->umsch_mm.data_fw_obj, 671 &adev->umsch_mm.data_fw_gpu_addr, 672 (void **)&adev->umsch_mm.data_fw_ptr); 673 if (r) { 674 dev_err(adev->dev, "(%d) failed to create umsch_mm fw data bo\n", r); 675 return r; 676 } 677 678 memcpy(adev->umsch_mm.data_fw_ptr, fw_data, fw_size); 679 680 amdgpu_bo_kunmap(adev->umsch_mm.data_fw_obj); 681 amdgpu_bo_unreserve(adev->umsch_mm.data_fw_obj); 682 return 0; 683 } 684 685 int amdgpu_umsch_mm_psp_execute_cmd_buf(struct amdgpu_umsch_mm *umsch) 686 { 687 struct amdgpu_device *adev = umsch->ring.adev; 688 struct amdgpu_firmware_info ucode = { 689 .ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_CMD_BUFFER, 690 .mc_addr = adev->umsch_mm.cmd_buf_gpu_addr, 691 .ucode_size = ((uintptr_t)adev->umsch_mm.cmd_buf_curr_ptr - 692 (uintptr_t)adev->umsch_mm.cmd_buf_ptr), 693 }; 694 695 return psp_execute_ip_fw_load(&adev->psp, &ucode); 696 } 697 698 static void umsch_mm_agdb_index_init(struct amdgpu_device *adev) 699 { 700 uint32_t umsch_mm_agdb_start; 701 int i; 702 703 umsch_mm_agdb_start = adev->doorbell_index.max_assignment + 1; 704 umsch_mm_agdb_start = roundup(umsch_mm_agdb_start, 1024); 705 umsch_mm_agdb_start += (AMDGPU_NAVI10_DOORBELL64_VCN0_1 << 1); 706 707 for (i = 0; i < CONTEXT_PRIORITY_NUM_LEVELS; i++) 708 adev->umsch_mm.agdb_index[i] = umsch_mm_agdb_start + i; 709 } 710 711 static int umsch_mm_init(struct amdgpu_device *adev) 712 { 713 int r; 714 715 adev->umsch_mm.vmid_mask_mm_vpe = 0xf00; 716 adev->umsch_mm.engine_mask = (1 << UMSCH_SWIP_ENGINE_TYPE_VPE); 717 adev->umsch_mm.vpe_hqd_mask = 0xfe; 718 719 r = amdgpu_device_wb_get(adev, &adev->umsch_mm.wb_index); 720 if (r) { 721 dev_err(adev->dev, "failed to alloc wb for umsch: %d\n", r); 722 return r; 723 } 724 725 adev->umsch_mm.sch_ctx_gpu_addr = adev->wb.gpu_addr + 726 (adev->umsch_mm.wb_index * 4); 727 728 r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE, 729 AMDGPU_GEM_DOMAIN_GTT, 730 &adev->umsch_mm.cmd_buf_obj, 731 &adev->umsch_mm.cmd_buf_gpu_addr, 732 (void **)&adev->umsch_mm.cmd_buf_ptr); 733 if (r) { 734 dev_err(adev->dev, "failed to allocate cmdbuf bo %d\n", r); 735 amdgpu_device_wb_free(adev, adev->umsch_mm.wb_index); 736 return r; 737 } 738 739 mutex_init(&adev->umsch_mm.mutex_hidden); 740 741 umsch_mm_agdb_index_init(adev); 742 743 return 0; 744 } 745 746 747 static int umsch_mm_early_init(void *handle) 748 { 749 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 750 751 switch (amdgpu_ip_version(adev, VCN_HWIP, 0)) { 752 case IP_VERSION(4, 0, 5): 753 umsch_mm_v4_0_set_funcs(&adev->umsch_mm); 754 break; 755 default: 756 return -EINVAL; 757 } 758 759 adev->umsch_mm.ring.funcs = &umsch_v4_0_ring_funcs; 760 umsch_mm_set_regs(&adev->umsch_mm); 761 762 return 0; 763 } 764 765 static int umsch_mm_late_init(void *handle) 766 { 767 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 768 769 return umsch_mm_test(adev); 770 } 771 772 static int umsch_mm_sw_init(void *handle) 773 { 774 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 775 int r; 776 777 r = umsch_mm_init(adev); 778 if (r) 779 return r; 780 781 r = umsch_mm_ring_init(&adev->umsch_mm); 782 if (r) 783 return r; 784 785 r = umsch_mm_init_microcode(&adev->umsch_mm); 786 if (r) 787 return r; 788 789 return 0; 790 } 791 792 static int umsch_mm_sw_fini(void *handle) 793 { 794 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 795 796 release_firmware(adev->umsch_mm.fw); 797 adev->umsch_mm.fw = NULL; 798 799 amdgpu_ring_fini(&adev->umsch_mm.ring); 800 801 mutex_destroy(&adev->umsch_mm.mutex_hidden); 802 803 amdgpu_bo_free_kernel(&adev->umsch_mm.cmd_buf_obj, 804 &adev->umsch_mm.cmd_buf_gpu_addr, 805 (void **)&adev->umsch_mm.cmd_buf_ptr); 806 807 amdgpu_device_wb_free(adev, adev->umsch_mm.wb_index); 808 809 return 0; 810 } 811 812 static int umsch_mm_hw_init(void *handle) 813 { 814 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 815 int r; 816 817 r = umsch_mm_load_microcode(&adev->umsch_mm); 818 if (r) 819 return r; 820 821 umsch_mm_ring_start(&adev->umsch_mm); 822 823 r = umsch_mm_set_hw_resources(&adev->umsch_mm); 824 if (r) 825 return r; 826 827 return 0; 828 } 829 830 static int umsch_mm_hw_fini(void *handle) 831 { 832 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 833 834 umsch_mm_ring_stop(&adev->umsch_mm); 835 836 amdgpu_bo_free_kernel(&adev->umsch_mm.data_fw_obj, 837 &adev->umsch_mm.data_fw_gpu_addr, 838 (void **)&adev->umsch_mm.data_fw_ptr); 839 840 amdgpu_bo_free_kernel(&adev->umsch_mm.ucode_fw_obj, 841 &adev->umsch_mm.ucode_fw_gpu_addr, 842 (void **)&adev->umsch_mm.ucode_fw_ptr); 843 return 0; 844 } 845 846 static int umsch_mm_suspend(void *handle) 847 { 848 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 849 850 return umsch_mm_hw_fini(adev); 851 } 852 853 static int umsch_mm_resume(void *handle) 854 { 855 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 856 857 return umsch_mm_hw_init(adev); 858 } 859 860 static const struct amd_ip_funcs umsch_mm_v4_0_ip_funcs = { 861 .name = "umsch_mm_v4_0", 862 .early_init = umsch_mm_early_init, 863 .late_init = umsch_mm_late_init, 864 .sw_init = umsch_mm_sw_init, 865 .sw_fini = umsch_mm_sw_fini, 866 .hw_init = umsch_mm_hw_init, 867 .hw_fini = umsch_mm_hw_fini, 868 .suspend = umsch_mm_suspend, 869 .resume = umsch_mm_resume, 870 }; 871 872 const struct amdgpu_ip_block_version umsch_mm_v4_0_ip_block = { 873 .type = AMD_IP_BLOCK_TYPE_UMSCH_MM, 874 .major = 4, 875 .minor = 0, 876 .rev = 0, 877 .funcs = &umsch_mm_v4_0_ip_funcs, 878 }; 879