1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2023 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/firmware.h> 26 #include <drm/drm_exec.h> 27 28 #include "amdgpu.h" 29 #include "amdgpu_umsch_mm.h" 30 #include "umsch_mm_v4_0.h" 31 32 struct umsch_mm_test_ctx_data { 33 uint8_t process_csa[PAGE_SIZE]; 34 uint8_t vpe_ctx_csa[PAGE_SIZE]; 35 uint8_t vcn_ctx_csa[PAGE_SIZE]; 36 }; 37 38 struct umsch_mm_test_mqd_data { 39 uint8_t vpe_mqd[PAGE_SIZE]; 40 uint8_t vcn_mqd[PAGE_SIZE]; 41 }; 42 43 struct umsch_mm_test_ring_data { 44 uint8_t vpe_ring[PAGE_SIZE]; 45 uint8_t vpe_ib[PAGE_SIZE]; 46 uint8_t vcn_ring[PAGE_SIZE]; 47 uint8_t vcn_ib[PAGE_SIZE]; 48 }; 49 50 struct umsch_mm_test_queue_info { 51 uint64_t mqd_addr; 52 uint64_t csa_addr; 53 uint32_t doorbell_offset_0; 54 uint32_t doorbell_offset_1; 55 enum UMSCH_SWIP_ENGINE_TYPE engine; 56 }; 57 58 struct umsch_mm_test { 59 struct amdgpu_bo *ctx_data_obj; 60 uint64_t ctx_data_gpu_addr; 61 uint32_t *ctx_data_cpu_addr; 62 63 struct amdgpu_bo *mqd_data_obj; 64 uint64_t mqd_data_gpu_addr; 65 uint32_t *mqd_data_cpu_addr; 66 67 struct amdgpu_bo *ring_data_obj; 68 uint64_t ring_data_gpu_addr; 69 uint32_t *ring_data_cpu_addr; 70 71 72 struct amdgpu_vm *vm; 73 struct amdgpu_bo_va *bo_va; 74 uint32_t pasid; 75 uint32_t vm_cntx_cntl; 76 uint32_t num_queues; 77 }; 78 79 int umsch_mm_psp_update_sram(struct amdgpu_device *adev, u32 ucode_size) 80 { 81 struct amdgpu_firmware_info ucode = { 82 .ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_CMD_BUFFER, 83 .mc_addr = adev->umsch_mm.cmd_buf_gpu_addr, 84 .ucode_size = ucode_size, 85 }; 86 87 return psp_execute_ip_fw_load(&adev->psp, &ucode); 88 } 89 90 static int map_ring_data(struct amdgpu_device *adev, struct amdgpu_vm *vm, 91 struct amdgpu_bo *bo, struct amdgpu_bo_va **bo_va, 92 uint64_t addr, uint32_t size) 93 { 94 struct amdgpu_sync sync; 95 struct drm_exec exec; 96 int r; 97 98 amdgpu_sync_create(&sync); 99 100 drm_exec_init(&exec, 0); 101 drm_exec_until_all_locked(&exec) { 102 r = drm_exec_lock_obj(&exec, &bo->tbo.base); 103 drm_exec_retry_on_contention(&exec); 104 if (unlikely(r)) 105 goto error_fini_exec; 106 107 r = amdgpu_vm_lock_pd(vm, &exec, 0); 108 drm_exec_retry_on_contention(&exec); 109 if (unlikely(r)) 110 goto error_fini_exec; 111 } 112 113 *bo_va = amdgpu_vm_bo_add(adev, vm, bo); 114 if (!*bo_va) { 115 r = -ENOMEM; 116 goto error_fini_exec; 117 } 118 119 r = amdgpu_vm_bo_map(adev, *bo_va, addr, 0, size, 120 AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE | 121 AMDGPU_PTE_EXECUTABLE); 122 123 if (r) 124 goto error_del_bo_va; 125 126 127 r = amdgpu_vm_bo_update(adev, *bo_va, false); 128 if (r) 129 goto error_del_bo_va; 130 131 amdgpu_sync_fence(&sync, (*bo_va)->last_pt_update); 132 133 r = amdgpu_vm_update_pdes(adev, vm, false); 134 if (r) 135 goto error_del_bo_va; 136 137 amdgpu_sync_fence(&sync, vm->last_update); 138 139 amdgpu_sync_wait(&sync, false); 140 drm_exec_fini(&exec); 141 142 amdgpu_sync_free(&sync); 143 144 return 0; 145 146 error_del_bo_va: 147 amdgpu_vm_bo_del(adev, *bo_va); 148 amdgpu_sync_free(&sync); 149 150 error_fini_exec: 151 drm_exec_fini(&exec); 152 amdgpu_sync_free(&sync); 153 return r; 154 } 155 156 static int unmap_ring_data(struct amdgpu_device *adev, struct amdgpu_vm *vm, 157 struct amdgpu_bo *bo, struct amdgpu_bo_va *bo_va, 158 uint64_t addr) 159 { 160 struct drm_exec exec; 161 long r; 162 163 drm_exec_init(&exec, 0); 164 drm_exec_until_all_locked(&exec) { 165 r = drm_exec_lock_obj(&exec, &bo->tbo.base); 166 drm_exec_retry_on_contention(&exec); 167 if (unlikely(r)) 168 goto out_unlock; 169 170 r = amdgpu_vm_lock_pd(vm, &exec, 0); 171 drm_exec_retry_on_contention(&exec); 172 if (unlikely(r)) 173 goto out_unlock; 174 } 175 176 177 r = amdgpu_vm_bo_unmap(adev, bo_va, addr); 178 if (r) 179 goto out_unlock; 180 181 amdgpu_vm_bo_del(adev, bo_va); 182 183 out_unlock: 184 drm_exec_fini(&exec); 185 186 return r; 187 } 188 189 static void setup_vpe_queue(struct amdgpu_device *adev, 190 struct umsch_mm_test *test, 191 struct umsch_mm_test_queue_info *qinfo) 192 { 193 struct MQD_INFO *mqd = (struct MQD_INFO *)test->mqd_data_cpu_addr; 194 uint64_t ring_gpu_addr = test->ring_data_gpu_addr; 195 196 mqd->rb_base_lo = (ring_gpu_addr >> 8); 197 mqd->rb_base_hi = (ring_gpu_addr >> 40); 198 mqd->rb_size = PAGE_SIZE / 4; 199 mqd->wptr_val = 0; 200 mqd->rptr_val = 0; 201 mqd->unmapped = 1; 202 203 qinfo->mqd_addr = test->mqd_data_gpu_addr; 204 qinfo->csa_addr = test->ctx_data_gpu_addr + 205 offsetof(struct umsch_mm_test_ctx_data, vpe_ctx_csa); 206 qinfo->doorbell_offset_0 = (adev->doorbell_index.vpe_ring + 1) << 1; 207 qinfo->doorbell_offset_1 = 0; 208 } 209 210 static void setup_vcn_queue(struct amdgpu_device *adev, 211 struct umsch_mm_test *test, 212 struct umsch_mm_test_queue_info *qinfo) 213 { 214 } 215 216 static int add_test_queue(struct amdgpu_device *adev, 217 struct umsch_mm_test *test, 218 struct umsch_mm_test_queue_info *qinfo) 219 { 220 struct umsch_mm_add_queue_input queue_input = {}; 221 int r; 222 223 queue_input.process_id = test->pasid; 224 queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(test->vm->root.bo); 225 226 queue_input.process_va_start = 0; 227 queue_input.process_va_end = (adev->vm_manager.max_pfn - 1) << AMDGPU_GPU_PAGE_SHIFT; 228 229 queue_input.process_quantum = 100000; /* 10ms */ 230 queue_input.process_csa_addr = test->ctx_data_gpu_addr + 231 offsetof(struct umsch_mm_test_ctx_data, process_csa); 232 233 queue_input.context_quantum = 10000; /* 1ms */ 234 queue_input.context_csa_addr = qinfo->csa_addr; 235 236 queue_input.inprocess_context_priority = CONTEXT_PRIORITY_LEVEL_NORMAL; 237 queue_input.context_global_priority_level = CONTEXT_PRIORITY_LEVEL_NORMAL; 238 queue_input.doorbell_offset_0 = qinfo->doorbell_offset_0; 239 queue_input.doorbell_offset_1 = qinfo->doorbell_offset_1; 240 241 queue_input.engine_type = qinfo->engine; 242 queue_input.mqd_addr = qinfo->mqd_addr; 243 queue_input.vm_context_cntl = test->vm_cntx_cntl; 244 245 amdgpu_umsch_mm_lock(&adev->umsch_mm); 246 r = adev->umsch_mm.funcs->add_queue(&adev->umsch_mm, &queue_input); 247 amdgpu_umsch_mm_unlock(&adev->umsch_mm); 248 if (r) 249 return r; 250 251 return 0; 252 } 253 254 static int remove_test_queue(struct amdgpu_device *adev, 255 struct umsch_mm_test *test, 256 struct umsch_mm_test_queue_info *qinfo) 257 { 258 struct umsch_mm_remove_queue_input queue_input = {}; 259 int r; 260 261 queue_input.doorbell_offset_0 = qinfo->doorbell_offset_0; 262 queue_input.doorbell_offset_1 = qinfo->doorbell_offset_1; 263 queue_input.context_csa_addr = qinfo->csa_addr; 264 265 amdgpu_umsch_mm_lock(&adev->umsch_mm); 266 r = adev->umsch_mm.funcs->remove_queue(&adev->umsch_mm, &queue_input); 267 amdgpu_umsch_mm_unlock(&adev->umsch_mm); 268 if (r) 269 return r; 270 271 return 0; 272 } 273 274 static int submit_vpe_queue(struct amdgpu_device *adev, struct umsch_mm_test *test) 275 { 276 struct MQD_INFO *mqd = (struct MQD_INFO *)test->mqd_data_cpu_addr; 277 uint32_t *ring = test->ring_data_cpu_addr + 278 offsetof(struct umsch_mm_test_ring_data, vpe_ring) / 4; 279 uint32_t *ib = test->ring_data_cpu_addr + 280 offsetof(struct umsch_mm_test_ring_data, vpe_ib) / 4; 281 uint64_t ib_gpu_addr = test->ring_data_gpu_addr + 282 offsetof(struct umsch_mm_test_ring_data, vpe_ib); 283 uint32_t *fence = ib + 2048 / 4; 284 uint64_t fence_gpu_addr = ib_gpu_addr + 2048; 285 const uint32_t test_pattern = 0xdeadbeef; 286 int i; 287 288 ib[0] = VPE_CMD_HEADER(VPE_CMD_OPCODE_FENCE, 0); 289 ib[1] = lower_32_bits(fence_gpu_addr); 290 ib[2] = upper_32_bits(fence_gpu_addr); 291 ib[3] = test_pattern; 292 293 ring[0] = VPE_CMD_HEADER(VPE_CMD_OPCODE_INDIRECT, 0); 294 ring[1] = (ib_gpu_addr & 0xffffffe0); 295 ring[2] = upper_32_bits(ib_gpu_addr); 296 ring[3] = 4; 297 ring[4] = 0; 298 ring[5] = 0; 299 300 mqd->wptr_val = (6 << 2); 301 // WDOORBELL32(adev->umsch_mm.agdb_index[CONTEXT_PRIORITY_LEVEL_NORMAL], mqd->wptr_val); 302 303 for (i = 0; i < adev->usec_timeout; i++) { 304 if (*fence == test_pattern) 305 return 0; 306 udelay(1); 307 } 308 309 dev_err(adev->dev, "vpe queue submission timeout\n"); 310 311 return -ETIMEDOUT; 312 } 313 314 static int submit_vcn_queue(struct amdgpu_device *adev, struct umsch_mm_test *test) 315 { 316 return 0; 317 } 318 319 static int setup_umsch_mm_test(struct amdgpu_device *adev, 320 struct umsch_mm_test *test) 321 { 322 struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)]; 323 int r; 324 325 test->vm_cntx_cntl = hub->vm_cntx_cntl; 326 327 test->vm = kzalloc(sizeof(*test->vm), GFP_KERNEL); 328 if (!test->vm) { 329 r = -ENOMEM; 330 return r; 331 } 332 333 r = amdgpu_vm_init(adev, test->vm, -1); 334 if (r) 335 goto error_free_vm; 336 337 r = amdgpu_pasid_alloc(16); 338 if (r < 0) 339 goto error_fini_vm; 340 test->pasid = r; 341 342 r = amdgpu_bo_create_kernel(adev, sizeof(struct umsch_mm_test_ctx_data), 343 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 344 &test->ctx_data_obj, 345 &test->ctx_data_gpu_addr, 346 (void **)&test->ctx_data_cpu_addr); 347 if (r) 348 goto error_free_pasid; 349 350 memset(test->ctx_data_cpu_addr, 0, sizeof(struct umsch_mm_test_ctx_data)); 351 352 r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, 353 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 354 &test->mqd_data_obj, 355 &test->mqd_data_gpu_addr, 356 (void **)&test->mqd_data_cpu_addr); 357 if (r) 358 goto error_free_ctx_data_obj; 359 360 memset(test->mqd_data_cpu_addr, 0, PAGE_SIZE); 361 362 r = amdgpu_bo_create_kernel(adev, sizeof(struct umsch_mm_test_ring_data), 363 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 364 &test->ring_data_obj, 365 NULL, 366 (void **)&test->ring_data_cpu_addr); 367 if (r) 368 goto error_free_mqd_data_obj; 369 370 memset(test->ring_data_cpu_addr, 0, sizeof(struct umsch_mm_test_ring_data)); 371 372 test->ring_data_gpu_addr = AMDGPU_VA_RESERVED_SIZE; 373 r = map_ring_data(adev, test->vm, test->ring_data_obj, &test->bo_va, 374 test->ring_data_gpu_addr, sizeof(struct umsch_mm_test_ring_data)); 375 if (r) 376 goto error_free_ring_data_obj; 377 378 return 0; 379 380 error_free_ring_data_obj: 381 amdgpu_bo_free_kernel(&test->ring_data_obj, NULL, 382 (void **)&test->ring_data_cpu_addr); 383 error_free_mqd_data_obj: 384 amdgpu_bo_free_kernel(&test->mqd_data_obj, &test->mqd_data_gpu_addr, 385 (void **)&test->mqd_data_cpu_addr); 386 error_free_ctx_data_obj: 387 amdgpu_bo_free_kernel(&test->ctx_data_obj, &test->ctx_data_gpu_addr, 388 (void **)&test->ctx_data_cpu_addr); 389 error_free_pasid: 390 amdgpu_pasid_free(test->pasid); 391 error_fini_vm: 392 amdgpu_vm_fini(adev, test->vm); 393 error_free_vm: 394 kfree(test->vm); 395 396 return r; 397 } 398 399 static void cleanup_umsch_mm_test(struct amdgpu_device *adev, 400 struct umsch_mm_test *test) 401 { 402 unmap_ring_data(adev, test->vm, test->ring_data_obj, 403 test->bo_va, test->ring_data_gpu_addr); 404 amdgpu_bo_free_kernel(&test->mqd_data_obj, &test->mqd_data_gpu_addr, 405 (void **)&test->mqd_data_cpu_addr); 406 amdgpu_bo_free_kernel(&test->ring_data_obj, NULL, 407 (void **)&test->ring_data_cpu_addr); 408 amdgpu_bo_free_kernel(&test->ctx_data_obj, &test->ctx_data_gpu_addr, 409 (void **)&test->ctx_data_cpu_addr); 410 amdgpu_pasid_free(test->pasid); 411 amdgpu_vm_fini(adev, test->vm); 412 kfree(test->vm); 413 } 414 415 static int setup_test_queues(struct amdgpu_device *adev, 416 struct umsch_mm_test *test, 417 struct umsch_mm_test_queue_info *qinfo) 418 { 419 int i, r; 420 421 for (i = 0; i < test->num_queues; i++) { 422 if (qinfo[i].engine == UMSCH_SWIP_ENGINE_TYPE_VPE) 423 setup_vpe_queue(adev, test, &qinfo[i]); 424 else 425 setup_vcn_queue(adev, test, &qinfo[i]); 426 427 r = add_test_queue(adev, test, &qinfo[i]); 428 if (r) 429 return r; 430 } 431 432 return 0; 433 } 434 435 static int submit_test_queues(struct amdgpu_device *adev, 436 struct umsch_mm_test *test, 437 struct umsch_mm_test_queue_info *qinfo) 438 { 439 int i, r; 440 441 for (i = 0; i < test->num_queues; i++) { 442 if (qinfo[i].engine == UMSCH_SWIP_ENGINE_TYPE_VPE) 443 r = submit_vpe_queue(adev, test); 444 else 445 r = submit_vcn_queue(adev, test); 446 if (r) 447 return r; 448 } 449 450 return 0; 451 } 452 453 static void cleanup_test_queues(struct amdgpu_device *adev, 454 struct umsch_mm_test *test, 455 struct umsch_mm_test_queue_info *qinfo) 456 { 457 int i; 458 459 for (i = 0; i < test->num_queues; i++) 460 remove_test_queue(adev, test, &qinfo[i]); 461 } 462 463 static int umsch_mm_test(struct amdgpu_device *adev) 464 { 465 struct umsch_mm_test_queue_info qinfo[] = { 466 { .engine = UMSCH_SWIP_ENGINE_TYPE_VPE }, 467 }; 468 struct umsch_mm_test test = { .num_queues = ARRAY_SIZE(qinfo) }; 469 int r; 470 471 r = setup_umsch_mm_test(adev, &test); 472 if (r) 473 return r; 474 475 r = setup_test_queues(adev, &test, qinfo); 476 if (r) 477 goto cleanup; 478 479 r = submit_test_queues(adev, &test, qinfo); 480 if (r) 481 goto cleanup; 482 483 cleanup_test_queues(adev, &test, qinfo); 484 cleanup_umsch_mm_test(adev, &test); 485 486 return 0; 487 488 cleanup: 489 cleanup_test_queues(adev, &test, qinfo); 490 cleanup_umsch_mm_test(adev, &test); 491 return r; 492 } 493 494 int amdgpu_umsch_mm_submit_pkt(struct amdgpu_umsch_mm *umsch, void *pkt, int ndws) 495 { 496 struct amdgpu_ring *ring = &umsch->ring; 497 498 if (amdgpu_ring_alloc(ring, ndws)) 499 return -ENOMEM; 500 501 amdgpu_ring_write_multiple(ring, pkt, ndws); 502 amdgpu_ring_commit(ring); 503 504 return 0; 505 } 506 507 int amdgpu_umsch_mm_query_fence(struct amdgpu_umsch_mm *umsch) 508 { 509 struct amdgpu_ring *ring = &umsch->ring; 510 struct amdgpu_device *adev = ring->adev; 511 int r; 512 513 r = amdgpu_fence_wait_polling(ring, ring->fence_drv.sync_seq, adev->usec_timeout); 514 if (r < 1) { 515 dev_err(adev->dev, "ring umsch timeout, emitted fence %u\n", 516 ring->fence_drv.sync_seq); 517 return -ETIMEDOUT; 518 } 519 520 return 0; 521 } 522 523 static void umsch_mm_ring_set_wptr(struct amdgpu_ring *ring) 524 { 525 struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring; 526 struct amdgpu_device *adev = ring->adev; 527 528 if (ring->use_doorbell) 529 WDOORBELL32(ring->doorbell_index, ring->wptr << 2); 530 else 531 WREG32(umsch->rb_wptr, ring->wptr << 2); 532 } 533 534 static u64 umsch_mm_ring_get_rptr(struct amdgpu_ring *ring) 535 { 536 struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring; 537 struct amdgpu_device *adev = ring->adev; 538 539 return RREG32(umsch->rb_rptr); 540 } 541 542 static u64 umsch_mm_ring_get_wptr(struct amdgpu_ring *ring) 543 { 544 struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring; 545 struct amdgpu_device *adev = ring->adev; 546 547 return RREG32(umsch->rb_wptr); 548 } 549 550 static const struct amdgpu_ring_funcs umsch_v4_0_ring_funcs = { 551 .type = AMDGPU_RING_TYPE_UMSCH_MM, 552 .align_mask = 0, 553 .nop = 0, 554 .support_64bit_ptrs = false, 555 .get_rptr = umsch_mm_ring_get_rptr, 556 .get_wptr = umsch_mm_ring_get_wptr, 557 .set_wptr = umsch_mm_ring_set_wptr, 558 .insert_nop = amdgpu_ring_insert_nop, 559 }; 560 561 int amdgpu_umsch_mm_ring_init(struct amdgpu_umsch_mm *umsch) 562 { 563 struct amdgpu_device *adev = container_of(umsch, struct amdgpu_device, umsch_mm); 564 struct amdgpu_ring *ring = &umsch->ring; 565 566 ring->vm_hub = AMDGPU_MMHUB0(0); 567 ring->use_doorbell = 0; 568 ring->no_scheduler = true; 569 ring->doorbell_index = (AMDGPU_NAVI10_DOORBELL64_VCN0_1 << 1) + 6; 570 571 snprintf(ring->name, sizeof(ring->name), "umsch"); 572 573 return amdgpu_ring_init(adev, ring, 1024, NULL, 0, AMDGPU_RING_PRIO_DEFAULT, NULL); 574 } 575 576 int amdgpu_umsch_mm_init_microcode(struct amdgpu_umsch_mm *umsch) 577 { 578 const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr; 579 struct amdgpu_device *adev = umsch->ring.adev; 580 const char *fw_name = NULL; 581 int r; 582 583 switch (amdgpu_ip_version(adev, VCN_HWIP, 0)) { 584 case IP_VERSION(4, 0, 5): 585 fw_name = "amdgpu/umsch_mm_4_0_0.bin"; 586 break; 587 default: 588 break; 589 } 590 591 r = amdgpu_ucode_request(adev, &adev->umsch_mm.fw, fw_name); 592 if (r) { 593 release_firmware(adev->umsch_mm.fw); 594 adev->umsch_mm.fw = NULL; 595 return r; 596 } 597 598 umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *)adev->umsch_mm.fw->data; 599 600 adev->umsch_mm.ucode_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes); 601 adev->umsch_mm.data_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes); 602 603 adev->umsch_mm.irq_start_addr = 604 le32_to_cpu(umsch_mm_hdr->umsch_mm_irq_start_addr_lo) | 605 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_irq_start_addr_hi)) << 32); 606 adev->umsch_mm.uc_start_addr = 607 le32_to_cpu(umsch_mm_hdr->umsch_mm_uc_start_addr_lo) | 608 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_uc_start_addr_hi)) << 32); 609 adev->umsch_mm.data_start_addr = 610 le32_to_cpu(umsch_mm_hdr->umsch_mm_data_start_addr_lo) | 611 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_data_start_addr_hi)) << 32); 612 613 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 614 struct amdgpu_firmware_info *info; 615 616 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_UMSCH_MM_UCODE]; 617 info->ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_UCODE; 618 info->fw = adev->umsch_mm.fw; 619 adev->firmware.fw_size += 620 ALIGN(le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes), PAGE_SIZE); 621 622 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_UMSCH_MM_DATA]; 623 info->ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_DATA; 624 info->fw = adev->umsch_mm.fw; 625 adev->firmware.fw_size += 626 ALIGN(le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes), PAGE_SIZE); 627 } 628 629 return 0; 630 } 631 632 int amdgpu_umsch_mm_allocate_ucode_buffer(struct amdgpu_umsch_mm *umsch) 633 { 634 const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr; 635 struct amdgpu_device *adev = umsch->ring.adev; 636 const __le32 *fw_data; 637 uint32_t fw_size; 638 int r; 639 640 umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *) 641 adev->umsch_mm.fw->data; 642 643 fw_data = (const __le32 *)(adev->umsch_mm.fw->data + 644 le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_offset_bytes)); 645 fw_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes); 646 647 r = amdgpu_bo_create_reserved(adev, fw_size, 648 4 * 1024, AMDGPU_GEM_DOMAIN_VRAM, 649 &adev->umsch_mm.ucode_fw_obj, 650 &adev->umsch_mm.ucode_fw_gpu_addr, 651 (void **)&adev->umsch_mm.ucode_fw_ptr); 652 if (r) { 653 dev_err(adev->dev, "(%d) failed to create umsch_mm fw ucode bo\n", r); 654 return r; 655 } 656 657 memcpy(adev->umsch_mm.ucode_fw_ptr, fw_data, fw_size); 658 659 amdgpu_bo_kunmap(adev->umsch_mm.ucode_fw_obj); 660 amdgpu_bo_unreserve(adev->umsch_mm.ucode_fw_obj); 661 return 0; 662 } 663 664 int amdgpu_umsch_mm_allocate_ucode_data_buffer(struct amdgpu_umsch_mm *umsch) 665 { 666 const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr; 667 struct amdgpu_device *adev = umsch->ring.adev; 668 const __le32 *fw_data; 669 uint32_t fw_size; 670 int r; 671 672 umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *) 673 adev->umsch_mm.fw->data; 674 675 fw_data = (const __le32 *)(adev->umsch_mm.fw->data + 676 le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_offset_bytes)); 677 fw_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes); 678 679 r = amdgpu_bo_create_reserved(adev, fw_size, 680 64 * 1024, AMDGPU_GEM_DOMAIN_VRAM, 681 &adev->umsch_mm.data_fw_obj, 682 &adev->umsch_mm.data_fw_gpu_addr, 683 (void **)&adev->umsch_mm.data_fw_ptr); 684 if (r) { 685 dev_err(adev->dev, "(%d) failed to create umsch_mm fw data bo\n", r); 686 return r; 687 } 688 689 memcpy(adev->umsch_mm.data_fw_ptr, fw_data, fw_size); 690 691 amdgpu_bo_kunmap(adev->umsch_mm.data_fw_obj); 692 amdgpu_bo_unreserve(adev->umsch_mm.data_fw_obj); 693 return 0; 694 } 695 696 void* amdgpu_umsch_mm_add_cmd(struct amdgpu_umsch_mm *umsch, 697 void* cmd_ptr, uint32_t reg_offset, uint32_t reg_data) 698 { 699 uint32_t* ptr = (uint32_t *)cmd_ptr; 700 701 *ptr++ = (reg_offset << 2); 702 *ptr++ = reg_data; 703 704 return ptr; 705 } 706 707 static void umsch_mm_agdb_index_init(struct amdgpu_device *adev) 708 { 709 uint32_t umsch_mm_agdb_start; 710 int i; 711 712 umsch_mm_agdb_start = adev->doorbell_index.max_assignment + 1; 713 umsch_mm_agdb_start = roundup(umsch_mm_agdb_start, 1024); 714 umsch_mm_agdb_start += (AMDGPU_NAVI10_DOORBELL64_VCN0_1 << 1); 715 716 for (i = 0; i < CONTEXT_PRIORITY_NUM_LEVELS; i++) 717 adev->umsch_mm.agdb_index[i] = umsch_mm_agdb_start + i; 718 } 719 720 static int umsch_mm_init(struct amdgpu_device *adev) 721 { 722 int r; 723 724 adev->umsch_mm.vmid_mask_mm_vpe = 0xf00; 725 adev->umsch_mm.engine_mask = (1 << UMSCH_SWIP_ENGINE_TYPE_VPE); 726 adev->umsch_mm.vpe_hqd_mask = 0xfe; 727 728 r = amdgpu_device_wb_get(adev, &adev->umsch_mm.wb_index); 729 if (r) { 730 dev_err(adev->dev, "failed to alloc wb for umsch: %d\n", r); 731 return r; 732 } 733 734 adev->umsch_mm.sch_ctx_gpu_addr = adev->wb.gpu_addr + 735 (adev->umsch_mm.wb_index * 4); 736 737 r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE, 738 AMDGPU_GEM_DOMAIN_GTT, 739 &adev->umsch_mm.cmd_buf_obj, 740 &adev->umsch_mm.cmd_buf_gpu_addr, 741 (void **)&adev->umsch_mm.cmd_buf_ptr); 742 if (r) { 743 dev_err(adev->dev, "failed to allocate cmdbuf bo %d\n", r); 744 amdgpu_device_wb_free(adev, adev->umsch_mm.wb_index); 745 return r; 746 } 747 748 mutex_init(&adev->umsch_mm.mutex_hidden); 749 750 umsch_mm_agdb_index_init(adev); 751 752 return 0; 753 } 754 755 756 static int umsch_mm_early_init(void *handle) 757 { 758 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 759 760 switch (amdgpu_ip_version(adev, VCN_HWIP, 0)) { 761 case IP_VERSION(4, 0, 5): 762 umsch_mm_v4_0_set_funcs(&adev->umsch_mm); 763 break; 764 default: 765 return -EINVAL; 766 } 767 768 adev->umsch_mm.ring.funcs = &umsch_v4_0_ring_funcs; 769 umsch_mm_set_regs(&adev->umsch_mm); 770 771 return 0; 772 } 773 774 static int umsch_mm_late_init(void *handle) 775 { 776 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 777 778 return umsch_mm_test(adev); 779 } 780 781 static int umsch_mm_sw_init(void *handle) 782 { 783 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 784 int r; 785 786 r = umsch_mm_init(adev); 787 if (r) 788 return r; 789 790 r = umsch_mm_ring_init(&adev->umsch_mm); 791 if (r) 792 return r; 793 794 r = umsch_mm_init_microcode(&adev->umsch_mm); 795 if (r) 796 return r; 797 798 return 0; 799 } 800 801 static int umsch_mm_sw_fini(void *handle) 802 { 803 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 804 805 release_firmware(adev->umsch_mm.fw); 806 adev->umsch_mm.fw = NULL; 807 808 amdgpu_ring_fini(&adev->umsch_mm.ring); 809 810 mutex_destroy(&adev->umsch_mm.mutex_hidden); 811 812 amdgpu_bo_free_kernel(&adev->umsch_mm.cmd_buf_obj, 813 &adev->umsch_mm.cmd_buf_gpu_addr, 814 (void **)&adev->umsch_mm.cmd_buf_ptr); 815 816 amdgpu_device_wb_free(adev, adev->umsch_mm.wb_index); 817 818 return 0; 819 } 820 821 static int umsch_mm_hw_init(void *handle) 822 { 823 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 824 int r; 825 826 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 827 r = umsch_mm_load_microcode(&adev->umsch_mm); 828 if (r) 829 return r; 830 } 831 832 umsch_mm_ring_start(&adev->umsch_mm); 833 834 r = umsch_mm_set_hw_resources(&adev->umsch_mm); 835 if (r) 836 return r; 837 838 return 0; 839 } 840 841 static int umsch_mm_hw_fini(void *handle) 842 { 843 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 844 845 umsch_mm_ring_stop(&adev->umsch_mm); 846 847 amdgpu_bo_free_kernel(&adev->umsch_mm.data_fw_obj, 848 &adev->umsch_mm.data_fw_gpu_addr, 849 (void **)&adev->umsch_mm.data_fw_ptr); 850 851 amdgpu_bo_free_kernel(&adev->umsch_mm.ucode_fw_obj, 852 &adev->umsch_mm.ucode_fw_gpu_addr, 853 (void **)&adev->umsch_mm.ucode_fw_ptr); 854 return 0; 855 } 856 857 static const struct amd_ip_funcs umsch_mm_v4_0_ip_funcs = { 858 .name = "umsch_mm_v4_0", 859 .early_init = umsch_mm_early_init, 860 .late_init = umsch_mm_late_init, 861 .sw_init = umsch_mm_sw_init, 862 .sw_fini = umsch_mm_sw_fini, 863 .hw_init = umsch_mm_hw_init, 864 .hw_fini = umsch_mm_hw_fini, 865 }; 866 867 const struct amdgpu_ip_block_version umsch_mm_v4_0_ip_block = { 868 .type = AMD_IP_BLOCK_TYPE_UMSCH_MM, 869 .major = 4, 870 .minor = 0, 871 .rev = 0, 872 .funcs = &umsch_mm_v4_0_ip_funcs, 873 }; 874