1 /* 2 * Copyright 2019 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/firmware.h> 25 #include <drm/drm_exec.h> 26 27 #include "amdgpu_mes.h" 28 #include "amdgpu.h" 29 #include "soc15_common.h" 30 #include "amdgpu_mes_ctx.h" 31 32 #define AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS 1024 33 #define AMDGPU_ONE_DOORBELL_SIZE 8 34 35 int amdgpu_mes_doorbell_process_slice(struct amdgpu_device *adev) 36 { 37 return roundup(AMDGPU_ONE_DOORBELL_SIZE * 38 AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS, 39 PAGE_SIZE); 40 } 41 42 static int amdgpu_mes_doorbell_init(struct amdgpu_device *adev) 43 { 44 int i; 45 struct amdgpu_mes *mes = &adev->mes; 46 47 /* Bitmap for dynamic allocation of kernel doorbells */ 48 mes->doorbell_bitmap = bitmap_zalloc(PAGE_SIZE / sizeof(u32), GFP_KERNEL); 49 if (!mes->doorbell_bitmap) { 50 DRM_ERROR("Failed to allocate MES doorbell bitmap\n"); 51 return -ENOMEM; 52 } 53 54 mes->num_mes_dbs = PAGE_SIZE / AMDGPU_ONE_DOORBELL_SIZE; 55 for (i = 0; i < AMDGPU_MES_PRIORITY_NUM_LEVELS; i++) { 56 adev->mes.aggregated_doorbells[i] = mes->db_start_dw_offset + i * 2; 57 set_bit(i, mes->doorbell_bitmap); 58 } 59 60 return 0; 61 } 62 63 static int amdgpu_mes_event_log_init(struct amdgpu_device *adev) 64 { 65 int r; 66 67 if (!amdgpu_mes_log_enable) 68 return 0; 69 70 r = amdgpu_bo_create_kernel(adev, adev->mes.event_log_size, PAGE_SIZE, 71 AMDGPU_GEM_DOMAIN_VRAM, 72 &adev->mes.event_log_gpu_obj, 73 &adev->mes.event_log_gpu_addr, 74 &adev->mes.event_log_cpu_addr); 75 if (r) { 76 dev_warn(adev->dev, "failed to create MES event log buffer (%d)", r); 77 return r; 78 } 79 80 memset(adev->mes.event_log_cpu_addr, 0, adev->mes.event_log_size); 81 82 return 0; 83 84 } 85 86 static void amdgpu_mes_doorbell_free(struct amdgpu_device *adev) 87 { 88 bitmap_free(adev->mes.doorbell_bitmap); 89 } 90 91 int amdgpu_mes_init(struct amdgpu_device *adev) 92 { 93 int i, r, num_pipes; 94 95 adev->mes.adev = adev; 96 97 idr_init(&adev->mes.pasid_idr); 98 idr_init(&adev->mes.gang_id_idr); 99 idr_init(&adev->mes.queue_id_idr); 100 ida_init(&adev->mes.doorbell_ida); 101 spin_lock_init(&adev->mes.queue_id_lock); 102 mutex_init(&adev->mes.mutex_hidden); 103 104 for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) 105 spin_lock_init(&adev->mes.ring_lock[i]); 106 107 adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK; 108 adev->mes.vmid_mask_mmhub = 0xffffff00; 109 adev->mes.vmid_mask_gfxhub = adev->gfx.disable_kq ? 0xfffffffe : 0xffffff00; 110 111 num_pipes = adev->gfx.me.num_pipe_per_me * adev->gfx.me.num_me; 112 if (num_pipes > AMDGPU_MES_MAX_GFX_PIPES) 113 dev_warn(adev->dev, "more gfx pipes than supported by MES! (%d vs %d)\n", 114 num_pipes, AMDGPU_MES_MAX_GFX_PIPES); 115 116 for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++) { 117 if (i >= num_pipes) 118 break; 119 if (amdgpu_ip_version(adev, GC_HWIP, 0) >= 120 IP_VERSION(12, 0, 0)) 121 /* 122 * GFX V12 has only one GFX pipe, but 8 queues in it. 123 * GFX pipe 0 queue 0 is being used by Kernel queue. 124 * Set GFX pipe 0 queue 1-7 for MES scheduling 125 * mask = 1111 1110b 126 */ 127 adev->mes.gfx_hqd_mask[i] = adev->gfx.disable_kq ? 0xFF : 0xFE; 128 else 129 /* 130 * GFX pipe 0 queue 0 is being used by Kernel queue. 131 * Set GFX pipe 0 queue 1 for MES scheduling 132 * mask = 10b 133 */ 134 adev->mes.gfx_hqd_mask[i] = adev->gfx.disable_kq ? 0x3 : 0x2; 135 } 136 137 num_pipes = adev->gfx.mec.num_pipe_per_mec * adev->gfx.mec.num_mec; 138 if (num_pipes > AMDGPU_MES_MAX_COMPUTE_PIPES) 139 dev_warn(adev->dev, "more compute pipes than supported by MES! (%d vs %d)\n", 140 num_pipes, AMDGPU_MES_MAX_COMPUTE_PIPES); 141 142 for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) { 143 if (i >= num_pipes) 144 break; 145 adev->mes.compute_hqd_mask[i] = adev->gfx.disable_kq ? 0xF : 0xC; 146 } 147 148 num_pipes = adev->sdma.num_instances; 149 if (num_pipes > AMDGPU_MES_MAX_SDMA_PIPES) 150 dev_warn(adev->dev, "more SDMA pipes than supported by MES! (%d vs %d)\n", 151 num_pipes, AMDGPU_MES_MAX_SDMA_PIPES); 152 153 for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) { 154 if (i >= num_pipes) 155 break; 156 adev->mes.sdma_hqd_mask[i] = 0xfc; 157 } 158 159 for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) { 160 r = amdgpu_device_wb_get(adev, &adev->mes.sch_ctx_offs[i]); 161 if (r) { 162 dev_err(adev->dev, 163 "(%d) ring trail_fence_offs wb alloc failed\n", 164 r); 165 goto error; 166 } 167 adev->mes.sch_ctx_gpu_addr[i] = 168 adev->wb.gpu_addr + (adev->mes.sch_ctx_offs[i] * 4); 169 adev->mes.sch_ctx_ptr[i] = 170 (uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs[i]]; 171 172 r = amdgpu_device_wb_get(adev, 173 &adev->mes.query_status_fence_offs[i]); 174 if (r) { 175 dev_err(adev->dev, 176 "(%d) query_status_fence_offs wb alloc failed\n", 177 r); 178 goto error; 179 } 180 adev->mes.query_status_fence_gpu_addr[i] = adev->wb.gpu_addr + 181 (adev->mes.query_status_fence_offs[i] * 4); 182 adev->mes.query_status_fence_ptr[i] = 183 (uint64_t *)&adev->wb.wb[adev->mes.query_status_fence_offs[i]]; 184 } 185 186 r = amdgpu_mes_doorbell_init(adev); 187 if (r) 188 goto error; 189 190 r = amdgpu_mes_event_log_init(adev); 191 if (r) 192 goto error_doorbell; 193 194 return 0; 195 196 error_doorbell: 197 amdgpu_mes_doorbell_free(adev); 198 error: 199 for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) { 200 if (adev->mes.sch_ctx_ptr[i]) 201 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]); 202 if (adev->mes.query_status_fence_ptr[i]) 203 amdgpu_device_wb_free(adev, 204 adev->mes.query_status_fence_offs[i]); 205 } 206 207 idr_destroy(&adev->mes.pasid_idr); 208 idr_destroy(&adev->mes.gang_id_idr); 209 idr_destroy(&adev->mes.queue_id_idr); 210 ida_destroy(&adev->mes.doorbell_ida); 211 mutex_destroy(&adev->mes.mutex_hidden); 212 return r; 213 } 214 215 void amdgpu_mes_fini(struct amdgpu_device *adev) 216 { 217 int i; 218 219 amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj, 220 &adev->mes.event_log_gpu_addr, 221 &adev->mes.event_log_cpu_addr); 222 223 for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) { 224 if (adev->mes.sch_ctx_ptr[i]) 225 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]); 226 if (adev->mes.query_status_fence_ptr[i]) 227 amdgpu_device_wb_free(adev, 228 adev->mes.query_status_fence_offs[i]); 229 } 230 231 amdgpu_mes_doorbell_free(adev); 232 233 idr_destroy(&adev->mes.pasid_idr); 234 idr_destroy(&adev->mes.gang_id_idr); 235 idr_destroy(&adev->mes.queue_id_idr); 236 ida_destroy(&adev->mes.doorbell_ida); 237 mutex_destroy(&adev->mes.mutex_hidden); 238 } 239 240 int amdgpu_mes_suspend(struct amdgpu_device *adev) 241 { 242 struct mes_suspend_gang_input input; 243 int r; 244 245 if (!amdgpu_mes_suspend_resume_all_supported(adev)) 246 return 0; 247 248 memset(&input, 0x0, sizeof(struct mes_suspend_gang_input)); 249 input.suspend_all_gangs = 1; 250 251 /* 252 * Avoid taking any other locks under MES lock to avoid circular 253 * lock dependencies. 254 */ 255 amdgpu_mes_lock(&adev->mes); 256 r = adev->mes.funcs->suspend_gang(&adev->mes, &input); 257 amdgpu_mes_unlock(&adev->mes); 258 if (r) 259 DRM_ERROR("failed to suspend all gangs"); 260 261 return r; 262 } 263 264 int amdgpu_mes_resume(struct amdgpu_device *adev) 265 { 266 struct mes_resume_gang_input input; 267 int r; 268 269 if (!amdgpu_mes_suspend_resume_all_supported(adev)) 270 return 0; 271 272 memset(&input, 0x0, sizeof(struct mes_resume_gang_input)); 273 input.resume_all_gangs = 1; 274 275 /* 276 * Avoid taking any other locks under MES lock to avoid circular 277 * lock dependencies. 278 */ 279 amdgpu_mes_lock(&adev->mes); 280 r = adev->mes.funcs->resume_gang(&adev->mes, &input); 281 amdgpu_mes_unlock(&adev->mes); 282 if (r) 283 DRM_ERROR("failed to resume all gangs"); 284 285 return r; 286 } 287 288 int amdgpu_mes_reset_hw_queue(struct amdgpu_device *adev, int queue_id) 289 { 290 unsigned long flags; 291 struct amdgpu_mes_queue *queue; 292 struct amdgpu_mes_gang *gang; 293 struct mes_reset_queue_input queue_input; 294 int r; 295 296 /* 297 * Avoid taking any other locks under MES lock to avoid circular 298 * lock dependencies. 299 */ 300 amdgpu_mes_lock(&adev->mes); 301 302 /* remove the mes gang from idr list */ 303 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); 304 305 queue = idr_find(&adev->mes.queue_id_idr, queue_id); 306 if (!queue) { 307 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); 308 amdgpu_mes_unlock(&adev->mes); 309 DRM_ERROR("queue id %d doesn't exist\n", queue_id); 310 return -EINVAL; 311 } 312 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); 313 314 DRM_DEBUG("try to reset queue, doorbell off = 0x%llx\n", 315 queue->doorbell_off); 316 317 gang = queue->gang; 318 queue_input.doorbell_offset = queue->doorbell_off; 319 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr; 320 321 r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); 322 if (r) 323 DRM_ERROR("failed to reset hardware queue, queue id = %d\n", 324 queue_id); 325 326 amdgpu_mes_unlock(&adev->mes); 327 328 return 0; 329 } 330 331 int amdgpu_mes_reset_hw_queue_mmio(struct amdgpu_device *adev, int queue_type, 332 int me_id, int pipe_id, int queue_id, int vmid) 333 { 334 struct mes_reset_queue_input queue_input; 335 int r; 336 337 queue_input.queue_type = queue_type; 338 queue_input.use_mmio = true; 339 queue_input.me_id = me_id; 340 queue_input.pipe_id = pipe_id; 341 queue_input.queue_id = queue_id; 342 queue_input.vmid = vmid; 343 r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); 344 if (r) 345 DRM_ERROR("failed to reset hardware queue by mmio, queue id = %d\n", 346 queue_id); 347 return r; 348 } 349 350 int amdgpu_mes_map_legacy_queue(struct amdgpu_device *adev, 351 struct amdgpu_ring *ring) 352 { 353 struct mes_map_legacy_queue_input queue_input; 354 int r; 355 356 memset(&queue_input, 0, sizeof(queue_input)); 357 358 queue_input.queue_type = ring->funcs->type; 359 queue_input.doorbell_offset = ring->doorbell_index; 360 queue_input.pipe_id = ring->pipe; 361 queue_input.queue_id = ring->queue; 362 queue_input.mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 363 queue_input.wptr_addr = ring->wptr_gpu_addr; 364 365 r = adev->mes.funcs->map_legacy_queue(&adev->mes, &queue_input); 366 if (r) 367 DRM_ERROR("failed to map legacy queue\n"); 368 369 return r; 370 } 371 372 int amdgpu_mes_unmap_legacy_queue(struct amdgpu_device *adev, 373 struct amdgpu_ring *ring, 374 enum amdgpu_unmap_queues_action action, 375 u64 gpu_addr, u64 seq) 376 { 377 struct mes_unmap_legacy_queue_input queue_input; 378 int r; 379 380 queue_input.action = action; 381 queue_input.queue_type = ring->funcs->type; 382 queue_input.doorbell_offset = ring->doorbell_index; 383 queue_input.pipe_id = ring->pipe; 384 queue_input.queue_id = ring->queue; 385 queue_input.trail_fence_addr = gpu_addr; 386 queue_input.trail_fence_data = seq; 387 388 r = adev->mes.funcs->unmap_legacy_queue(&adev->mes, &queue_input); 389 if (r) 390 DRM_ERROR("failed to unmap legacy queue\n"); 391 392 return r; 393 } 394 395 int amdgpu_mes_reset_legacy_queue(struct amdgpu_device *adev, 396 struct amdgpu_ring *ring, 397 unsigned int vmid, 398 bool use_mmio) 399 { 400 struct mes_reset_legacy_queue_input queue_input; 401 int r; 402 403 memset(&queue_input, 0, sizeof(queue_input)); 404 405 queue_input.queue_type = ring->funcs->type; 406 queue_input.doorbell_offset = ring->doorbell_index; 407 queue_input.me_id = ring->me; 408 queue_input.pipe_id = ring->pipe; 409 queue_input.queue_id = ring->queue; 410 queue_input.mqd_addr = ring->mqd_obj ? amdgpu_bo_gpu_offset(ring->mqd_obj) : 0; 411 queue_input.wptr_addr = ring->wptr_gpu_addr; 412 queue_input.vmid = vmid; 413 queue_input.use_mmio = use_mmio; 414 415 r = adev->mes.funcs->reset_legacy_queue(&adev->mes, &queue_input); 416 if (r) 417 DRM_ERROR("failed to reset legacy queue\n"); 418 419 return r; 420 } 421 422 uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg) 423 { 424 struct mes_misc_op_input op_input; 425 int r, val = 0; 426 uint32_t addr_offset = 0; 427 uint64_t read_val_gpu_addr; 428 uint32_t *read_val_ptr; 429 430 if (amdgpu_device_wb_get(adev, &addr_offset)) { 431 DRM_ERROR("critical bug! too many mes readers\n"); 432 goto error; 433 } 434 read_val_gpu_addr = adev->wb.gpu_addr + (addr_offset * 4); 435 read_val_ptr = (uint32_t *)&adev->wb.wb[addr_offset]; 436 op_input.op = MES_MISC_OP_READ_REG; 437 op_input.read_reg.reg_offset = reg; 438 op_input.read_reg.buffer_addr = read_val_gpu_addr; 439 440 if (!adev->mes.funcs->misc_op) { 441 DRM_ERROR("mes rreg is not supported!\n"); 442 goto error; 443 } 444 445 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 446 if (r) 447 DRM_ERROR("failed to read reg (0x%x)\n", reg); 448 else 449 val = *(read_val_ptr); 450 451 error: 452 if (addr_offset) 453 amdgpu_device_wb_free(adev, addr_offset); 454 return val; 455 } 456 457 int amdgpu_mes_wreg(struct amdgpu_device *adev, 458 uint32_t reg, uint32_t val) 459 { 460 struct mes_misc_op_input op_input; 461 int r; 462 463 op_input.op = MES_MISC_OP_WRITE_REG; 464 op_input.write_reg.reg_offset = reg; 465 op_input.write_reg.reg_value = val; 466 467 if (!adev->mes.funcs->misc_op) { 468 DRM_ERROR("mes wreg is not supported!\n"); 469 r = -EINVAL; 470 goto error; 471 } 472 473 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 474 if (r) 475 DRM_ERROR("failed to write reg (0x%x)\n", reg); 476 477 error: 478 return r; 479 } 480 481 int amdgpu_mes_reg_write_reg_wait(struct amdgpu_device *adev, 482 uint32_t reg0, uint32_t reg1, 483 uint32_t ref, uint32_t mask) 484 { 485 struct mes_misc_op_input op_input; 486 int r; 487 488 op_input.op = MES_MISC_OP_WRM_REG_WR_WAIT; 489 op_input.wrm_reg.reg0 = reg0; 490 op_input.wrm_reg.reg1 = reg1; 491 op_input.wrm_reg.ref = ref; 492 op_input.wrm_reg.mask = mask; 493 494 if (!adev->mes.funcs->misc_op) { 495 DRM_ERROR("mes reg_write_reg_wait is not supported!\n"); 496 r = -EINVAL; 497 goto error; 498 } 499 500 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 501 if (r) 502 DRM_ERROR("failed to reg_write_reg_wait\n"); 503 504 error: 505 return r; 506 } 507 508 int amdgpu_mes_reg_wait(struct amdgpu_device *adev, uint32_t reg, 509 uint32_t val, uint32_t mask) 510 { 511 struct mes_misc_op_input op_input; 512 int r; 513 514 op_input.op = MES_MISC_OP_WRM_REG_WAIT; 515 op_input.wrm_reg.reg0 = reg; 516 op_input.wrm_reg.ref = val; 517 op_input.wrm_reg.mask = mask; 518 519 if (!adev->mes.funcs->misc_op) { 520 DRM_ERROR("mes reg wait is not supported!\n"); 521 r = -EINVAL; 522 goto error; 523 } 524 525 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 526 if (r) 527 DRM_ERROR("failed to reg_write_reg_wait\n"); 528 529 error: 530 return r; 531 } 532 533 int amdgpu_mes_set_shader_debugger(struct amdgpu_device *adev, 534 uint64_t process_context_addr, 535 uint32_t spi_gdbg_per_vmid_cntl, 536 const uint32_t *tcp_watch_cntl, 537 uint32_t flags, 538 bool trap_en) 539 { 540 struct mes_misc_op_input op_input = {0}; 541 int r; 542 543 if (!adev->mes.funcs->misc_op) { 544 DRM_ERROR("mes set shader debugger is not supported!\n"); 545 return -EINVAL; 546 } 547 548 op_input.op = MES_MISC_OP_SET_SHADER_DEBUGGER; 549 op_input.set_shader_debugger.process_context_addr = process_context_addr; 550 op_input.set_shader_debugger.flags.u32all = flags; 551 552 /* use amdgpu mes_flush_shader_debugger instead */ 553 if (op_input.set_shader_debugger.flags.process_ctx_flush) 554 return -EINVAL; 555 556 op_input.set_shader_debugger.spi_gdbg_per_vmid_cntl = spi_gdbg_per_vmid_cntl; 557 memcpy(op_input.set_shader_debugger.tcp_watch_cntl, tcp_watch_cntl, 558 sizeof(op_input.set_shader_debugger.tcp_watch_cntl)); 559 560 if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >> 561 AMDGPU_MES_API_VERSION_SHIFT) >= 14) 562 op_input.set_shader_debugger.trap_en = trap_en; 563 564 amdgpu_mes_lock(&adev->mes); 565 566 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 567 if (r) 568 DRM_ERROR("failed to set_shader_debugger\n"); 569 570 amdgpu_mes_unlock(&adev->mes); 571 572 return r; 573 } 574 575 int amdgpu_mes_flush_shader_debugger(struct amdgpu_device *adev, 576 uint64_t process_context_addr) 577 { 578 struct mes_misc_op_input op_input = {0}; 579 int r; 580 581 if (!adev->mes.funcs->misc_op) { 582 DRM_ERROR("mes flush shader debugger is not supported!\n"); 583 return -EINVAL; 584 } 585 586 op_input.op = MES_MISC_OP_SET_SHADER_DEBUGGER; 587 op_input.set_shader_debugger.process_context_addr = process_context_addr; 588 op_input.set_shader_debugger.flags.process_ctx_flush = true; 589 590 amdgpu_mes_lock(&adev->mes); 591 592 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 593 if (r) 594 DRM_ERROR("failed to set_shader_debugger\n"); 595 596 amdgpu_mes_unlock(&adev->mes); 597 598 return r; 599 } 600 601 #define DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(_eng) \ 602 do { \ 603 if (id_offs < AMDGPU_MES_CTX_MAX_OFFS) \ 604 return offsetof(struct amdgpu_mes_ctx_meta_data, \ 605 _eng[ring->idx].slots[id_offs]); \ 606 else if (id_offs == AMDGPU_MES_CTX_RING_OFFS) \ 607 return offsetof(struct amdgpu_mes_ctx_meta_data, \ 608 _eng[ring->idx].ring); \ 609 else if (id_offs == AMDGPU_MES_CTX_IB_OFFS) \ 610 return offsetof(struct amdgpu_mes_ctx_meta_data, \ 611 _eng[ring->idx].ib); \ 612 else if (id_offs == AMDGPU_MES_CTX_PADDING_OFFS) \ 613 return offsetof(struct amdgpu_mes_ctx_meta_data, \ 614 _eng[ring->idx].padding); \ 615 } while(0) 616 617 int amdgpu_mes_ctx_get_offs(struct amdgpu_ring *ring, unsigned int id_offs) 618 { 619 switch (ring->funcs->type) { 620 case AMDGPU_RING_TYPE_GFX: 621 DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(gfx); 622 break; 623 case AMDGPU_RING_TYPE_COMPUTE: 624 DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(compute); 625 break; 626 case AMDGPU_RING_TYPE_SDMA: 627 DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(sdma); 628 break; 629 default: 630 break; 631 } 632 633 WARN_ON(1); 634 return -EINVAL; 635 } 636 637 uint32_t amdgpu_mes_get_aggregated_doorbell_index(struct amdgpu_device *adev, 638 enum amdgpu_mes_priority_level prio) 639 { 640 return adev->mes.aggregated_doorbells[prio]; 641 } 642 643 int amdgpu_mes_init_microcode(struct amdgpu_device *adev, int pipe) 644 { 645 const struct mes_firmware_header_v1_0 *mes_hdr; 646 struct amdgpu_firmware_info *info; 647 char ucode_prefix[30]; 648 char fw_name[50]; 649 bool need_retry = false; 650 u32 *ucode_ptr; 651 int r; 652 653 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, 654 sizeof(ucode_prefix)); 655 if (adev->enable_uni_mes) { 656 snprintf(fw_name, sizeof(fw_name), 657 "amdgpu/%s_uni_mes.bin", ucode_prefix); 658 } else if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) && 659 amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)) { 660 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes%s.bin", 661 ucode_prefix, 662 pipe == AMDGPU_MES_SCHED_PIPE ? "_2" : "1"); 663 need_retry = true; 664 } else { 665 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes%s.bin", 666 ucode_prefix, 667 pipe == AMDGPU_MES_SCHED_PIPE ? "" : "1"); 668 } 669 670 r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], AMDGPU_UCODE_REQUIRED, 671 "%s", fw_name); 672 if (r && need_retry && pipe == AMDGPU_MES_SCHED_PIPE) { 673 dev_info(adev->dev, "try to fall back to %s_mes.bin\n", ucode_prefix); 674 r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], 675 AMDGPU_UCODE_REQUIRED, 676 "amdgpu/%s_mes.bin", ucode_prefix); 677 } 678 679 if (r) 680 goto out; 681 682 mes_hdr = (const struct mes_firmware_header_v1_0 *) 683 adev->mes.fw[pipe]->data; 684 adev->mes.uc_start_addr[pipe] = 685 le32_to_cpu(mes_hdr->mes_uc_start_addr_lo) | 686 ((uint64_t)(le32_to_cpu(mes_hdr->mes_uc_start_addr_hi)) << 32); 687 adev->mes.data_start_addr[pipe] = 688 le32_to_cpu(mes_hdr->mes_data_start_addr_lo) | 689 ((uint64_t)(le32_to_cpu(mes_hdr->mes_data_start_addr_hi)) << 32); 690 ucode_ptr = (u32 *)(adev->mes.fw[pipe]->data + 691 sizeof(union amdgpu_firmware_header)); 692 adev->mes.fw_version[pipe] = 693 le32_to_cpu(ucode_ptr[24]) & AMDGPU_MES_VERSION_MASK; 694 695 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 696 int ucode, ucode_data; 697 698 if (pipe == AMDGPU_MES_SCHED_PIPE) { 699 ucode = AMDGPU_UCODE_ID_CP_MES; 700 ucode_data = AMDGPU_UCODE_ID_CP_MES_DATA; 701 } else { 702 ucode = AMDGPU_UCODE_ID_CP_MES1; 703 ucode_data = AMDGPU_UCODE_ID_CP_MES1_DATA; 704 } 705 706 info = &adev->firmware.ucode[ucode]; 707 info->ucode_id = ucode; 708 info->fw = adev->mes.fw[pipe]; 709 adev->firmware.fw_size += 710 ALIGN(le32_to_cpu(mes_hdr->mes_ucode_size_bytes), 711 PAGE_SIZE); 712 713 info = &adev->firmware.ucode[ucode_data]; 714 info->ucode_id = ucode_data; 715 info->fw = adev->mes.fw[pipe]; 716 adev->firmware.fw_size += 717 ALIGN(le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes), 718 PAGE_SIZE); 719 } 720 721 return 0; 722 out: 723 amdgpu_ucode_release(&adev->mes.fw[pipe]); 724 return r; 725 } 726 727 bool amdgpu_mes_suspend_resume_all_supported(struct amdgpu_device *adev) 728 { 729 uint32_t mes_rev = adev->mes.sched_version & AMDGPU_MES_VERSION_MASK; 730 bool is_supported = false; 731 732 if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) && 733 amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0) && 734 mes_rev >= 0x63) 735 is_supported = true; 736 737 return is_supported; 738 } 739 740 /* Fix me -- node_id is used to identify the correct MES instances in the future */ 741 static int amdgpu_mes_set_enforce_isolation(struct amdgpu_device *adev, 742 uint32_t node_id, bool enable) 743 { 744 struct mes_misc_op_input op_input = {0}; 745 int r; 746 747 op_input.op = MES_MISC_OP_CHANGE_CONFIG; 748 op_input.change_config.option.limit_single_process = enable ? 1 : 0; 749 750 if (!adev->mes.funcs->misc_op) { 751 dev_err(adev->dev, "mes change config is not supported!\n"); 752 r = -EINVAL; 753 goto error; 754 } 755 756 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 757 if (r) 758 dev_err(adev->dev, "failed to change_config.\n"); 759 760 error: 761 return r; 762 } 763 764 int amdgpu_mes_update_enforce_isolation(struct amdgpu_device *adev) 765 { 766 int i, r = 0; 767 768 if (adev->enable_mes && adev->gfx.enable_cleaner_shader) { 769 mutex_lock(&adev->enforce_isolation_mutex); 770 for (i = 0; i < (adev->xcp_mgr ? adev->xcp_mgr->num_xcps : 1); i++) { 771 if (adev->enforce_isolation[i] == AMDGPU_ENFORCE_ISOLATION_ENABLE) 772 r |= amdgpu_mes_set_enforce_isolation(adev, i, true); 773 else 774 r |= amdgpu_mes_set_enforce_isolation(adev, i, false); 775 } 776 mutex_unlock(&adev->enforce_isolation_mutex); 777 } 778 return r; 779 } 780 781 #if defined(CONFIG_DEBUG_FS) 782 783 static int amdgpu_debugfs_mes_event_log_show(struct seq_file *m, void *unused) 784 { 785 struct amdgpu_device *adev = m->private; 786 uint32_t *mem = (uint32_t *)(adev->mes.event_log_cpu_addr); 787 788 seq_hex_dump(m, "", DUMP_PREFIX_OFFSET, 32, 4, 789 mem, adev->mes.event_log_size, false); 790 791 return 0; 792 } 793 794 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_mes_event_log); 795 796 #endif 797 798 void amdgpu_debugfs_mes_event_log_init(struct amdgpu_device *adev) 799 { 800 801 #if defined(CONFIG_DEBUG_FS) 802 struct drm_minor *minor = adev_to_drm(adev)->primary; 803 struct dentry *root = minor->debugfs_root; 804 if (adev->enable_mes && amdgpu_mes_log_enable) 805 debugfs_create_file("amdgpu_mes_event_log", 0444, root, 806 adev, &amdgpu_debugfs_mes_event_log_fops); 807 808 #endif 809 } 810