1 /* 2 * Copyright 2019 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/firmware.h> 25 #include <drm/drm_exec.h> 26 27 #include "amdgpu_mes.h" 28 #include "amdgpu.h" 29 #include "soc15_common.h" 30 #include "amdgpu_mes_ctx.h" 31 32 #define AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS 1024 33 #define AMDGPU_ONE_DOORBELL_SIZE 8 34 35 int amdgpu_mes_doorbell_process_slice(struct amdgpu_device *adev) 36 { 37 return roundup(AMDGPU_ONE_DOORBELL_SIZE * 38 AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS, 39 PAGE_SIZE); 40 } 41 42 static int amdgpu_mes_doorbell_init(struct amdgpu_device *adev) 43 { 44 int i; 45 struct amdgpu_mes *mes = &adev->mes; 46 47 /* Bitmap for dynamic allocation of kernel doorbells */ 48 mes->doorbell_bitmap = bitmap_zalloc(PAGE_SIZE / sizeof(u32), GFP_KERNEL); 49 if (!mes->doorbell_bitmap) { 50 dev_err(adev->dev, "Failed to allocate MES doorbell bitmap\n"); 51 return -ENOMEM; 52 } 53 54 mes->num_mes_dbs = PAGE_SIZE / AMDGPU_ONE_DOORBELL_SIZE; 55 for (i = 0; i < AMDGPU_MES_PRIORITY_NUM_LEVELS; i++) { 56 adev->mes.aggregated_doorbells[i] = mes->db_start_dw_offset + i * 2; 57 set_bit(i, mes->doorbell_bitmap); 58 } 59 60 return 0; 61 } 62 63 static int amdgpu_mes_event_log_init(struct amdgpu_device *adev) 64 { 65 int r; 66 67 if (!amdgpu_mes_log_enable) 68 return 0; 69 70 r = amdgpu_bo_create_kernel(adev, adev->mes.event_log_size, PAGE_SIZE, 71 AMDGPU_GEM_DOMAIN_VRAM, 72 &adev->mes.event_log_gpu_obj, 73 &adev->mes.event_log_gpu_addr, 74 &adev->mes.event_log_cpu_addr); 75 if (r) { 76 dev_warn(adev->dev, "failed to create MES event log buffer (%d)", r); 77 return r; 78 } 79 80 memset(adev->mes.event_log_cpu_addr, 0, adev->mes.event_log_size); 81 82 return 0; 83 84 } 85 86 static void amdgpu_mes_doorbell_free(struct amdgpu_device *adev) 87 { 88 bitmap_free(adev->mes.doorbell_bitmap); 89 } 90 91 static inline u32 amdgpu_mes_get_hqd_mask(u32 num_pipe, 92 u32 num_hqd_per_pipe, 93 u32 num_reserved_hqd) 94 { 95 if (num_pipe == 0) 96 return 0; 97 98 u32 total_hqd_mask = (u32)((1ULL << num_hqd_per_pipe) - 1); 99 u32 reserved_hqd_mask = (u32)((1ULL << DIV_ROUND_UP(num_reserved_hqd, num_pipe)) - 1); 100 101 return (total_hqd_mask & ~reserved_hqd_mask); 102 } 103 104 int amdgpu_mes_init(struct amdgpu_device *adev) 105 { 106 int i, r, num_pipes, num_queues = 0; 107 u32 total_vmid_mask, reserved_vmid_mask; 108 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; 109 u32 gfx_hqd_mask = amdgpu_mes_get_hqd_mask(adev->gfx.me.num_pipe_per_me, 110 adev->gfx.me.num_queue_per_pipe, 111 adev->gfx.disable_kq ? 0 : adev->gfx.num_gfx_rings); 112 u32 compute_hqd_mask = amdgpu_mes_get_hqd_mask(adev->gfx.mec.num_pipe_per_mec, 113 adev->gfx.mec.num_queue_per_pipe, 114 adev->gfx.disable_kq ? 0 : adev->gfx.num_compute_rings); 115 116 adev->mes.adev = adev; 117 118 ida_init(&adev->mes.doorbell_ida); 119 spin_lock_init(&adev->mes.queue_id_lock); 120 mutex_init(&adev->mes.mutex_hidden); 121 122 for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) 123 spin_lock_init(&adev->mes.ring_lock[i]); 124 125 adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK; 126 total_vmid_mask = (u32)((1UL << 16) - 1); 127 reserved_vmid_mask = (u32)((1UL << adev->vm_manager.first_kfd_vmid) - 1); 128 129 adev->mes.vmid_mask_mmhub = 0xFF00; 130 adev->mes.vmid_mask_gfxhub = total_vmid_mask & ~reserved_vmid_mask; 131 132 num_pipes = adev->gfx.me.num_pipe_per_me * adev->gfx.me.num_me; 133 if (num_pipes > AMDGPU_MES_MAX_GFX_PIPES) 134 dev_warn(adev->dev, "more gfx pipes than supported by MES! (%d vs %d)\n", 135 num_pipes, AMDGPU_MES_MAX_GFX_PIPES); 136 137 for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++) { 138 if (i >= num_pipes) 139 break; 140 141 adev->mes.gfx_hqd_mask[i] = gfx_hqd_mask; 142 } 143 144 num_pipes = adev->gfx.mec.num_pipe_per_mec * adev->gfx.mec.num_mec; 145 if (num_pipes > AMDGPU_MES_MAX_COMPUTE_PIPES) 146 dev_warn(adev->dev, "more compute pipes than supported by MES! (%d vs %d)\n", 147 num_pipes, AMDGPU_MES_MAX_COMPUTE_PIPES); 148 149 for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) { 150 /* 151 * Currently, only MEC1 is used for both kernel and user compute queue. 152 * To enable other MEC, we need to redistribute queues per pipe and 153 * adjust queue resource shared with kfd that needs a separate patch. 154 * Skip other MEC for now to avoid potential issues. 155 */ 156 if (i >= adev->gfx.mec.num_pipe_per_mec) 157 break; 158 159 adev->mes.compute_hqd_mask[i] = compute_hqd_mask; 160 } 161 162 num_pipes = adev->sdma.num_inst_per_xcc ? 163 adev->sdma.num_inst_per_xcc : adev->sdma.num_instances; 164 if (num_pipes > AMDGPU_MES_MAX_SDMA_PIPES) 165 dev_warn(adev->dev, "more SDMA pipes than supported by MES! (%d vs %d)\n", 166 num_pipes, AMDGPU_MES_MAX_SDMA_PIPES); 167 168 for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) { 169 if (i >= num_pipes) 170 break; 171 adev->mes.sdma_hqd_mask[i] = 0xfc; 172 } 173 174 dev_info(adev->dev, 175 "MES: vmid_mask_mmhub 0x%08x, vmid_mask_gfxhub 0x%08x\n", 176 adev->mes.vmid_mask_mmhub, 177 adev->mes.vmid_mask_gfxhub); 178 179 dev_info(adev->dev, 180 "MES: gfx_hqd_mask 0x%08x, compute_hqd_mask 0x%08x, sdma_hqd_mask 0x%08x\n", 181 adev->mes.gfx_hqd_mask[0], 182 adev->mes.compute_hqd_mask[0], 183 adev->mes.sdma_hqd_mask[0]); 184 185 for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) { 186 r = amdgpu_device_wb_get(adev, &adev->mes.sch_ctx_offs[i]); 187 if (r) { 188 dev_err(adev->dev, 189 "(%d) ring trail_fence_offs wb alloc failed\n", 190 r); 191 goto error; 192 } 193 adev->mes.sch_ctx_gpu_addr[i] = 194 adev->wb.gpu_addr + (adev->mes.sch_ctx_offs[i] * 4); 195 adev->mes.sch_ctx_ptr[i] = 196 (uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs[i]]; 197 198 r = amdgpu_device_wb_get(adev, 199 &adev->mes.query_status_fence_offs[i]); 200 if (r) { 201 dev_err(adev->dev, 202 "(%d) query_status_fence_offs wb alloc failed\n", 203 r); 204 goto error; 205 } 206 adev->mes.query_status_fence_gpu_addr[i] = adev->wb.gpu_addr + 207 (adev->mes.query_status_fence_offs[i] * 4); 208 adev->mes.query_status_fence_ptr[i] = 209 (uint64_t *)&adev->wb.wb[adev->mes.query_status_fence_offs[i]]; 210 } 211 212 r = amdgpu_mes_doorbell_init(adev); 213 if (r) 214 goto error; 215 216 r = amdgpu_mes_event_log_init(adev); 217 if (r) 218 goto error_doorbell; 219 220 if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) { 221 /* When queue/pipe reset is done in MES instead of in the 222 * driver, MES passes hung queues information to the driver in 223 * hung_queue_hqd_info. Calculate required space to store this 224 * information. 225 */ 226 for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++) 227 num_queues += hweight32(adev->mes.gfx_hqd_mask[i]); 228 229 for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) 230 num_queues += hweight32(adev->mes.compute_hqd_mask[i]); 231 232 for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) 233 num_queues += hweight32(adev->mes.sdma_hqd_mask[i]) * num_xcc; 234 235 adev->mes.hung_queue_hqd_info_offset = num_queues; 236 adev->mes.hung_queue_db_array_size = num_queues * 2; 237 } 238 239 if (adev->mes.hung_queue_db_array_size) { 240 for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) { 241 r = amdgpu_bo_create_kernel(adev, 242 adev->mes.hung_queue_db_array_size * sizeof(u32), 243 PAGE_SIZE, 244 AMDGPU_GEM_DOMAIN_GTT, 245 &adev->mes.hung_queue_db_array_gpu_obj[i], 246 &adev->mes.hung_queue_db_array_gpu_addr[i], 247 &adev->mes.hung_queue_db_array_cpu_addr[i]); 248 if (r) { 249 dev_warn(adev->dev, "failed to create MES hung db array buffer (%d)", r); 250 goto error_doorbell; 251 } 252 } 253 } 254 255 adev->gfx.mec.mes_hung_db_array = 256 kcalloc(amdgpu_mes_get_hung_queue_db_array_size(adev), 257 sizeof(u32), GFP_KERNEL); 258 259 return 0; 260 261 error_doorbell: 262 amdgpu_mes_doorbell_free(adev); 263 error: 264 for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) { 265 if (adev->mes.sch_ctx_ptr[i]) 266 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]); 267 if (adev->mes.query_status_fence_ptr[i]) 268 amdgpu_device_wb_free(adev, 269 adev->mes.query_status_fence_offs[i]); 270 if (adev->mes.hung_queue_db_array_gpu_obj[i]) 271 amdgpu_bo_free_kernel(&adev->mes.hung_queue_db_array_gpu_obj[i], 272 &adev->mes.hung_queue_db_array_gpu_addr[i], 273 &adev->mes.hung_queue_db_array_cpu_addr[i]); 274 } 275 276 ida_destroy(&adev->mes.doorbell_ida); 277 mutex_destroy(&adev->mes.mutex_hidden); 278 return r; 279 } 280 281 void amdgpu_mes_fini(struct amdgpu_device *adev) 282 { 283 int i; 284 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; 285 286 kfree(adev->gfx.mec.mes_hung_db_array); 287 288 amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj, 289 &adev->mes.event_log_gpu_addr, 290 &adev->mes.event_log_cpu_addr); 291 292 for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) { 293 if (adev->mes.hung_queue_db_array_gpu_obj[i]) 294 amdgpu_bo_free_kernel(&adev->mes.hung_queue_db_array_gpu_obj[i], 295 &adev->mes.hung_queue_db_array_gpu_addr[i], 296 &adev->mes.hung_queue_db_array_cpu_addr[i]); 297 if (adev->mes.sch_ctx_ptr[i]) 298 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]); 299 if (adev->mes.query_status_fence_ptr[i]) 300 amdgpu_device_wb_free(adev, 301 adev->mes.query_status_fence_offs[i]); 302 } 303 304 amdgpu_mes_doorbell_free(adev); 305 306 ida_destroy(&adev->mes.doorbell_ida); 307 mutex_destroy(&adev->mes.mutex_hidden); 308 } 309 310 int amdgpu_mes_suspend(struct amdgpu_device *adev, u32 xcc_id) 311 { 312 struct mes_suspend_gang_input input; 313 int r; 314 315 if (!amdgpu_mes_suspend_resume_all_supported(adev)) 316 return 0; 317 318 memset(&input, 0x0, sizeof(struct mes_suspend_gang_input)); 319 input.suspend_all_gangs = 1; 320 input.xcc_id = xcc_id; 321 if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) && 322 ((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x71)) 323 input.suspend_all_sdma_gangs = 1; 324 325 /* 326 * Avoid taking any other locks under MES lock to avoid circular 327 * lock dependencies. 328 */ 329 amdgpu_mes_lock(&adev->mes); 330 r = adev->mes.funcs->suspend_gang(&adev->mes, &input); 331 amdgpu_mes_unlock(&adev->mes); 332 if (r) 333 dev_err(adev->dev, "failed to suspend all gangs"); 334 335 return r; 336 } 337 338 int amdgpu_mes_resume(struct amdgpu_device *adev, u32 xcc_id) 339 { 340 struct mes_resume_gang_input input; 341 int r; 342 343 if (!amdgpu_mes_suspend_resume_all_supported(adev)) 344 return 0; 345 346 memset(&input, 0x0, sizeof(struct mes_resume_gang_input)); 347 input.resume_all_gangs = 1; 348 input.xcc_id = xcc_id; 349 350 /* 351 * Avoid taking any other locks under MES lock to avoid circular 352 * lock dependencies. 353 */ 354 amdgpu_mes_lock(&adev->mes); 355 r = adev->mes.funcs->resume_gang(&adev->mes, &input); 356 amdgpu_mes_unlock(&adev->mes); 357 if (r) 358 dev_err(adev->dev, "failed to resume all gangs"); 359 360 return r; 361 } 362 363 int amdgpu_mes_map_legacy_queue(struct amdgpu_device *adev, 364 struct amdgpu_ring *ring, uint32_t xcc_id) 365 { 366 struct mes_map_legacy_queue_input queue_input; 367 int r; 368 369 memset(&queue_input, 0, sizeof(queue_input)); 370 371 queue_input.xcc_id = xcc_id; 372 queue_input.queue_type = ring->funcs->type; 373 queue_input.doorbell_offset = ring->doorbell_index; 374 queue_input.pipe_id = ring->pipe; 375 queue_input.queue_id = ring->queue; 376 queue_input.mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 377 queue_input.wptr_addr = ring->wptr_gpu_addr; 378 379 amdgpu_mes_lock(&adev->mes); 380 r = adev->mes.funcs->map_legacy_queue(&adev->mes, &queue_input); 381 amdgpu_mes_unlock(&adev->mes); 382 if (r) 383 dev_err(adev->dev, "failed to map legacy queue\n"); 384 385 return r; 386 } 387 388 int amdgpu_mes_unmap_legacy_queue(struct amdgpu_device *adev, 389 struct amdgpu_ring *ring, 390 enum amdgpu_unmap_queues_action action, 391 u64 gpu_addr, u64 seq, uint32_t xcc_id) 392 { 393 struct mes_unmap_legacy_queue_input queue_input; 394 int r; 395 396 queue_input.xcc_id = xcc_id; 397 queue_input.action = action; 398 queue_input.queue_type = ring->funcs->type; 399 queue_input.doorbell_offset = ring->doorbell_index; 400 queue_input.pipe_id = ring->pipe; 401 queue_input.queue_id = ring->queue; 402 queue_input.trail_fence_addr = gpu_addr; 403 queue_input.trail_fence_data = seq; 404 405 amdgpu_mes_lock(&adev->mes); 406 r = adev->mes.funcs->unmap_legacy_queue(&adev->mes, &queue_input); 407 amdgpu_mes_unlock(&adev->mes); 408 if (r) 409 dev_err(adev->dev, "failed to unmap legacy queue\n"); 410 411 return r; 412 } 413 414 int amdgpu_mes_reset_legacy_queue(struct amdgpu_device *adev, 415 struct amdgpu_ring *ring, 416 unsigned int vmid, 417 bool use_mmio, 418 uint32_t xcc_id) 419 { 420 struct mes_reset_queue_input queue_input; 421 int r; 422 423 memset(&queue_input, 0, sizeof(queue_input)); 424 425 queue_input.xcc_id = xcc_id; 426 queue_input.queue_type = ring->funcs->type; 427 queue_input.doorbell_offset = ring->doorbell_index; 428 queue_input.me_id = ring->me; 429 queue_input.pipe_id = ring->pipe; 430 queue_input.queue_id = ring->queue; 431 queue_input.mqd_addr = ring->mqd_obj ? amdgpu_bo_gpu_offset(ring->mqd_obj) : 0; 432 queue_input.wptr_addr = ring->wptr_gpu_addr; 433 queue_input.vmid = vmid; 434 queue_input.use_mmio = use_mmio; 435 queue_input.is_kq = true; 436 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) 437 queue_input.legacy_gfx = true; 438 439 amdgpu_mes_lock(&adev->mes); 440 r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); 441 amdgpu_mes_unlock(&adev->mes); 442 if (r) 443 dev_err(adev->dev, "failed to reset legacy queue\n"); 444 445 return r; 446 } 447 448 int amdgpu_mes_reset_queue_mmio(struct amdgpu_device *adev, 449 int queue_type, 450 unsigned int vmid, 451 unsigned int me, 452 unsigned int pipe, 453 unsigned int queue, 454 uint32_t xcc_id) 455 { 456 struct mes_reset_queue_input queue_input; 457 int r; 458 459 memset(&queue_input, 0, sizeof(queue_input)); 460 461 queue_input.xcc_id = xcc_id; 462 queue_input.me_id = me; 463 queue_input.pipe_id = pipe; 464 queue_input.queue_id = queue; 465 queue_input.vmid = vmid; 466 queue_input.queue_type = queue_type; 467 queue_input.use_mmio = true; 468 469 amdgpu_mes_lock(&adev->mes); 470 r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); 471 amdgpu_mes_unlock(&adev->mes); 472 if (r) 473 dev_err(adev->dev, "failed to reset legacy queue\n"); 474 475 return r; 476 } 477 478 int amdgpu_mes_reset_user_queue(struct amdgpu_device *adev, 479 int queue_type, 480 unsigned int doorbell_index, 481 unsigned int xcc_id) 482 { 483 struct mes_reset_queue_input queue_input; 484 int r; 485 486 memset(&queue_input, 0, sizeof(queue_input)); 487 488 queue_input.xcc_id = xcc_id; 489 queue_input.queue_type = queue_type; 490 queue_input.doorbell_offset = doorbell_index; 491 492 amdgpu_mes_lock(&adev->mes); 493 r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); 494 amdgpu_mes_unlock(&adev->mes); 495 if (r) 496 dev_err(adev->dev, "failed to reset user queue\n"); 497 498 return r; 499 } 500 501 int amdgpu_mes_get_hung_queue_db_array_size(struct amdgpu_device *adev) 502 { 503 return adev->mes.hung_queue_db_array_size; 504 } 505 506 int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev, 507 int queue_type, 508 bool detect_only, 509 unsigned int *hung_db_num, 510 u32 *hung_db_array, 511 uint32_t xcc_id) 512 { 513 struct mes_detect_and_reset_queue_input input; 514 u32 *db_array = adev->mes.hung_queue_db_array_cpu_addr[xcc_id]; 515 int hqd_info_offset = adev->mes.hung_queue_hqd_info_offset, r, i; 516 517 if (!hung_db_num || !hung_db_array) 518 return -EINVAL; 519 520 if ((queue_type != AMDGPU_RING_TYPE_GFX) && 521 (queue_type != AMDGPU_RING_TYPE_COMPUTE) && 522 (queue_type != AMDGPU_RING_TYPE_SDMA)) 523 return -EINVAL; 524 525 /* Clear the doorbell array before detection */ 526 memset(adev->mes.hung_queue_db_array_cpu_addr[xcc_id], AMDGPU_MES_INVALID_DB_OFFSET, 527 adev->mes.hung_queue_db_array_size * sizeof(u32)); 528 input.queue_type = queue_type; 529 input.detect_only = detect_only; 530 input.xcc_id = xcc_id; 531 532 r = adev->mes.funcs->detect_and_reset_hung_queues(&adev->mes, 533 &input); 534 535 if (r && detect_only) { 536 dev_err(adev->dev, "Failed to detect hung queues\n"); 537 return r; 538 } 539 540 *hung_db_num = 0; 541 /* MES passes hung queues' doorbell to driver */ 542 for (i = 0; i < adev->mes.hung_queue_hqd_info_offset; i++) { 543 /* Finding hung queues where db_array[i] is a valid doorbell */ 544 if (db_array[i] != AMDGPU_MES_INVALID_DB_OFFSET) { 545 hung_db_array[i] = db_array[i]; 546 *hung_db_num += 1; 547 } 548 } 549 550 if (r && !(*hung_db_num)) { 551 dev_err(adev->dev, "Failed to detect and reset hung queues\n"); 552 return r; 553 } 554 555 for (i = hqd_info_offset; i < hqd_info_offset + *hung_db_num; i++) 556 hung_db_array[i] = db_array[i]; 557 558 return r; 559 } 560 561 uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg, 562 uint32_t xcc_id) 563 { 564 struct mes_misc_op_input op_input; 565 int r, val = 0; 566 uint32_t addr_offset = 0; 567 uint64_t read_val_gpu_addr; 568 uint32_t *read_val_ptr; 569 570 if (amdgpu_device_wb_get(adev, &addr_offset)) { 571 dev_err(adev->dev, "critical bug! too many mes readers\n"); 572 goto error; 573 } 574 read_val_gpu_addr = adev->wb.gpu_addr + (addr_offset * 4); 575 read_val_ptr = (uint32_t *)&adev->wb.wb[addr_offset]; 576 op_input.xcc_id = xcc_id; 577 op_input.op = MES_MISC_OP_READ_REG; 578 op_input.read_reg.reg_offset = reg; 579 op_input.read_reg.buffer_addr = read_val_gpu_addr; 580 581 if (!adev->mes.funcs->misc_op) { 582 dev_err(adev->dev, "mes rreg is not supported!\n"); 583 goto error; 584 } 585 586 amdgpu_mes_lock(&adev->mes); 587 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 588 amdgpu_mes_unlock(&adev->mes); 589 if (r) 590 dev_err(adev->dev, "failed to read reg (0x%x)\n", reg); 591 else 592 val = *(read_val_ptr); 593 594 error: 595 if (addr_offset) 596 amdgpu_device_wb_free(adev, addr_offset); 597 return val; 598 } 599 600 int amdgpu_mes_wreg(struct amdgpu_device *adev, uint32_t reg, 601 uint32_t val, uint32_t xcc_id) 602 { 603 struct mes_misc_op_input op_input; 604 int r; 605 606 op_input.xcc_id = xcc_id; 607 op_input.op = MES_MISC_OP_WRITE_REG; 608 op_input.write_reg.reg_offset = reg; 609 op_input.write_reg.reg_value = val; 610 611 if (!adev->mes.funcs->misc_op) { 612 dev_err(adev->dev, "mes wreg is not supported!\n"); 613 r = -EINVAL; 614 goto error; 615 } 616 617 amdgpu_mes_lock(&adev->mes); 618 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 619 amdgpu_mes_unlock(&adev->mes); 620 if (r) 621 dev_err(adev->dev, "failed to write reg (0x%x)\n", reg); 622 623 error: 624 return r; 625 } 626 627 int amdgpu_mes_reg_write_reg_wait(struct amdgpu_device *adev, 628 uint32_t reg0, uint32_t reg1, 629 uint32_t ref, uint32_t mask, 630 uint32_t xcc_id) 631 { 632 struct mes_misc_op_input op_input; 633 int r; 634 635 op_input.xcc_id = xcc_id; 636 op_input.op = MES_MISC_OP_WRM_REG_WR_WAIT; 637 op_input.wrm_reg.reg0 = reg0; 638 op_input.wrm_reg.reg1 = reg1; 639 op_input.wrm_reg.ref = ref; 640 op_input.wrm_reg.mask = mask; 641 642 if (!adev->mes.funcs->misc_op) { 643 dev_err(adev->dev, "mes reg_write_reg_wait is not supported!\n"); 644 r = -EINVAL; 645 goto error; 646 } 647 648 amdgpu_mes_lock(&adev->mes); 649 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 650 amdgpu_mes_unlock(&adev->mes); 651 if (r) 652 dev_err(adev->dev, "failed to reg_write_reg_wait\n"); 653 654 error: 655 return r; 656 } 657 658 int amdgpu_mes_hdp_flush(struct amdgpu_device *adev) 659 { 660 uint32_t hdp_flush_req_offset, hdp_flush_done_offset; 661 struct amdgpu_ring *mes_ring; 662 uint32_t ref_and_mask = 0, reg_mem_engine = 0; 663 664 if (!adev->gfx.funcs->get_hdp_flush_mask) { 665 dev_err(adev->dev, "mes hdp flush is not supported.\n"); 666 return -EINVAL; 667 } 668 669 mes_ring = &adev->mes.ring[0]; 670 hdp_flush_req_offset = adev->nbio.funcs->get_hdp_flush_req_offset(adev); 671 hdp_flush_done_offset = adev->nbio.funcs->get_hdp_flush_done_offset(adev); 672 673 adev->gfx.funcs->get_hdp_flush_mask(mes_ring, &ref_and_mask, ®_mem_engine); 674 675 return amdgpu_mes_reg_write_reg_wait(adev, hdp_flush_req_offset, hdp_flush_done_offset, 676 ref_and_mask, ref_and_mask, 0); 677 } 678 679 int amdgpu_mes_set_shader_debugger(struct amdgpu_device *adev, 680 uint64_t process_context_addr, 681 uint32_t spi_gdbg_per_vmid_cntl, 682 const uint32_t *tcp_watch_cntl, 683 uint32_t flags, 684 bool trap_en, 685 uint32_t xcc_id) 686 { 687 struct mes_misc_op_input op_input = {0}; 688 int r; 689 690 if (!adev->mes.funcs->misc_op) { 691 dev_err(adev->dev, 692 "mes set shader debugger is not supported!\n"); 693 return -EINVAL; 694 } 695 696 op_input.xcc_id = xcc_id; 697 op_input.op = MES_MISC_OP_SET_SHADER_DEBUGGER; 698 op_input.set_shader_debugger.process_context_addr = process_context_addr; 699 op_input.set_shader_debugger.flags.u32all = flags; 700 701 /* use amdgpu mes_flush_shader_debugger instead */ 702 if (op_input.set_shader_debugger.flags.process_ctx_flush) 703 return -EINVAL; 704 705 op_input.set_shader_debugger.spi_gdbg_per_vmid_cntl = spi_gdbg_per_vmid_cntl; 706 memcpy(op_input.set_shader_debugger.tcp_watch_cntl, tcp_watch_cntl, 707 sizeof(op_input.set_shader_debugger.tcp_watch_cntl)); 708 709 if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >> 710 AMDGPU_MES_API_VERSION_SHIFT) >= 14) 711 op_input.set_shader_debugger.trap_en = trap_en; 712 713 amdgpu_mes_lock(&adev->mes); 714 715 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 716 if (r) 717 dev_err(adev->dev, "failed to set_shader_debugger\n"); 718 719 amdgpu_mes_unlock(&adev->mes); 720 721 return r; 722 } 723 724 int amdgpu_mes_flush_shader_debugger(struct amdgpu_device *adev, 725 uint64_t process_context_addr, 726 uint32_t xcc_id) 727 { 728 struct mes_misc_op_input op_input = {0}; 729 int r; 730 731 if (!adev->mes.funcs->misc_op) { 732 dev_err(adev->dev, 733 "mes flush shader debugger is not supported!\n"); 734 return -EINVAL; 735 } 736 737 op_input.xcc_id = xcc_id; 738 op_input.op = MES_MISC_OP_SET_SHADER_DEBUGGER; 739 op_input.set_shader_debugger.process_context_addr = process_context_addr; 740 op_input.set_shader_debugger.flags.process_ctx_flush = true; 741 742 amdgpu_mes_lock(&adev->mes); 743 744 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 745 if (r) 746 dev_err(adev->dev, "failed to set_shader_debugger\n"); 747 748 amdgpu_mes_unlock(&adev->mes); 749 750 return r; 751 } 752 753 uint32_t amdgpu_mes_get_aggregated_doorbell_index(struct amdgpu_device *adev, 754 enum amdgpu_mes_priority_level prio) 755 { 756 return adev->mes.aggregated_doorbells[prio]; 757 } 758 759 int amdgpu_mes_init_microcode(struct amdgpu_device *adev, int pipe) 760 { 761 const struct mes_firmware_header_v1_0 *mes_hdr; 762 struct amdgpu_firmware_info *info; 763 char ucode_prefix[30]; 764 char fw_name[50]; 765 bool need_retry = false; 766 u32 *ucode_ptr; 767 int r; 768 769 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, 770 sizeof(ucode_prefix)); 771 if (adev->enable_uni_mes) { 772 snprintf(fw_name, sizeof(fw_name), 773 "amdgpu/%s_uni_mes.bin", ucode_prefix); 774 } else if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) && 775 amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)) { 776 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes%s.bin", 777 ucode_prefix, 778 pipe == AMDGPU_MES_SCHED_PIPE ? "_2" : "1"); 779 need_retry = true; 780 } else { 781 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes%s.bin", 782 ucode_prefix, 783 pipe == AMDGPU_MES_SCHED_PIPE ? "" : "1"); 784 } 785 786 r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], AMDGPU_UCODE_REQUIRED, 787 "%s", fw_name); 788 if (r && need_retry && pipe == AMDGPU_MES_SCHED_PIPE) { 789 dev_info(adev->dev, "try to fall back to %s_mes.bin\n", ucode_prefix); 790 r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], 791 AMDGPU_UCODE_REQUIRED, 792 "amdgpu/%s_mes.bin", ucode_prefix); 793 } 794 795 if (r) 796 goto out; 797 798 mes_hdr = (const struct mes_firmware_header_v1_0 *) 799 adev->mes.fw[pipe]->data; 800 adev->mes.uc_start_addr[pipe] = 801 le32_to_cpu(mes_hdr->mes_uc_start_addr_lo) | 802 ((uint64_t)(le32_to_cpu(mes_hdr->mes_uc_start_addr_hi)) << 32); 803 adev->mes.data_start_addr[pipe] = 804 le32_to_cpu(mes_hdr->mes_data_start_addr_lo) | 805 ((uint64_t)(le32_to_cpu(mes_hdr->mes_data_start_addr_hi)) << 32); 806 ucode_ptr = (u32 *)(adev->mes.fw[pipe]->data + 807 sizeof(union amdgpu_firmware_header)); 808 adev->mes.fw_version[pipe] = 809 le32_to_cpu(ucode_ptr[24]) & AMDGPU_MES_VERSION_MASK; 810 811 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 812 int ucode, ucode_data; 813 814 if (pipe == AMDGPU_MES_SCHED_PIPE) { 815 ucode = AMDGPU_UCODE_ID_CP_MES; 816 ucode_data = AMDGPU_UCODE_ID_CP_MES_DATA; 817 } else { 818 ucode = AMDGPU_UCODE_ID_CP_MES1; 819 ucode_data = AMDGPU_UCODE_ID_CP_MES1_DATA; 820 } 821 822 info = &adev->firmware.ucode[ucode]; 823 info->ucode_id = ucode; 824 info->fw = adev->mes.fw[pipe]; 825 adev->firmware.fw_size += 826 ALIGN(le32_to_cpu(mes_hdr->mes_ucode_size_bytes), 827 PAGE_SIZE); 828 829 info = &adev->firmware.ucode[ucode_data]; 830 info->ucode_id = ucode_data; 831 info->fw = adev->mes.fw[pipe]; 832 adev->firmware.fw_size += 833 ALIGN(le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes), 834 PAGE_SIZE); 835 } 836 837 return 0; 838 out: 839 amdgpu_ucode_release(&adev->mes.fw[pipe]); 840 return r; 841 } 842 843 void amdgpu_mes_validate_fw_version(struct amdgpu_device *adev) 844 { 845 u32 fw_from_ucode = adev->mes.fw_version[AMDGPU_MES_SCHED_PIPE]; 846 u32 fw_from_reg = adev->mes.sched_version & AMDGPU_MES_VERSION_MASK; 847 848 if (fw_from_ucode != fw_from_reg) 849 dev_info(adev->dev, 850 "MES firmware reports incorrect version in ucode binary (0x%x vs 0x%x)\n", 851 fw_from_ucode, fw_from_reg); 852 } 853 854 855 bool amdgpu_mes_suspend_resume_all_supported(struct amdgpu_device *adev) 856 { 857 uint32_t mes_rev = adev->mes.sched_version & AMDGPU_MES_VERSION_MASK; 858 859 return ((amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) && 860 amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0) && 861 mes_rev >= 0x63) || 862 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 0, 0)); 863 } 864 865 bool amdgpu_mes_queue_reset_by_mes_supported(struct amdgpu_device *adev) 866 { 867 u32 ip_maj = IP_VERSION_MAJ(amdgpu_ip_version(adev, GC_HWIP, 0)); 868 u32 ip_min = IP_VERSION_MIN(amdgpu_ip_version(adev, GC_HWIP, 0)); 869 u32 mes_sched = adev->mes.sched_version & AMDGPU_MES_VERSION_MASK; 870 871 return (ip_maj == 11 && mes_sched >= 0x8c) || 872 ((ip_maj == 12 && ip_min == 0) && mes_sched >= 0x8d) || 873 ((ip_maj == 12 && ip_min == 1) && mes_sched >= 0x73); 874 } 875 876 /* Fix me -- node_id is used to identify the correct MES instances in the future */ 877 static int amdgpu_mes_set_enforce_isolation(struct amdgpu_device *adev, 878 uint32_t node_id, bool enable) 879 { 880 struct mes_misc_op_input op_input = {0}; 881 int r; 882 883 op_input.op = MES_MISC_OP_CHANGE_CONFIG; 884 op_input.change_config.option.limit_single_process = enable ? 1 : 0; 885 886 if (!adev->mes.funcs->misc_op) { 887 dev_err(adev->dev, "mes change config is not supported!\n"); 888 r = -EINVAL; 889 goto error; 890 } 891 892 amdgpu_mes_lock(&adev->mes); 893 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 894 amdgpu_mes_unlock(&adev->mes); 895 if (r) 896 dev_err(adev->dev, "failed to change_config.\n"); 897 898 error: 899 return r; 900 } 901 902 int amdgpu_mes_update_enforce_isolation(struct amdgpu_device *adev) 903 { 904 int i, r = 0; 905 906 if (adev->enable_mes && adev->gfx.enable_cleaner_shader) { 907 mutex_lock(&adev->enforce_isolation_mutex); 908 for (i = 0; i < (adev->xcp_mgr ? adev->xcp_mgr->num_xcps : 1); i++) { 909 if (adev->enforce_isolation[i] == AMDGPU_ENFORCE_ISOLATION_ENABLE) 910 r |= amdgpu_mes_set_enforce_isolation(adev, i, true); 911 else 912 r |= amdgpu_mes_set_enforce_isolation(adev, i, false); 913 } 914 mutex_unlock(&adev->enforce_isolation_mutex); 915 } 916 return r; 917 } 918 919 #if defined(CONFIG_DEBUG_FS) 920 921 static int amdgpu_debugfs_mes_event_log_show(struct seq_file *m, void *unused) 922 { 923 struct amdgpu_device *adev = m->private; 924 uint32_t *mem = (uint32_t *)(adev->mes.event_log_cpu_addr); 925 926 seq_hex_dump(m, "", DUMP_PREFIX_OFFSET, 32, 4, 927 mem, adev->mes.event_log_size, false); 928 929 return 0; 930 } 931 932 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_mes_event_log); 933 934 #endif 935 936 void amdgpu_debugfs_mes_event_log_init(struct amdgpu_device *adev) 937 { 938 939 #if defined(CONFIG_DEBUG_FS) 940 struct drm_minor *minor = adev_to_drm(adev)->primary; 941 struct dentry *root = minor->debugfs_root; 942 if (adev->enable_mes && amdgpu_mes_log_enable) 943 debugfs_create_file("amdgpu_mes_event_log", 0444, root, 944 adev, &amdgpu_debugfs_mes_event_log_fops); 945 946 #endif 947 } 948