1 /* 2 * Copyright 2019 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/firmware.h> 25 #include <drm/drm_exec.h> 26 27 #include "amdgpu_mes.h" 28 #include "amdgpu.h" 29 #include "soc15_common.h" 30 #include "amdgpu_mes_ctx.h" 31 32 #define AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS 1024 33 #define AMDGPU_ONE_DOORBELL_SIZE 8 34 35 int amdgpu_mes_doorbell_process_slice(struct amdgpu_device *adev) 36 { 37 return roundup(AMDGPU_ONE_DOORBELL_SIZE * 38 AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS, 39 PAGE_SIZE); 40 } 41 42 static int amdgpu_mes_kernel_doorbell_get(struct amdgpu_device *adev, 43 int ip_type, uint64_t *doorbell_index) 44 { 45 unsigned int offset, found; 46 struct amdgpu_mes *mes = &adev->mes; 47 48 if (ip_type == AMDGPU_RING_TYPE_SDMA) 49 offset = adev->doorbell_index.sdma_engine[0]; 50 else 51 offset = 0; 52 53 found = find_next_zero_bit(mes->doorbell_bitmap, mes->num_mes_dbs, offset); 54 if (found >= mes->num_mes_dbs) { 55 DRM_WARN("No doorbell available\n"); 56 return -ENOSPC; 57 } 58 59 set_bit(found, mes->doorbell_bitmap); 60 61 /* Get the absolute doorbell index on BAR */ 62 *doorbell_index = mes->db_start_dw_offset + found * 2; 63 return 0; 64 } 65 66 static void amdgpu_mes_kernel_doorbell_free(struct amdgpu_device *adev, 67 uint32_t doorbell_index) 68 { 69 unsigned int old, rel_index; 70 struct amdgpu_mes *mes = &adev->mes; 71 72 /* Find the relative index of the doorbell in this object */ 73 rel_index = (doorbell_index - mes->db_start_dw_offset) / 2; 74 old = test_and_clear_bit(rel_index, mes->doorbell_bitmap); 75 WARN_ON(!old); 76 } 77 78 static int amdgpu_mes_doorbell_init(struct amdgpu_device *adev) 79 { 80 int i; 81 struct amdgpu_mes *mes = &adev->mes; 82 83 /* Bitmap for dynamic allocation of kernel doorbells */ 84 mes->doorbell_bitmap = bitmap_zalloc(PAGE_SIZE / sizeof(u32), GFP_KERNEL); 85 if (!mes->doorbell_bitmap) { 86 DRM_ERROR("Failed to allocate MES doorbell bitmap\n"); 87 return -ENOMEM; 88 } 89 90 mes->num_mes_dbs = PAGE_SIZE / AMDGPU_ONE_DOORBELL_SIZE; 91 for (i = 0; i < AMDGPU_MES_PRIORITY_NUM_LEVELS; i++) { 92 adev->mes.aggregated_doorbells[i] = mes->db_start_dw_offset + i * 2; 93 set_bit(i, mes->doorbell_bitmap); 94 } 95 96 return 0; 97 } 98 99 static int amdgpu_mes_event_log_init(struct amdgpu_device *adev) 100 { 101 int r; 102 103 if (!amdgpu_mes_log_enable) 104 return 0; 105 106 r = amdgpu_bo_create_kernel(adev, adev->mes.event_log_size, PAGE_SIZE, 107 AMDGPU_GEM_DOMAIN_GTT, 108 &adev->mes.event_log_gpu_obj, 109 &adev->mes.event_log_gpu_addr, 110 &adev->mes.event_log_cpu_addr); 111 if (r) { 112 dev_warn(adev->dev, "failed to create MES event log buffer (%d)", r); 113 return r; 114 } 115 116 memset(adev->mes.event_log_cpu_addr, 0, adev->mes.event_log_size); 117 118 return 0; 119 120 } 121 122 static void amdgpu_mes_doorbell_free(struct amdgpu_device *adev) 123 { 124 bitmap_free(adev->mes.doorbell_bitmap); 125 } 126 127 int amdgpu_mes_init(struct amdgpu_device *adev) 128 { 129 int i, r; 130 131 adev->mes.adev = adev; 132 133 idr_init(&adev->mes.pasid_idr); 134 idr_init(&adev->mes.gang_id_idr); 135 idr_init(&adev->mes.queue_id_idr); 136 ida_init(&adev->mes.doorbell_ida); 137 spin_lock_init(&adev->mes.queue_id_lock); 138 mutex_init(&adev->mes.mutex_hidden); 139 140 for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) 141 spin_lock_init(&adev->mes.ring_lock[i]); 142 143 adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK; 144 adev->mes.vmid_mask_mmhub = 0xffffff00; 145 adev->mes.vmid_mask_gfxhub = 0xffffff00; 146 147 for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) { 148 /* use only 1st MEC pipes */ 149 if (i >= adev->gfx.mec.num_pipe_per_mec) 150 continue; 151 adev->mes.compute_hqd_mask[i] = 0xc; 152 } 153 154 for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++) 155 adev->mes.gfx_hqd_mask[i] = i ? 0 : 0xfffffffe; 156 157 for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) { 158 if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) < 159 IP_VERSION(6, 0, 0)) 160 adev->mes.sdma_hqd_mask[i] = i ? 0 : 0x3fc; 161 /* zero sdma_hqd_mask for non-existent engine */ 162 else if (adev->sdma.num_instances == 1) 163 adev->mes.sdma_hqd_mask[i] = i ? 0 : 0xfc; 164 else 165 adev->mes.sdma_hqd_mask[i] = 0xfc; 166 } 167 168 for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) { 169 r = amdgpu_device_wb_get(adev, &adev->mes.sch_ctx_offs[i]); 170 if (r) { 171 dev_err(adev->dev, 172 "(%d) ring trail_fence_offs wb alloc failed\n", 173 r); 174 goto error; 175 } 176 adev->mes.sch_ctx_gpu_addr[i] = 177 adev->wb.gpu_addr + (adev->mes.sch_ctx_offs[i] * 4); 178 adev->mes.sch_ctx_ptr[i] = 179 (uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs[i]]; 180 181 r = amdgpu_device_wb_get(adev, 182 &adev->mes.query_status_fence_offs[i]); 183 if (r) { 184 dev_err(adev->dev, 185 "(%d) query_status_fence_offs wb alloc failed\n", 186 r); 187 goto error; 188 } 189 adev->mes.query_status_fence_gpu_addr[i] = adev->wb.gpu_addr + 190 (adev->mes.query_status_fence_offs[i] * 4); 191 adev->mes.query_status_fence_ptr[i] = 192 (uint64_t *)&adev->wb.wb[adev->mes.query_status_fence_offs[i]]; 193 } 194 195 r = amdgpu_device_wb_get(adev, &adev->mes.read_val_offs); 196 if (r) { 197 dev_err(adev->dev, 198 "(%d) read_val_offs alloc failed\n", r); 199 goto error; 200 } 201 adev->mes.read_val_gpu_addr = 202 adev->wb.gpu_addr + (adev->mes.read_val_offs * 4); 203 adev->mes.read_val_ptr = 204 (uint32_t *)&adev->wb.wb[adev->mes.read_val_offs]; 205 206 r = amdgpu_mes_doorbell_init(adev); 207 if (r) 208 goto error; 209 210 r = amdgpu_mes_event_log_init(adev); 211 if (r) 212 goto error_doorbell; 213 214 return 0; 215 216 error_doorbell: 217 amdgpu_mes_doorbell_free(adev); 218 error: 219 for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) { 220 if (adev->mes.sch_ctx_ptr[i]) 221 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]); 222 if (adev->mes.query_status_fence_ptr[i]) 223 amdgpu_device_wb_free(adev, 224 adev->mes.query_status_fence_offs[i]); 225 } 226 if (adev->mes.read_val_ptr) 227 amdgpu_device_wb_free(adev, adev->mes.read_val_offs); 228 229 idr_destroy(&adev->mes.pasid_idr); 230 idr_destroy(&adev->mes.gang_id_idr); 231 idr_destroy(&adev->mes.queue_id_idr); 232 ida_destroy(&adev->mes.doorbell_ida); 233 mutex_destroy(&adev->mes.mutex_hidden); 234 return r; 235 } 236 237 void amdgpu_mes_fini(struct amdgpu_device *adev) 238 { 239 int i; 240 241 amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj, 242 &adev->mes.event_log_gpu_addr, 243 &adev->mes.event_log_cpu_addr); 244 245 for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) { 246 if (adev->mes.sch_ctx_ptr[i]) 247 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]); 248 if (adev->mes.query_status_fence_ptr[i]) 249 amdgpu_device_wb_free(adev, 250 adev->mes.query_status_fence_offs[i]); 251 } 252 if (adev->mes.read_val_ptr) 253 amdgpu_device_wb_free(adev, adev->mes.read_val_offs); 254 255 amdgpu_mes_doorbell_free(adev); 256 257 idr_destroy(&adev->mes.pasid_idr); 258 idr_destroy(&adev->mes.gang_id_idr); 259 idr_destroy(&adev->mes.queue_id_idr); 260 ida_destroy(&adev->mes.doorbell_ida); 261 mutex_destroy(&adev->mes.mutex_hidden); 262 } 263 264 static void amdgpu_mes_queue_free_mqd(struct amdgpu_mes_queue *q) 265 { 266 amdgpu_bo_free_kernel(&q->mqd_obj, 267 &q->mqd_gpu_addr, 268 &q->mqd_cpu_ptr); 269 } 270 271 int amdgpu_mes_create_process(struct amdgpu_device *adev, int pasid, 272 struct amdgpu_vm *vm) 273 { 274 struct amdgpu_mes_process *process; 275 int r; 276 277 /* allocate the mes process buffer */ 278 process = kzalloc(sizeof(struct amdgpu_mes_process), GFP_KERNEL); 279 if (!process) { 280 DRM_ERROR("no more memory to create mes process\n"); 281 return -ENOMEM; 282 } 283 284 /* allocate the process context bo and map it */ 285 r = amdgpu_bo_create_kernel(adev, AMDGPU_MES_PROC_CTX_SIZE, PAGE_SIZE, 286 AMDGPU_GEM_DOMAIN_GTT, 287 &process->proc_ctx_bo, 288 &process->proc_ctx_gpu_addr, 289 &process->proc_ctx_cpu_ptr); 290 if (r) { 291 DRM_ERROR("failed to allocate process context bo\n"); 292 goto clean_up_memory; 293 } 294 memset(process->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE); 295 296 /* 297 * Avoid taking any other locks under MES lock to avoid circular 298 * lock dependencies. 299 */ 300 amdgpu_mes_lock(&adev->mes); 301 302 /* add the mes process to idr list */ 303 r = idr_alloc(&adev->mes.pasid_idr, process, pasid, pasid + 1, 304 GFP_KERNEL); 305 if (r < 0) { 306 DRM_ERROR("failed to lock pasid=%d\n", pasid); 307 goto clean_up_ctx; 308 } 309 310 INIT_LIST_HEAD(&process->gang_list); 311 process->vm = vm; 312 process->pasid = pasid; 313 process->process_quantum = adev->mes.default_process_quantum; 314 process->pd_gpu_addr = amdgpu_bo_gpu_offset(vm->root.bo); 315 316 amdgpu_mes_unlock(&adev->mes); 317 return 0; 318 319 clean_up_ctx: 320 amdgpu_mes_unlock(&adev->mes); 321 amdgpu_bo_free_kernel(&process->proc_ctx_bo, 322 &process->proc_ctx_gpu_addr, 323 &process->proc_ctx_cpu_ptr); 324 clean_up_memory: 325 kfree(process); 326 return r; 327 } 328 329 void amdgpu_mes_destroy_process(struct amdgpu_device *adev, int pasid) 330 { 331 struct amdgpu_mes_process *process; 332 struct amdgpu_mes_gang *gang, *tmp1; 333 struct amdgpu_mes_queue *queue, *tmp2; 334 struct mes_remove_queue_input queue_input; 335 unsigned long flags; 336 int r; 337 338 /* 339 * Avoid taking any other locks under MES lock to avoid circular 340 * lock dependencies. 341 */ 342 amdgpu_mes_lock(&adev->mes); 343 344 process = idr_find(&adev->mes.pasid_idr, pasid); 345 if (!process) { 346 DRM_WARN("pasid %d doesn't exist\n", pasid); 347 amdgpu_mes_unlock(&adev->mes); 348 return; 349 } 350 351 /* Remove all queues from hardware */ 352 list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) { 353 list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) { 354 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); 355 idr_remove(&adev->mes.queue_id_idr, queue->queue_id); 356 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); 357 358 queue_input.doorbell_offset = queue->doorbell_off; 359 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr; 360 361 r = adev->mes.funcs->remove_hw_queue(&adev->mes, 362 &queue_input); 363 if (r) 364 DRM_WARN("failed to remove hardware queue\n"); 365 } 366 367 idr_remove(&adev->mes.gang_id_idr, gang->gang_id); 368 } 369 370 idr_remove(&adev->mes.pasid_idr, pasid); 371 amdgpu_mes_unlock(&adev->mes); 372 373 /* free all memory allocated by the process */ 374 list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) { 375 /* free all queues in the gang */ 376 list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) { 377 amdgpu_mes_queue_free_mqd(queue); 378 list_del(&queue->list); 379 kfree(queue); 380 } 381 amdgpu_bo_free_kernel(&gang->gang_ctx_bo, 382 &gang->gang_ctx_gpu_addr, 383 &gang->gang_ctx_cpu_ptr); 384 list_del(&gang->list); 385 kfree(gang); 386 387 } 388 amdgpu_bo_free_kernel(&process->proc_ctx_bo, 389 &process->proc_ctx_gpu_addr, 390 &process->proc_ctx_cpu_ptr); 391 kfree(process); 392 } 393 394 int amdgpu_mes_add_gang(struct amdgpu_device *adev, int pasid, 395 struct amdgpu_mes_gang_properties *gprops, 396 int *gang_id) 397 { 398 struct amdgpu_mes_process *process; 399 struct amdgpu_mes_gang *gang; 400 int r; 401 402 /* allocate the mes gang buffer */ 403 gang = kzalloc(sizeof(struct amdgpu_mes_gang), GFP_KERNEL); 404 if (!gang) { 405 return -ENOMEM; 406 } 407 408 /* allocate the gang context bo and map it to cpu space */ 409 r = amdgpu_bo_create_kernel(adev, AMDGPU_MES_GANG_CTX_SIZE, PAGE_SIZE, 410 AMDGPU_GEM_DOMAIN_GTT, 411 &gang->gang_ctx_bo, 412 &gang->gang_ctx_gpu_addr, 413 &gang->gang_ctx_cpu_ptr); 414 if (r) { 415 DRM_ERROR("failed to allocate process context bo\n"); 416 goto clean_up_mem; 417 } 418 memset(gang->gang_ctx_cpu_ptr, 0, AMDGPU_MES_GANG_CTX_SIZE); 419 420 /* 421 * Avoid taking any other locks under MES lock to avoid circular 422 * lock dependencies. 423 */ 424 amdgpu_mes_lock(&adev->mes); 425 426 process = idr_find(&adev->mes.pasid_idr, pasid); 427 if (!process) { 428 DRM_ERROR("pasid %d doesn't exist\n", pasid); 429 r = -EINVAL; 430 goto clean_up_ctx; 431 } 432 433 /* add the mes gang to idr list */ 434 r = idr_alloc(&adev->mes.gang_id_idr, gang, 1, 0, 435 GFP_KERNEL); 436 if (r < 0) { 437 DRM_ERROR("failed to allocate idr for gang\n"); 438 goto clean_up_ctx; 439 } 440 441 gang->gang_id = r; 442 *gang_id = r; 443 444 INIT_LIST_HEAD(&gang->queue_list); 445 gang->process = process; 446 gang->priority = gprops->priority; 447 gang->gang_quantum = gprops->gang_quantum ? 448 gprops->gang_quantum : adev->mes.default_gang_quantum; 449 gang->global_priority_level = gprops->global_priority_level; 450 gang->inprocess_gang_priority = gprops->inprocess_gang_priority; 451 list_add_tail(&gang->list, &process->gang_list); 452 453 amdgpu_mes_unlock(&adev->mes); 454 return 0; 455 456 clean_up_ctx: 457 amdgpu_mes_unlock(&adev->mes); 458 amdgpu_bo_free_kernel(&gang->gang_ctx_bo, 459 &gang->gang_ctx_gpu_addr, 460 &gang->gang_ctx_cpu_ptr); 461 clean_up_mem: 462 kfree(gang); 463 return r; 464 } 465 466 int amdgpu_mes_remove_gang(struct amdgpu_device *adev, int gang_id) 467 { 468 struct amdgpu_mes_gang *gang; 469 470 /* 471 * Avoid taking any other locks under MES lock to avoid circular 472 * lock dependencies. 473 */ 474 amdgpu_mes_lock(&adev->mes); 475 476 gang = idr_find(&adev->mes.gang_id_idr, gang_id); 477 if (!gang) { 478 DRM_ERROR("gang id %d doesn't exist\n", gang_id); 479 amdgpu_mes_unlock(&adev->mes); 480 return -EINVAL; 481 } 482 483 if (!list_empty(&gang->queue_list)) { 484 DRM_ERROR("queue list is not empty\n"); 485 amdgpu_mes_unlock(&adev->mes); 486 return -EBUSY; 487 } 488 489 idr_remove(&adev->mes.gang_id_idr, gang->gang_id); 490 list_del(&gang->list); 491 amdgpu_mes_unlock(&adev->mes); 492 493 amdgpu_bo_free_kernel(&gang->gang_ctx_bo, 494 &gang->gang_ctx_gpu_addr, 495 &gang->gang_ctx_cpu_ptr); 496 497 kfree(gang); 498 499 return 0; 500 } 501 502 int amdgpu_mes_suspend(struct amdgpu_device *adev) 503 { 504 struct idr *idp; 505 struct amdgpu_mes_process *process; 506 struct amdgpu_mes_gang *gang; 507 struct mes_suspend_gang_input input; 508 int r, pasid; 509 510 /* 511 * Avoid taking any other locks under MES lock to avoid circular 512 * lock dependencies. 513 */ 514 amdgpu_mes_lock(&adev->mes); 515 516 idp = &adev->mes.pasid_idr; 517 518 idr_for_each_entry(idp, process, pasid) { 519 list_for_each_entry(gang, &process->gang_list, list) { 520 r = adev->mes.funcs->suspend_gang(&adev->mes, &input); 521 if (r) 522 DRM_ERROR("failed to suspend pasid %d gangid %d", 523 pasid, gang->gang_id); 524 } 525 } 526 527 amdgpu_mes_unlock(&adev->mes); 528 return 0; 529 } 530 531 int amdgpu_mes_resume(struct amdgpu_device *adev) 532 { 533 struct idr *idp; 534 struct amdgpu_mes_process *process; 535 struct amdgpu_mes_gang *gang; 536 struct mes_resume_gang_input input; 537 int r, pasid; 538 539 /* 540 * Avoid taking any other locks under MES lock to avoid circular 541 * lock dependencies. 542 */ 543 amdgpu_mes_lock(&adev->mes); 544 545 idp = &adev->mes.pasid_idr; 546 547 idr_for_each_entry(idp, process, pasid) { 548 list_for_each_entry(gang, &process->gang_list, list) { 549 r = adev->mes.funcs->resume_gang(&adev->mes, &input); 550 if (r) 551 DRM_ERROR("failed to resume pasid %d gangid %d", 552 pasid, gang->gang_id); 553 } 554 } 555 556 amdgpu_mes_unlock(&adev->mes); 557 return 0; 558 } 559 560 static int amdgpu_mes_queue_alloc_mqd(struct amdgpu_device *adev, 561 struct amdgpu_mes_queue *q, 562 struct amdgpu_mes_queue_properties *p) 563 { 564 struct amdgpu_mqd *mqd_mgr = &adev->mqds[p->queue_type]; 565 u32 mqd_size = mqd_mgr->mqd_size; 566 int r; 567 568 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, 569 AMDGPU_GEM_DOMAIN_GTT, 570 &q->mqd_obj, 571 &q->mqd_gpu_addr, &q->mqd_cpu_ptr); 572 if (r) { 573 dev_warn(adev->dev, "failed to create queue mqd bo (%d)", r); 574 return r; 575 } 576 memset(q->mqd_cpu_ptr, 0, mqd_size); 577 578 r = amdgpu_bo_reserve(q->mqd_obj, false); 579 if (unlikely(r != 0)) 580 goto clean_up; 581 582 return 0; 583 584 clean_up: 585 amdgpu_bo_free_kernel(&q->mqd_obj, 586 &q->mqd_gpu_addr, 587 &q->mqd_cpu_ptr); 588 return r; 589 } 590 591 static void amdgpu_mes_queue_init_mqd(struct amdgpu_device *adev, 592 struct amdgpu_mes_queue *q, 593 struct amdgpu_mes_queue_properties *p) 594 { 595 struct amdgpu_mqd *mqd_mgr = &adev->mqds[p->queue_type]; 596 struct amdgpu_mqd_prop mqd_prop = {0}; 597 598 mqd_prop.mqd_gpu_addr = q->mqd_gpu_addr; 599 mqd_prop.hqd_base_gpu_addr = p->hqd_base_gpu_addr; 600 mqd_prop.rptr_gpu_addr = p->rptr_gpu_addr; 601 mqd_prop.wptr_gpu_addr = p->wptr_gpu_addr; 602 mqd_prop.queue_size = p->queue_size; 603 mqd_prop.use_doorbell = true; 604 mqd_prop.doorbell_index = p->doorbell_off; 605 mqd_prop.eop_gpu_addr = p->eop_gpu_addr; 606 mqd_prop.hqd_pipe_priority = p->hqd_pipe_priority; 607 mqd_prop.hqd_queue_priority = p->hqd_queue_priority; 608 mqd_prop.hqd_active = false; 609 610 if (p->queue_type == AMDGPU_RING_TYPE_GFX || 611 p->queue_type == AMDGPU_RING_TYPE_COMPUTE) { 612 mutex_lock(&adev->srbm_mutex); 613 amdgpu_gfx_select_me_pipe_q(adev, p->ring->me, p->ring->pipe, 0, 0, 0); 614 } 615 616 mqd_mgr->init_mqd(adev, q->mqd_cpu_ptr, &mqd_prop); 617 618 if (p->queue_type == AMDGPU_RING_TYPE_GFX || 619 p->queue_type == AMDGPU_RING_TYPE_COMPUTE) { 620 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); 621 mutex_unlock(&adev->srbm_mutex); 622 } 623 624 amdgpu_bo_unreserve(q->mqd_obj); 625 } 626 627 int amdgpu_mes_add_hw_queue(struct amdgpu_device *adev, int gang_id, 628 struct amdgpu_mes_queue_properties *qprops, 629 int *queue_id) 630 { 631 struct amdgpu_mes_queue *queue; 632 struct amdgpu_mes_gang *gang; 633 struct mes_add_queue_input queue_input; 634 unsigned long flags; 635 int r; 636 637 memset(&queue_input, 0, sizeof(struct mes_add_queue_input)); 638 639 /* allocate the mes queue buffer */ 640 queue = kzalloc(sizeof(struct amdgpu_mes_queue), GFP_KERNEL); 641 if (!queue) { 642 DRM_ERROR("Failed to allocate memory for queue\n"); 643 return -ENOMEM; 644 } 645 646 /* Allocate the queue mqd */ 647 r = amdgpu_mes_queue_alloc_mqd(adev, queue, qprops); 648 if (r) 649 goto clean_up_memory; 650 651 /* 652 * Avoid taking any other locks under MES lock to avoid circular 653 * lock dependencies. 654 */ 655 amdgpu_mes_lock(&adev->mes); 656 657 gang = idr_find(&adev->mes.gang_id_idr, gang_id); 658 if (!gang) { 659 DRM_ERROR("gang id %d doesn't exist\n", gang_id); 660 r = -EINVAL; 661 goto clean_up_mqd; 662 } 663 664 /* add the mes gang to idr list */ 665 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); 666 r = idr_alloc(&adev->mes.queue_id_idr, queue, 1, 0, 667 GFP_ATOMIC); 668 if (r < 0) { 669 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); 670 goto clean_up_mqd; 671 } 672 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); 673 *queue_id = queue->queue_id = r; 674 675 /* allocate a doorbell index for the queue */ 676 r = amdgpu_mes_kernel_doorbell_get(adev, 677 qprops->queue_type, 678 &qprops->doorbell_off); 679 if (r) 680 goto clean_up_queue_id; 681 682 /* initialize the queue mqd */ 683 amdgpu_mes_queue_init_mqd(adev, queue, qprops); 684 685 /* add hw queue to mes */ 686 queue_input.process_id = gang->process->pasid; 687 688 queue_input.page_table_base_addr = 689 adev->vm_manager.vram_base_offset + gang->process->pd_gpu_addr - 690 adev->gmc.vram_start; 691 692 queue_input.process_va_start = 0; 693 queue_input.process_va_end = 694 (adev->vm_manager.max_pfn - 1) << AMDGPU_GPU_PAGE_SHIFT; 695 queue_input.process_quantum = gang->process->process_quantum; 696 queue_input.process_context_addr = gang->process->proc_ctx_gpu_addr; 697 queue_input.gang_quantum = gang->gang_quantum; 698 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr; 699 queue_input.inprocess_gang_priority = gang->inprocess_gang_priority; 700 queue_input.gang_global_priority_level = gang->global_priority_level; 701 queue_input.doorbell_offset = qprops->doorbell_off; 702 queue_input.mqd_addr = queue->mqd_gpu_addr; 703 queue_input.wptr_addr = qprops->wptr_gpu_addr; 704 queue_input.wptr_mc_addr = qprops->wptr_mc_addr; 705 queue_input.queue_type = qprops->queue_type; 706 queue_input.paging = qprops->paging; 707 queue_input.is_kfd_process = 0; 708 709 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input); 710 if (r) { 711 DRM_ERROR("failed to add hardware queue to MES, doorbell=0x%llx\n", 712 qprops->doorbell_off); 713 goto clean_up_doorbell; 714 } 715 716 DRM_DEBUG("MES hw queue was added, pasid=%d, gang id=%d, " 717 "queue type=%d, doorbell=0x%llx\n", 718 gang->process->pasid, gang_id, qprops->queue_type, 719 qprops->doorbell_off); 720 721 queue->ring = qprops->ring; 722 queue->doorbell_off = qprops->doorbell_off; 723 queue->wptr_gpu_addr = qprops->wptr_gpu_addr; 724 queue->queue_type = qprops->queue_type; 725 queue->paging = qprops->paging; 726 queue->gang = gang; 727 queue->ring->mqd_ptr = queue->mqd_cpu_ptr; 728 list_add_tail(&queue->list, &gang->queue_list); 729 730 amdgpu_mes_unlock(&adev->mes); 731 return 0; 732 733 clean_up_doorbell: 734 amdgpu_mes_kernel_doorbell_free(adev, qprops->doorbell_off); 735 clean_up_queue_id: 736 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); 737 idr_remove(&adev->mes.queue_id_idr, queue->queue_id); 738 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); 739 clean_up_mqd: 740 amdgpu_mes_unlock(&adev->mes); 741 amdgpu_mes_queue_free_mqd(queue); 742 clean_up_memory: 743 kfree(queue); 744 return r; 745 } 746 747 int amdgpu_mes_remove_hw_queue(struct amdgpu_device *adev, int queue_id) 748 { 749 unsigned long flags; 750 struct amdgpu_mes_queue *queue; 751 struct amdgpu_mes_gang *gang; 752 struct mes_remove_queue_input queue_input; 753 int r; 754 755 /* 756 * Avoid taking any other locks under MES lock to avoid circular 757 * lock dependencies. 758 */ 759 amdgpu_mes_lock(&adev->mes); 760 761 /* remove the mes gang from idr list */ 762 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); 763 764 queue = idr_find(&adev->mes.queue_id_idr, queue_id); 765 if (!queue) { 766 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); 767 amdgpu_mes_unlock(&adev->mes); 768 DRM_ERROR("queue id %d doesn't exist\n", queue_id); 769 return -EINVAL; 770 } 771 772 idr_remove(&adev->mes.queue_id_idr, queue_id); 773 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); 774 775 DRM_DEBUG("try to remove queue, doorbell off = 0x%llx\n", 776 queue->doorbell_off); 777 778 gang = queue->gang; 779 queue_input.doorbell_offset = queue->doorbell_off; 780 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr; 781 782 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); 783 if (r) 784 DRM_ERROR("failed to remove hardware queue, queue id = %d\n", 785 queue_id); 786 787 list_del(&queue->list); 788 amdgpu_mes_kernel_doorbell_free(adev, queue->doorbell_off); 789 amdgpu_mes_unlock(&adev->mes); 790 791 amdgpu_mes_queue_free_mqd(queue); 792 kfree(queue); 793 return 0; 794 } 795 796 int amdgpu_mes_map_legacy_queue(struct amdgpu_device *adev, 797 struct amdgpu_ring *ring) 798 { 799 struct mes_map_legacy_queue_input queue_input; 800 int r; 801 802 memset(&queue_input, 0, sizeof(queue_input)); 803 804 queue_input.queue_type = ring->funcs->type; 805 queue_input.doorbell_offset = ring->doorbell_index; 806 queue_input.pipe_id = ring->pipe; 807 queue_input.queue_id = ring->queue; 808 queue_input.mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 809 queue_input.wptr_addr = ring->wptr_gpu_addr; 810 811 r = adev->mes.funcs->map_legacy_queue(&adev->mes, &queue_input); 812 if (r) 813 DRM_ERROR("failed to map legacy queue\n"); 814 815 return r; 816 } 817 818 int amdgpu_mes_unmap_legacy_queue(struct amdgpu_device *adev, 819 struct amdgpu_ring *ring, 820 enum amdgpu_unmap_queues_action action, 821 u64 gpu_addr, u64 seq) 822 { 823 struct mes_unmap_legacy_queue_input queue_input; 824 int r; 825 826 queue_input.action = action; 827 queue_input.queue_type = ring->funcs->type; 828 queue_input.doorbell_offset = ring->doorbell_index; 829 queue_input.pipe_id = ring->pipe; 830 queue_input.queue_id = ring->queue; 831 queue_input.trail_fence_addr = gpu_addr; 832 queue_input.trail_fence_data = seq; 833 834 r = adev->mes.funcs->unmap_legacy_queue(&adev->mes, &queue_input); 835 if (r) 836 DRM_ERROR("failed to unmap legacy queue\n"); 837 838 return r; 839 } 840 841 uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg) 842 { 843 struct mes_misc_op_input op_input; 844 int r, val = 0; 845 846 op_input.op = MES_MISC_OP_READ_REG; 847 op_input.read_reg.reg_offset = reg; 848 op_input.read_reg.buffer_addr = adev->mes.read_val_gpu_addr; 849 850 if (!adev->mes.funcs->misc_op) { 851 DRM_ERROR("mes rreg is not supported!\n"); 852 goto error; 853 } 854 855 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 856 if (r) 857 DRM_ERROR("failed to read reg (0x%x)\n", reg); 858 else 859 val = *(adev->mes.read_val_ptr); 860 861 error: 862 return val; 863 } 864 865 int amdgpu_mes_wreg(struct amdgpu_device *adev, 866 uint32_t reg, uint32_t val) 867 { 868 struct mes_misc_op_input op_input; 869 int r; 870 871 op_input.op = MES_MISC_OP_WRITE_REG; 872 op_input.write_reg.reg_offset = reg; 873 op_input.write_reg.reg_value = val; 874 875 if (!adev->mes.funcs->misc_op) { 876 DRM_ERROR("mes wreg is not supported!\n"); 877 r = -EINVAL; 878 goto error; 879 } 880 881 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 882 if (r) 883 DRM_ERROR("failed to write reg (0x%x)\n", reg); 884 885 error: 886 return r; 887 } 888 889 int amdgpu_mes_reg_write_reg_wait(struct amdgpu_device *adev, 890 uint32_t reg0, uint32_t reg1, 891 uint32_t ref, uint32_t mask) 892 { 893 struct mes_misc_op_input op_input; 894 int r; 895 896 op_input.op = MES_MISC_OP_WRM_REG_WR_WAIT; 897 op_input.wrm_reg.reg0 = reg0; 898 op_input.wrm_reg.reg1 = reg1; 899 op_input.wrm_reg.ref = ref; 900 op_input.wrm_reg.mask = mask; 901 902 if (!adev->mes.funcs->misc_op) { 903 DRM_ERROR("mes reg_write_reg_wait is not supported!\n"); 904 r = -EINVAL; 905 goto error; 906 } 907 908 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 909 if (r) 910 DRM_ERROR("failed to reg_write_reg_wait\n"); 911 912 error: 913 return r; 914 } 915 916 int amdgpu_mes_reg_wait(struct amdgpu_device *adev, uint32_t reg, 917 uint32_t val, uint32_t mask) 918 { 919 struct mes_misc_op_input op_input; 920 int r; 921 922 op_input.op = MES_MISC_OP_WRM_REG_WAIT; 923 op_input.wrm_reg.reg0 = reg; 924 op_input.wrm_reg.ref = val; 925 op_input.wrm_reg.mask = mask; 926 927 if (!adev->mes.funcs->misc_op) { 928 DRM_ERROR("mes reg wait is not supported!\n"); 929 r = -EINVAL; 930 goto error; 931 } 932 933 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 934 if (r) 935 DRM_ERROR("failed to reg_write_reg_wait\n"); 936 937 error: 938 return r; 939 } 940 941 int amdgpu_mes_set_shader_debugger(struct amdgpu_device *adev, 942 uint64_t process_context_addr, 943 uint32_t spi_gdbg_per_vmid_cntl, 944 const uint32_t *tcp_watch_cntl, 945 uint32_t flags, 946 bool trap_en) 947 { 948 struct mes_misc_op_input op_input = {0}; 949 int r; 950 951 if (!adev->mes.funcs->misc_op) { 952 DRM_ERROR("mes set shader debugger is not supported!\n"); 953 return -EINVAL; 954 } 955 956 op_input.op = MES_MISC_OP_SET_SHADER_DEBUGGER; 957 op_input.set_shader_debugger.process_context_addr = process_context_addr; 958 op_input.set_shader_debugger.flags.u32all = flags; 959 960 /* use amdgpu mes_flush_shader_debugger instead */ 961 if (op_input.set_shader_debugger.flags.process_ctx_flush) 962 return -EINVAL; 963 964 op_input.set_shader_debugger.spi_gdbg_per_vmid_cntl = spi_gdbg_per_vmid_cntl; 965 memcpy(op_input.set_shader_debugger.tcp_watch_cntl, tcp_watch_cntl, 966 sizeof(op_input.set_shader_debugger.tcp_watch_cntl)); 967 968 if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >> 969 AMDGPU_MES_API_VERSION_SHIFT) >= 14) 970 op_input.set_shader_debugger.trap_en = trap_en; 971 972 amdgpu_mes_lock(&adev->mes); 973 974 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 975 if (r) 976 DRM_ERROR("failed to set_shader_debugger\n"); 977 978 amdgpu_mes_unlock(&adev->mes); 979 980 return r; 981 } 982 983 int amdgpu_mes_flush_shader_debugger(struct amdgpu_device *adev, 984 uint64_t process_context_addr) 985 { 986 struct mes_misc_op_input op_input = {0}; 987 int r; 988 989 if (!adev->mes.funcs->misc_op) { 990 DRM_ERROR("mes flush shader debugger is not supported!\n"); 991 return -EINVAL; 992 } 993 994 op_input.op = MES_MISC_OP_SET_SHADER_DEBUGGER; 995 op_input.set_shader_debugger.process_context_addr = process_context_addr; 996 op_input.set_shader_debugger.flags.process_ctx_flush = true; 997 998 amdgpu_mes_lock(&adev->mes); 999 1000 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); 1001 if (r) 1002 DRM_ERROR("failed to set_shader_debugger\n"); 1003 1004 amdgpu_mes_unlock(&adev->mes); 1005 1006 return r; 1007 } 1008 1009 static void 1010 amdgpu_mes_ring_to_queue_props(struct amdgpu_device *adev, 1011 struct amdgpu_ring *ring, 1012 struct amdgpu_mes_queue_properties *props) 1013 { 1014 props->queue_type = ring->funcs->type; 1015 props->hqd_base_gpu_addr = ring->gpu_addr; 1016 props->rptr_gpu_addr = ring->rptr_gpu_addr; 1017 props->wptr_gpu_addr = ring->wptr_gpu_addr; 1018 props->wptr_mc_addr = 1019 ring->mes_ctx->meta_data_mc_addr + ring->wptr_offs; 1020 props->queue_size = ring->ring_size; 1021 props->eop_gpu_addr = ring->eop_gpu_addr; 1022 props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL; 1023 props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM; 1024 props->paging = false; 1025 props->ring = ring; 1026 } 1027 1028 #define DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(_eng) \ 1029 do { \ 1030 if (id_offs < AMDGPU_MES_CTX_MAX_OFFS) \ 1031 return offsetof(struct amdgpu_mes_ctx_meta_data, \ 1032 _eng[ring->idx].slots[id_offs]); \ 1033 else if (id_offs == AMDGPU_MES_CTX_RING_OFFS) \ 1034 return offsetof(struct amdgpu_mes_ctx_meta_data, \ 1035 _eng[ring->idx].ring); \ 1036 else if (id_offs == AMDGPU_MES_CTX_IB_OFFS) \ 1037 return offsetof(struct amdgpu_mes_ctx_meta_data, \ 1038 _eng[ring->idx].ib); \ 1039 else if (id_offs == AMDGPU_MES_CTX_PADDING_OFFS) \ 1040 return offsetof(struct amdgpu_mes_ctx_meta_data, \ 1041 _eng[ring->idx].padding); \ 1042 } while(0) 1043 1044 int amdgpu_mes_ctx_get_offs(struct amdgpu_ring *ring, unsigned int id_offs) 1045 { 1046 switch (ring->funcs->type) { 1047 case AMDGPU_RING_TYPE_GFX: 1048 DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(gfx); 1049 break; 1050 case AMDGPU_RING_TYPE_COMPUTE: 1051 DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(compute); 1052 break; 1053 case AMDGPU_RING_TYPE_SDMA: 1054 DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(sdma); 1055 break; 1056 default: 1057 break; 1058 } 1059 1060 WARN_ON(1); 1061 return -EINVAL; 1062 } 1063 1064 int amdgpu_mes_add_ring(struct amdgpu_device *adev, int gang_id, 1065 int queue_type, int idx, 1066 struct amdgpu_mes_ctx_data *ctx_data, 1067 struct amdgpu_ring **out) 1068 { 1069 struct amdgpu_ring *ring; 1070 struct amdgpu_mes_gang *gang; 1071 struct amdgpu_mes_queue_properties qprops = {0}; 1072 int r, queue_id, pasid; 1073 1074 /* 1075 * Avoid taking any other locks under MES lock to avoid circular 1076 * lock dependencies. 1077 */ 1078 amdgpu_mes_lock(&adev->mes); 1079 gang = idr_find(&adev->mes.gang_id_idr, gang_id); 1080 if (!gang) { 1081 DRM_ERROR("gang id %d doesn't exist\n", gang_id); 1082 amdgpu_mes_unlock(&adev->mes); 1083 return -EINVAL; 1084 } 1085 pasid = gang->process->pasid; 1086 1087 ring = kzalloc(sizeof(struct amdgpu_ring), GFP_KERNEL); 1088 if (!ring) { 1089 amdgpu_mes_unlock(&adev->mes); 1090 return -ENOMEM; 1091 } 1092 1093 ring->ring_obj = NULL; 1094 ring->use_doorbell = true; 1095 ring->is_mes_queue = true; 1096 ring->mes_ctx = ctx_data; 1097 ring->idx = idx; 1098 ring->no_scheduler = true; 1099 1100 if (queue_type == AMDGPU_RING_TYPE_COMPUTE) { 1101 int offset = offsetof(struct amdgpu_mes_ctx_meta_data, 1102 compute[ring->idx].mec_hpd); 1103 ring->eop_gpu_addr = 1104 amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); 1105 } 1106 1107 switch (queue_type) { 1108 case AMDGPU_RING_TYPE_GFX: 1109 ring->funcs = adev->gfx.gfx_ring[0].funcs; 1110 ring->me = adev->gfx.gfx_ring[0].me; 1111 ring->pipe = adev->gfx.gfx_ring[0].pipe; 1112 break; 1113 case AMDGPU_RING_TYPE_COMPUTE: 1114 ring->funcs = adev->gfx.compute_ring[0].funcs; 1115 ring->me = adev->gfx.compute_ring[0].me; 1116 ring->pipe = adev->gfx.compute_ring[0].pipe; 1117 break; 1118 case AMDGPU_RING_TYPE_SDMA: 1119 ring->funcs = adev->sdma.instance[0].ring.funcs; 1120 break; 1121 default: 1122 BUG(); 1123 } 1124 1125 r = amdgpu_ring_init(adev, ring, 1024, NULL, 0, 1126 AMDGPU_RING_PRIO_DEFAULT, NULL); 1127 if (r) 1128 goto clean_up_memory; 1129 1130 amdgpu_mes_ring_to_queue_props(adev, ring, &qprops); 1131 1132 dma_fence_wait(gang->process->vm->last_update, false); 1133 dma_fence_wait(ctx_data->meta_data_va->last_pt_update, false); 1134 amdgpu_mes_unlock(&adev->mes); 1135 1136 r = amdgpu_mes_add_hw_queue(adev, gang_id, &qprops, &queue_id); 1137 if (r) 1138 goto clean_up_ring; 1139 1140 ring->hw_queue_id = queue_id; 1141 ring->doorbell_index = qprops.doorbell_off; 1142 1143 if (queue_type == AMDGPU_RING_TYPE_GFX) 1144 sprintf(ring->name, "gfx_%d.%d.%d", pasid, gang_id, queue_id); 1145 else if (queue_type == AMDGPU_RING_TYPE_COMPUTE) 1146 sprintf(ring->name, "compute_%d.%d.%d", pasid, gang_id, 1147 queue_id); 1148 else if (queue_type == AMDGPU_RING_TYPE_SDMA) 1149 sprintf(ring->name, "sdma_%d.%d.%d", pasid, gang_id, 1150 queue_id); 1151 else 1152 BUG(); 1153 1154 *out = ring; 1155 return 0; 1156 1157 clean_up_ring: 1158 amdgpu_ring_fini(ring); 1159 clean_up_memory: 1160 kfree(ring); 1161 amdgpu_mes_unlock(&adev->mes); 1162 return r; 1163 } 1164 1165 void amdgpu_mes_remove_ring(struct amdgpu_device *adev, 1166 struct amdgpu_ring *ring) 1167 { 1168 if (!ring) 1169 return; 1170 1171 amdgpu_mes_remove_hw_queue(adev, ring->hw_queue_id); 1172 del_timer_sync(&ring->fence_drv.fallback_timer); 1173 amdgpu_ring_fini(ring); 1174 kfree(ring); 1175 } 1176 1177 uint32_t amdgpu_mes_get_aggregated_doorbell_index(struct amdgpu_device *adev, 1178 enum amdgpu_mes_priority_level prio) 1179 { 1180 return adev->mes.aggregated_doorbells[prio]; 1181 } 1182 1183 int amdgpu_mes_ctx_alloc_meta_data(struct amdgpu_device *adev, 1184 struct amdgpu_mes_ctx_data *ctx_data) 1185 { 1186 int r; 1187 1188 r = amdgpu_bo_create_kernel(adev, 1189 sizeof(struct amdgpu_mes_ctx_meta_data), 1190 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1191 &ctx_data->meta_data_obj, 1192 &ctx_data->meta_data_mc_addr, 1193 &ctx_data->meta_data_ptr); 1194 if (r) { 1195 dev_warn(adev->dev, "(%d) create CTX bo failed\n", r); 1196 return r; 1197 } 1198 1199 if (!ctx_data->meta_data_obj) 1200 return -ENOMEM; 1201 1202 memset(ctx_data->meta_data_ptr, 0, 1203 sizeof(struct amdgpu_mes_ctx_meta_data)); 1204 1205 return 0; 1206 } 1207 1208 void amdgpu_mes_ctx_free_meta_data(struct amdgpu_mes_ctx_data *ctx_data) 1209 { 1210 if (ctx_data->meta_data_obj) 1211 amdgpu_bo_free_kernel(&ctx_data->meta_data_obj, 1212 &ctx_data->meta_data_mc_addr, 1213 &ctx_data->meta_data_ptr); 1214 } 1215 1216 int amdgpu_mes_ctx_map_meta_data(struct amdgpu_device *adev, 1217 struct amdgpu_vm *vm, 1218 struct amdgpu_mes_ctx_data *ctx_data) 1219 { 1220 struct amdgpu_bo_va *bo_va; 1221 struct amdgpu_sync sync; 1222 struct drm_exec exec; 1223 int r; 1224 1225 amdgpu_sync_create(&sync); 1226 1227 drm_exec_init(&exec, 0, 0); 1228 drm_exec_until_all_locked(&exec) { 1229 r = drm_exec_lock_obj(&exec, 1230 &ctx_data->meta_data_obj->tbo.base); 1231 drm_exec_retry_on_contention(&exec); 1232 if (unlikely(r)) 1233 goto error_fini_exec; 1234 1235 r = amdgpu_vm_lock_pd(vm, &exec, 0); 1236 drm_exec_retry_on_contention(&exec); 1237 if (unlikely(r)) 1238 goto error_fini_exec; 1239 } 1240 1241 bo_va = amdgpu_vm_bo_add(adev, vm, ctx_data->meta_data_obj); 1242 if (!bo_va) { 1243 DRM_ERROR("failed to create bo_va for meta data BO\n"); 1244 r = -ENOMEM; 1245 goto error_fini_exec; 1246 } 1247 1248 r = amdgpu_vm_bo_map(adev, bo_va, ctx_data->meta_data_gpu_addr, 0, 1249 sizeof(struct amdgpu_mes_ctx_meta_data), 1250 AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE | 1251 AMDGPU_PTE_EXECUTABLE); 1252 1253 if (r) { 1254 DRM_ERROR("failed to do bo_map on meta data, err=%d\n", r); 1255 goto error_del_bo_va; 1256 } 1257 1258 r = amdgpu_vm_bo_update(adev, bo_va, false); 1259 if (r) { 1260 DRM_ERROR("failed to do vm_bo_update on meta data\n"); 1261 goto error_del_bo_va; 1262 } 1263 amdgpu_sync_fence(&sync, bo_va->last_pt_update); 1264 1265 r = amdgpu_vm_update_pdes(adev, vm, false); 1266 if (r) { 1267 DRM_ERROR("failed to update pdes on meta data\n"); 1268 goto error_del_bo_va; 1269 } 1270 amdgpu_sync_fence(&sync, vm->last_update); 1271 1272 amdgpu_sync_wait(&sync, false); 1273 drm_exec_fini(&exec); 1274 1275 amdgpu_sync_free(&sync); 1276 ctx_data->meta_data_va = bo_va; 1277 return 0; 1278 1279 error_del_bo_va: 1280 amdgpu_vm_bo_del(adev, bo_va); 1281 1282 error_fini_exec: 1283 drm_exec_fini(&exec); 1284 amdgpu_sync_free(&sync); 1285 return r; 1286 } 1287 1288 int amdgpu_mes_ctx_unmap_meta_data(struct amdgpu_device *adev, 1289 struct amdgpu_mes_ctx_data *ctx_data) 1290 { 1291 struct amdgpu_bo_va *bo_va = ctx_data->meta_data_va; 1292 struct amdgpu_bo *bo = ctx_data->meta_data_obj; 1293 struct amdgpu_vm *vm = bo_va->base.vm; 1294 struct dma_fence *fence; 1295 struct drm_exec exec; 1296 long r; 1297 1298 drm_exec_init(&exec, 0, 0); 1299 drm_exec_until_all_locked(&exec) { 1300 r = drm_exec_lock_obj(&exec, 1301 &ctx_data->meta_data_obj->tbo.base); 1302 drm_exec_retry_on_contention(&exec); 1303 if (unlikely(r)) 1304 goto out_unlock; 1305 1306 r = amdgpu_vm_lock_pd(vm, &exec, 0); 1307 drm_exec_retry_on_contention(&exec); 1308 if (unlikely(r)) 1309 goto out_unlock; 1310 } 1311 1312 amdgpu_vm_bo_del(adev, bo_va); 1313 if (!amdgpu_vm_ready(vm)) 1314 goto out_unlock; 1315 1316 r = dma_resv_get_singleton(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP, 1317 &fence); 1318 if (r) 1319 goto out_unlock; 1320 if (fence) { 1321 amdgpu_bo_fence(bo, fence, true); 1322 fence = NULL; 1323 } 1324 1325 r = amdgpu_vm_clear_freed(adev, vm, &fence); 1326 if (r || !fence) 1327 goto out_unlock; 1328 1329 dma_fence_wait(fence, false); 1330 amdgpu_bo_fence(bo, fence, true); 1331 dma_fence_put(fence); 1332 1333 out_unlock: 1334 if (unlikely(r < 0)) 1335 dev_err(adev->dev, "failed to clear page tables (%ld)\n", r); 1336 drm_exec_fini(&exec); 1337 1338 return r; 1339 } 1340 1341 static int amdgpu_mes_test_create_gang_and_queues(struct amdgpu_device *adev, 1342 int pasid, int *gang_id, 1343 int queue_type, int num_queue, 1344 struct amdgpu_ring **added_rings, 1345 struct amdgpu_mes_ctx_data *ctx_data) 1346 { 1347 struct amdgpu_ring *ring; 1348 struct amdgpu_mes_gang_properties gprops = {0}; 1349 int r, j; 1350 1351 /* create a gang for the process */ 1352 gprops.priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 1353 gprops.gang_quantum = adev->mes.default_gang_quantum; 1354 gprops.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 1355 gprops.priority_level = AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 1356 gprops.global_priority_level = AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 1357 1358 r = amdgpu_mes_add_gang(adev, pasid, &gprops, gang_id); 1359 if (r) { 1360 DRM_ERROR("failed to add gang\n"); 1361 return r; 1362 } 1363 1364 /* create queues for the gang */ 1365 for (j = 0; j < num_queue; j++) { 1366 r = amdgpu_mes_add_ring(adev, *gang_id, queue_type, j, 1367 ctx_data, &ring); 1368 if (r) { 1369 DRM_ERROR("failed to add ring\n"); 1370 break; 1371 } 1372 1373 DRM_INFO("ring %s was added\n", ring->name); 1374 added_rings[j] = ring; 1375 } 1376 1377 return 0; 1378 } 1379 1380 static int amdgpu_mes_test_queues(struct amdgpu_ring **added_rings) 1381 { 1382 struct amdgpu_ring *ring; 1383 int i, r; 1384 1385 for (i = 0; i < AMDGPU_MES_CTX_MAX_RINGS; i++) { 1386 ring = added_rings[i]; 1387 if (!ring) 1388 continue; 1389 1390 r = amdgpu_ring_test_helper(ring); 1391 if (r) 1392 return r; 1393 1394 r = amdgpu_ring_test_ib(ring, 1000 * 10); 1395 if (r) { 1396 DRM_DEV_ERROR(ring->adev->dev, 1397 "ring %s ib test failed (%d)\n", 1398 ring->name, r); 1399 return r; 1400 } else 1401 DRM_INFO("ring %s ib test pass\n", ring->name); 1402 } 1403 1404 return 0; 1405 } 1406 1407 int amdgpu_mes_self_test(struct amdgpu_device *adev) 1408 { 1409 struct amdgpu_vm *vm = NULL; 1410 struct amdgpu_mes_ctx_data ctx_data = {0}; 1411 struct amdgpu_ring *added_rings[AMDGPU_MES_CTX_MAX_RINGS] = { NULL }; 1412 int gang_ids[3] = {0}; 1413 int queue_types[][2] = { { AMDGPU_RING_TYPE_GFX, 1 }, 1414 { AMDGPU_RING_TYPE_COMPUTE, 1 }, 1415 { AMDGPU_RING_TYPE_SDMA, 1} }; 1416 int i, r, pasid, k = 0; 1417 1418 pasid = amdgpu_pasid_alloc(16); 1419 if (pasid < 0) { 1420 dev_warn(adev->dev, "No more PASIDs available!"); 1421 pasid = 0; 1422 } 1423 1424 vm = kzalloc(sizeof(*vm), GFP_KERNEL); 1425 if (!vm) { 1426 r = -ENOMEM; 1427 goto error_pasid; 1428 } 1429 1430 r = amdgpu_vm_init(adev, vm, -1); 1431 if (r) { 1432 DRM_ERROR("failed to initialize vm\n"); 1433 goto error_pasid; 1434 } 1435 1436 r = amdgpu_mes_ctx_alloc_meta_data(adev, &ctx_data); 1437 if (r) { 1438 DRM_ERROR("failed to alloc ctx meta data\n"); 1439 goto error_fini; 1440 } 1441 1442 ctx_data.meta_data_gpu_addr = AMDGPU_VA_RESERVED_BOTTOM; 1443 r = amdgpu_mes_ctx_map_meta_data(adev, vm, &ctx_data); 1444 if (r) { 1445 DRM_ERROR("failed to map ctx meta data\n"); 1446 goto error_vm; 1447 } 1448 1449 r = amdgpu_mes_create_process(adev, pasid, vm); 1450 if (r) { 1451 DRM_ERROR("failed to create MES process\n"); 1452 goto error_vm; 1453 } 1454 1455 for (i = 0; i < ARRAY_SIZE(queue_types); i++) { 1456 /* On GFX v10.3, fw hasn't supported to map sdma queue. */ 1457 if (amdgpu_ip_version(adev, GC_HWIP, 0) >= 1458 IP_VERSION(10, 3, 0) && 1459 amdgpu_ip_version(adev, GC_HWIP, 0) < 1460 IP_VERSION(11, 0, 0) && 1461 queue_types[i][0] == AMDGPU_RING_TYPE_SDMA) 1462 continue; 1463 1464 r = amdgpu_mes_test_create_gang_and_queues(adev, pasid, 1465 &gang_ids[i], 1466 queue_types[i][0], 1467 queue_types[i][1], 1468 &added_rings[k], 1469 &ctx_data); 1470 if (r) 1471 goto error_queues; 1472 1473 k += queue_types[i][1]; 1474 } 1475 1476 /* start ring test and ib test for MES queues */ 1477 amdgpu_mes_test_queues(added_rings); 1478 1479 error_queues: 1480 /* remove all queues */ 1481 for (i = 0; i < ARRAY_SIZE(added_rings); i++) { 1482 if (!added_rings[i]) 1483 continue; 1484 amdgpu_mes_remove_ring(adev, added_rings[i]); 1485 } 1486 1487 for (i = 0; i < ARRAY_SIZE(gang_ids); i++) { 1488 if (!gang_ids[i]) 1489 continue; 1490 amdgpu_mes_remove_gang(adev, gang_ids[i]); 1491 } 1492 1493 amdgpu_mes_destroy_process(adev, pasid); 1494 1495 error_vm: 1496 amdgpu_mes_ctx_unmap_meta_data(adev, &ctx_data); 1497 1498 error_fini: 1499 amdgpu_vm_fini(adev, vm); 1500 1501 error_pasid: 1502 if (pasid) 1503 amdgpu_pasid_free(pasid); 1504 1505 amdgpu_mes_ctx_free_meta_data(&ctx_data); 1506 kfree(vm); 1507 return 0; 1508 } 1509 1510 int amdgpu_mes_init_microcode(struct amdgpu_device *adev, int pipe) 1511 { 1512 const struct mes_firmware_header_v1_0 *mes_hdr; 1513 struct amdgpu_firmware_info *info; 1514 char ucode_prefix[30]; 1515 char fw_name[50]; 1516 bool need_retry = false; 1517 int r; 1518 1519 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, 1520 sizeof(ucode_prefix)); 1521 if (adev->enable_uni_mes) { 1522 snprintf(fw_name, sizeof(fw_name), 1523 "amdgpu/%s_uni_mes.bin", ucode_prefix); 1524 } else if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) && 1525 amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)) { 1526 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes%s.bin", 1527 ucode_prefix, 1528 pipe == AMDGPU_MES_SCHED_PIPE ? "_2" : "1"); 1529 need_retry = true; 1530 } else { 1531 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes%s.bin", 1532 ucode_prefix, 1533 pipe == AMDGPU_MES_SCHED_PIPE ? "" : "1"); 1534 } 1535 1536 r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], fw_name); 1537 if (r && need_retry && pipe == AMDGPU_MES_SCHED_PIPE) { 1538 dev_info(adev->dev, "try to fall back to %s_mes.bin\n", ucode_prefix); 1539 r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], 1540 "amdgpu/%s_mes.bin", ucode_prefix); 1541 } 1542 1543 if (r) 1544 goto out; 1545 1546 mes_hdr = (const struct mes_firmware_header_v1_0 *) 1547 adev->mes.fw[pipe]->data; 1548 adev->mes.uc_start_addr[pipe] = 1549 le32_to_cpu(mes_hdr->mes_uc_start_addr_lo) | 1550 ((uint64_t)(le32_to_cpu(mes_hdr->mes_uc_start_addr_hi)) << 32); 1551 adev->mes.data_start_addr[pipe] = 1552 le32_to_cpu(mes_hdr->mes_data_start_addr_lo) | 1553 ((uint64_t)(le32_to_cpu(mes_hdr->mes_data_start_addr_hi)) << 32); 1554 1555 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1556 int ucode, ucode_data; 1557 1558 if (pipe == AMDGPU_MES_SCHED_PIPE) { 1559 ucode = AMDGPU_UCODE_ID_CP_MES; 1560 ucode_data = AMDGPU_UCODE_ID_CP_MES_DATA; 1561 } else { 1562 ucode = AMDGPU_UCODE_ID_CP_MES1; 1563 ucode_data = AMDGPU_UCODE_ID_CP_MES1_DATA; 1564 } 1565 1566 info = &adev->firmware.ucode[ucode]; 1567 info->ucode_id = ucode; 1568 info->fw = adev->mes.fw[pipe]; 1569 adev->firmware.fw_size += 1570 ALIGN(le32_to_cpu(mes_hdr->mes_ucode_size_bytes), 1571 PAGE_SIZE); 1572 1573 info = &adev->firmware.ucode[ucode_data]; 1574 info->ucode_id = ucode_data; 1575 info->fw = adev->mes.fw[pipe]; 1576 adev->firmware.fw_size += 1577 ALIGN(le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes), 1578 PAGE_SIZE); 1579 } 1580 1581 return 0; 1582 out: 1583 amdgpu_ucode_release(&adev->mes.fw[pipe]); 1584 return r; 1585 } 1586 1587 #if defined(CONFIG_DEBUG_FS) 1588 1589 static int amdgpu_debugfs_mes_event_log_show(struct seq_file *m, void *unused) 1590 { 1591 struct amdgpu_device *adev = m->private; 1592 uint32_t *mem = (uint32_t *)(adev->mes.event_log_cpu_addr); 1593 1594 seq_hex_dump(m, "", DUMP_PREFIX_OFFSET, 32, 4, 1595 mem, adev->mes.event_log_size, false); 1596 1597 return 0; 1598 } 1599 1600 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_mes_event_log); 1601 1602 #endif 1603 1604 void amdgpu_debugfs_mes_event_log_init(struct amdgpu_device *adev) 1605 { 1606 1607 #if defined(CONFIG_DEBUG_FS) 1608 struct drm_minor *minor = adev_to_drm(adev)->primary; 1609 struct dentry *root = minor->debugfs_root; 1610 if (adev->enable_mes && amdgpu_mes_log_enable) 1611 debugfs_create_file("amdgpu_mes_event_log", 0444, root, 1612 adev, &amdgpu_debugfs_mes_event_log_fops); 1613 1614 #endif 1615 } 1616