1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2024 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 #include <drm/drm_drv.h> 25 #include "amdgpu.h" 26 #include "amdgpu_gfx.h" 27 #include "mes_userqueue.h" 28 #include "amdgpu_userq_fence.h" 29 30 #define AMDGPU_USERQ_PROC_CTX_SZ PAGE_SIZE 31 #define AMDGPU_USERQ_GANG_CTX_SZ PAGE_SIZE 32 33 static int 34 mes_userq_create_wptr_mapping(struct amdgpu_device *adev, 35 struct amdgpu_userq_mgr *uq_mgr, 36 struct amdgpu_usermode_queue *queue, 37 uint64_t wptr) 38 { 39 struct amdgpu_bo_va_mapping *wptr_mapping; 40 struct amdgpu_userq_obj *wptr_obj = &queue->wptr_obj; 41 struct amdgpu_bo *obj; 42 struct amdgpu_vm *vm = queue->vm; 43 struct drm_exec exec; 44 int ret; 45 46 wptr &= AMDGPU_GMC_HOLE_MASK; 47 48 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 2); 49 drm_exec_until_all_locked(&exec) { 50 ret = amdgpu_vm_lock_pd(vm, &exec, 1); 51 drm_exec_retry_on_contention(&exec); 52 if (unlikely(ret)) 53 goto fail_lock; 54 55 wptr_mapping = amdgpu_vm_bo_lookup_mapping(vm, wptr >> PAGE_SHIFT); 56 if (!wptr_mapping) { 57 ret = -EINVAL; 58 goto fail_lock; 59 } 60 61 obj = wptr_mapping->bo_va->base.bo; 62 ret = drm_exec_lock_obj(&exec, &obj->tbo.base); 63 drm_exec_retry_on_contention(&exec); 64 if (unlikely(ret)) 65 goto fail_lock; 66 } 67 68 wptr_obj->obj = amdgpu_bo_ref(wptr_mapping->bo_va->base.bo); 69 if (wptr_obj->obj->tbo.base.size > PAGE_SIZE) { 70 ret = -EINVAL; 71 goto fail_map; 72 } 73 /* Keep WPTR BO under eviction-fence control instead of pinning. */ 74 ret = amdgpu_evf_mgr_attach_fence(&uq_mgr_to_fpriv(uq_mgr)->evf_mgr, wptr_obj->obj); 75 if (ret) { 76 DRM_ERROR("Failed to attach eviction fence to wptr bo. ret %d\n", ret); 77 goto fail_map; 78 } 79 80 ret = amdgpu_ttm_alloc_gart(&wptr_obj->obj->tbo); 81 if (ret) { 82 DRM_ERROR("Failed to bind wptr bo to GART. ret %d\n", ret); 83 goto fail_map; 84 } 85 86 queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(wptr_obj->obj); 87 88 drm_exec_fini(&exec); 89 return 0; 90 fail_map: 91 amdgpu_bo_unref(&wptr_obj->obj); 92 fail_lock: 93 drm_exec_fini(&exec); 94 return ret; 95 96 } 97 98 static int convert_to_mes_priority(int priority) 99 { 100 switch (priority) { 101 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_LOW: 102 default: 103 return AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 104 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_LOW: 105 return AMDGPU_MES_PRIORITY_LEVEL_LOW; 106 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_HIGH: 107 return AMDGPU_MES_PRIORITY_LEVEL_MEDIUM; 108 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_HIGH: 109 return AMDGPU_MES_PRIORITY_LEVEL_HIGH; 110 } 111 } 112 113 static int mes_userq_map(struct amdgpu_usermode_queue *queue) 114 { 115 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 116 struct amdgpu_device *adev = uq_mgr->adev; 117 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 118 struct amdgpu_mqd_prop *userq_props = queue->userq_prop; 119 struct mes_add_queue_input queue_input; 120 int r; 121 122 memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input)); 123 124 queue_input.process_va_start = 0; 125 queue_input.process_va_end = adev->vm_manager.max_pfn - 1; 126 127 /* set process quantum to 10 ms and gang quantum to 1 ms as default */ 128 queue_input.process_quantum = 100000; 129 queue_input.gang_quantum = 10000; 130 queue_input.paging = false; 131 132 queue_input.process_context_addr = ctx->gpu_addr; 133 queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ; 134 queue_input.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 135 queue_input.gang_global_priority_level = convert_to_mes_priority(queue->priority); 136 137 queue_input.process_id = queue->vm->pasid; 138 queue_input.queue_type = queue->queue_type; 139 queue_input.mqd_addr = queue->mqd.gpu_addr; 140 queue_input.wptr_addr = userq_props->wptr_gpu_addr; 141 queue_input.queue_size = userq_props->queue_size >> 2; 142 queue_input.doorbell_offset = userq_props->doorbell_index; 143 queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo); 144 queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr; 145 146 amdgpu_mes_lock(&adev->mes); 147 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input); 148 amdgpu_mes_unlock(&adev->mes); 149 if (r) { 150 DRM_ERROR("Failed to map queue in HW, err (%d)\n", r); 151 return r; 152 } 153 154 DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index); 155 return 0; 156 } 157 158 static int mes_userq_unmap(struct amdgpu_usermode_queue *queue) 159 { 160 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 161 struct amdgpu_device *adev = uq_mgr->adev; 162 struct mes_remove_queue_input queue_input; 163 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 164 int r; 165 166 memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); 167 queue_input.doorbell_offset = queue->doorbell_index; 168 queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ; 169 170 amdgpu_mes_lock(&adev->mes); 171 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); 172 amdgpu_mes_unlock(&adev->mes); 173 if (r) 174 DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r); 175 return r; 176 } 177 178 static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr, 179 struct amdgpu_usermode_queue *queue, 180 struct drm_amdgpu_userq_in *mqd_user) 181 { 182 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 183 int r, size; 184 185 /* 186 * The FW expects at least one page space allocated for 187 * process ctx and gang ctx each. Create an object 188 * for the same. 189 */ 190 size = AMDGPU_USERQ_PROC_CTX_SZ + AMDGPU_USERQ_GANG_CTX_SZ; 191 r = amdgpu_bo_create_kernel(uq_mgr->adev, size, 0, 192 AMDGPU_GEM_DOMAIN_GTT, 193 &ctx->obj, &ctx->gpu_addr, 194 &ctx->cpu_ptr); 195 if (r) { 196 DRM_ERROR("Failed to allocate ctx space bo for userqueue, err:%d\n", r); 197 return r; 198 } 199 200 memset(ctx->cpu_ptr, 0, size); 201 return 0; 202 } 203 204 static int mes_userq_detect_and_reset(struct amdgpu_device *adev, 205 int queue_type) 206 { 207 int db_array_size = amdgpu_mes_get_hung_queue_db_array_size(adev); 208 struct mes_detect_and_reset_queue_input input; 209 struct amdgpu_usermode_queue *queue; 210 unsigned int hung_db_num = 0; 211 unsigned long queue_id; 212 u32 db_array[8]; 213 bool found_hung_queue = false; 214 int r, i; 215 216 if (db_array_size > 8) { 217 dev_err(adev->dev, "DB array size (%d vs 8) too small\n", 218 db_array_size); 219 return -EINVAL; 220 } 221 222 memset(&input, 0x0, sizeof(struct mes_detect_and_reset_queue_input)); 223 224 input.queue_type = queue_type; 225 226 amdgpu_mes_lock(&adev->mes); 227 r = amdgpu_mes_detect_and_reset_hung_queues(adev, queue_type, false, 228 &hung_db_num, db_array, 0); 229 amdgpu_mes_unlock(&adev->mes); 230 if (r) { 231 dev_err(adev->dev, "Failed to detect and reset queues, err (%d)\n", r); 232 } else if (hung_db_num) { 233 xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) { 234 if (queue->queue_type == queue_type) { 235 for (i = 0; i < hung_db_num; i++) { 236 if (queue->doorbell_index == db_array[i]) { 237 queue->state = AMDGPU_USERQ_STATE_HUNG; 238 found_hung_queue = true; 239 atomic_inc(&adev->gpu_reset_counter); 240 amdgpu_userq_fence_driver_force_completion(queue); 241 drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, NULL); 242 } 243 } 244 } 245 } 246 } 247 248 if (found_hung_queue) { 249 /* Resume scheduling after hang recovery */ 250 r = amdgpu_mes_resume(adev, input.xcc_id); 251 } 252 253 return r; 254 } 255 256 static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, 257 struct drm_amdgpu_userq_in *args_in) 258 { 259 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 260 struct amdgpu_device *adev = uq_mgr->adev; 261 struct amdgpu_mqd *mqd_hw_default = &adev->mqds[queue->queue_type]; 262 struct drm_amdgpu_userq_in *mqd_user = args_in; 263 struct amdgpu_mqd_prop *userq_props; 264 int r; 265 266 /* Structure to initialize MQD for userqueue using generic MQD init function */ 267 userq_props = kzalloc_obj(struct amdgpu_mqd_prop); 268 if (!userq_props) { 269 DRM_ERROR("Failed to allocate memory for userq_props\n"); 270 return -ENOMEM; 271 } 272 273 r = amdgpu_bo_create_kernel(adev, 274 AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size), 275 0, AMDGPU_GEM_DOMAIN_GTT, 276 &queue->mqd.obj, &queue->mqd.gpu_addr, 277 &queue->mqd.cpu_ptr); 278 if (r) { 279 DRM_ERROR("Failed to create MQD object for userqueue\n"); 280 goto free_props; 281 } 282 283 memset(queue->mqd.cpu_ptr, 0, 284 AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size)); 285 286 /* Initialize the MQD BO with user given values */ 287 userq_props->wptr_gpu_addr = mqd_user->wptr_va; 288 userq_props->rptr_gpu_addr = mqd_user->rptr_va; 289 userq_props->queue_size = mqd_user->queue_size; 290 userq_props->hqd_base_gpu_addr = mqd_user->queue_va; 291 userq_props->mqd_gpu_addr = queue->mqd.gpu_addr; 292 userq_props->use_doorbell = true; 293 userq_props->doorbell_index = queue->doorbell_index; 294 userq_props->fence_address = queue->fence_drv->gpu_addr; 295 296 if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) { 297 struct drm_amdgpu_userq_mqd_compute_gfx11 *compute_mqd; 298 299 if (mqd_user->mqd_size != sizeof(*compute_mqd)) { 300 DRM_ERROR("Invalid compute IP MQD size\n"); 301 r = -EINVAL; 302 goto free_mqd; 303 } 304 305 compute_mqd = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); 306 if (IS_ERR(compute_mqd)) { 307 DRM_ERROR("Failed to read user MQD\n"); 308 r = -ENOMEM; 309 goto free_mqd; 310 } 311 312 r = amdgpu_bo_reserve(queue->vm->root.bo, false); 313 if (r) { 314 kfree(compute_mqd); 315 goto free_mqd; 316 } 317 r = amdgpu_userq_input_va_validate(adev, queue, 318 compute_mqd->eop_va, 2048, 319 &queue->userq_vas.va.eop); 320 amdgpu_bo_unreserve(queue->vm->root.bo); 321 if (r) { 322 kfree(compute_mqd); 323 goto free_mqd; 324 } 325 326 userq_props->eop_gpu_addr = compute_mqd->eop_va; 327 userq_props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL; 328 userq_props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM; 329 userq_props->hqd_active = false; 330 userq_props->tmz_queue = 331 mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE; 332 kfree(compute_mqd); 333 } else if (queue->queue_type == AMDGPU_HW_IP_GFX) { 334 struct drm_amdgpu_userq_mqd_gfx11 *mqd_gfx_v11; 335 struct amdgpu_gfx_shadow_info shadow_info; 336 337 if (adev->gfx.funcs->get_gfx_shadow_info) { 338 adev->gfx.funcs->get_gfx_shadow_info(adev, &shadow_info, true); 339 } else { 340 r = -EINVAL; 341 goto free_mqd; 342 } 343 344 if (mqd_user->mqd_size != sizeof(*mqd_gfx_v11) || !mqd_user->mqd) { 345 DRM_ERROR("Invalid GFX MQD\n"); 346 r = -EINVAL; 347 goto free_mqd; 348 } 349 350 mqd_gfx_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); 351 if (IS_ERR(mqd_gfx_v11)) { 352 DRM_ERROR("Failed to read user MQD\n"); 353 r = -ENOMEM; 354 goto free_mqd; 355 } 356 357 userq_props->shadow_addr = mqd_gfx_v11->shadow_va; 358 userq_props->csa_addr = mqd_gfx_v11->csa_va; 359 userq_props->tmz_queue = 360 mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE; 361 362 r = amdgpu_bo_reserve(queue->vm->root.bo, false); 363 if (r) { 364 kfree(mqd_gfx_v11); 365 goto free_mqd; 366 } 367 r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->shadow_va, 368 shadow_info.shadow_size, 369 &queue->userq_vas.va.shadow); 370 if (r) { 371 amdgpu_bo_unreserve(queue->vm->root.bo); 372 kfree(mqd_gfx_v11); 373 goto free_mqd; 374 } 375 376 r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->csa_va, 377 shadow_info.csa_size, 378 &queue->userq_vas.va.csa); 379 amdgpu_bo_unreserve(queue->vm->root.bo); 380 if (r) { 381 kfree(mqd_gfx_v11); 382 goto free_mqd; 383 } 384 385 kfree(mqd_gfx_v11); 386 } else if (queue->queue_type == AMDGPU_HW_IP_DMA) { 387 struct drm_amdgpu_userq_mqd_sdma_gfx11 *mqd_sdma_v11; 388 389 if (mqd_user->mqd_size != sizeof(*mqd_sdma_v11) || !mqd_user->mqd) { 390 DRM_ERROR("Invalid SDMA MQD\n"); 391 r = -EINVAL; 392 goto free_mqd; 393 } 394 395 mqd_sdma_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); 396 if (IS_ERR(mqd_sdma_v11)) { 397 DRM_ERROR("Failed to read sdma user MQD\n"); 398 r = -ENOMEM; 399 goto free_mqd; 400 } 401 402 r = amdgpu_bo_reserve(queue->vm->root.bo, false); 403 if (r) { 404 kfree(mqd_sdma_v11); 405 goto free_mqd; 406 } 407 r = amdgpu_userq_input_va_validate(adev, queue, mqd_sdma_v11->csa_va, 408 32, 409 &queue->userq_vas.va.csa); 410 amdgpu_bo_unreserve(queue->vm->root.bo); 411 if (r) { 412 kfree(mqd_sdma_v11); 413 goto free_mqd; 414 } 415 416 userq_props->csa_addr = mqd_sdma_v11->csa_va; 417 kfree(mqd_sdma_v11); 418 } 419 420 queue->userq_prop = userq_props; 421 422 r = mqd_hw_default->init_mqd(adev, (void *)queue->mqd.cpu_ptr, userq_props); 423 if (r) { 424 DRM_ERROR("Failed to initialize MQD for userqueue\n"); 425 goto free_mqd; 426 } 427 428 /* Create BO for FW operations */ 429 r = mes_userq_create_ctx_space(uq_mgr, queue, mqd_user); 430 if (r) { 431 DRM_ERROR("Failed to allocate BO for userqueue (%d)", r); 432 goto free_mqd; 433 } 434 435 /* FW expects WPTR BOs to be mapped into GART */ 436 r = mes_userq_create_wptr_mapping(adev, uq_mgr, queue, userq_props->wptr_gpu_addr); 437 if (r) { 438 DRM_ERROR("Failed to create WPTR mapping\n"); 439 goto free_ctx; 440 } 441 442 return 0; 443 444 free_ctx: 445 amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr, 446 &queue->fw_obj.cpu_ptr); 447 448 free_mqd: 449 amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr, 450 &queue->mqd.cpu_ptr); 451 452 free_props: 453 kfree(userq_props); 454 455 return r; 456 } 457 458 static void mes_userq_mqd_destroy(struct amdgpu_usermode_queue *queue) 459 { 460 461 amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr, 462 &queue->fw_obj.cpu_ptr); 463 kfree(queue->userq_prop); 464 amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr, 465 &queue->mqd.cpu_ptr); 466 467 amdgpu_bo_unref(&queue->wptr_obj.obj); 468 } 469 470 static int mes_userq_preempt(struct amdgpu_usermode_queue *queue) 471 { 472 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 473 struct amdgpu_device *adev = uq_mgr->adev; 474 struct mes_suspend_gang_input queue_input; 475 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 476 signed long timeout = 2100000; /* 2100 ms */ 477 u64 fence_gpu_addr; 478 u32 fence_offset; 479 u64 *fence_ptr; 480 int i, r; 481 482 if (queue->state != AMDGPU_USERQ_STATE_MAPPED) 483 return 0; 484 r = amdgpu_device_wb_get(adev, &fence_offset); 485 if (r) 486 return r; 487 488 fence_gpu_addr = adev->wb.gpu_addr + (fence_offset * 4); 489 fence_ptr = (u64 *)&adev->wb.wb[fence_offset]; 490 *fence_ptr = 0; 491 492 memset(&queue_input, 0x0, sizeof(struct mes_suspend_gang_input)); 493 queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ; 494 queue_input.suspend_fence_addr = fence_gpu_addr; 495 queue_input.suspend_fence_value = 1; 496 amdgpu_mes_lock(&adev->mes); 497 r = adev->mes.funcs->suspend_gang(&adev->mes, &queue_input); 498 amdgpu_mes_unlock(&adev->mes); 499 if (r) { 500 DRM_ERROR("Failed to suspend gang: %d\n", r); 501 goto out; 502 } 503 504 for (i = 0; i < timeout; i++) { 505 if (*fence_ptr == 1) 506 goto out; 507 udelay(1); 508 } 509 r = -ETIMEDOUT; 510 511 out: 512 amdgpu_device_wb_free(adev, fence_offset); 513 return r; 514 } 515 516 static int mes_userq_restore(struct amdgpu_usermode_queue *queue) 517 { 518 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 519 struct amdgpu_device *adev = uq_mgr->adev; 520 struct mes_resume_gang_input queue_input; 521 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 522 int r; 523 524 if (queue->state == AMDGPU_USERQ_STATE_HUNG) 525 return -EINVAL; 526 if (queue->state != AMDGPU_USERQ_STATE_PREEMPTED) 527 return 0; 528 529 memset(&queue_input, 0x0, sizeof(struct mes_resume_gang_input)); 530 queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ; 531 532 amdgpu_mes_lock(&adev->mes); 533 r = adev->mes.funcs->resume_gang(&adev->mes, &queue_input); 534 amdgpu_mes_unlock(&adev->mes); 535 if (r) 536 dev_err(adev->dev, "Failed to resume queue, err (%d)\n", r); 537 return r; 538 } 539 540 const struct amdgpu_userq_funcs userq_mes_funcs = { 541 .mqd_create = mes_userq_mqd_create, 542 .mqd_destroy = mes_userq_mqd_destroy, 543 .unmap = mes_userq_unmap, 544 .map = mes_userq_map, 545 .detect_and_reset = mes_userq_detect_and_reset, 546 .preempt = mes_userq_preempt, 547 .restore = mes_userq_restore, 548 }; 549