1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2024 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 #include <drm/drm_drv.h> 25 #include "amdgpu.h" 26 #include "amdgpu_gfx.h" 27 #include "mes_userqueue.h" 28 #include "amdgpu_userq_fence.h" 29 30 #define AMDGPU_USERQ_PROC_CTX_SZ PAGE_SIZE 31 #define AMDGPU_USERQ_GANG_CTX_SZ PAGE_SIZE 32 33 static int 34 mes_userq_map_gtt_bo_to_gart(struct amdgpu_bo *bo) 35 { 36 int ret; 37 38 ret = amdgpu_bo_reserve(bo, true); 39 if (ret) { 40 DRM_ERROR("Failed to reserve bo. ret %d\n", ret); 41 goto err_reserve_bo_failed; 42 } 43 44 ret = amdgpu_ttm_alloc_gart(&bo->tbo); 45 if (ret) { 46 DRM_ERROR("Failed to bind bo to GART. ret %d\n", ret); 47 goto err_map_bo_gart_failed; 48 } 49 50 amdgpu_bo_unreserve(bo); 51 bo = amdgpu_bo_ref(bo); 52 53 return 0; 54 55 err_map_bo_gart_failed: 56 amdgpu_bo_unreserve(bo); 57 err_reserve_bo_failed: 58 return ret; 59 } 60 61 static int 62 mes_userq_create_wptr_mapping(struct amdgpu_device *adev, 63 struct amdgpu_userq_mgr *uq_mgr, 64 struct amdgpu_usermode_queue *queue, 65 uint64_t wptr) 66 { 67 struct amdgpu_bo_va_mapping *wptr_mapping; 68 struct amdgpu_vm *wptr_vm; 69 struct amdgpu_userq_obj *wptr_obj = &queue->wptr_obj; 70 int ret; 71 72 wptr_vm = queue->vm; 73 ret = amdgpu_bo_reserve(wptr_vm->root.bo, false); 74 if (ret) 75 return ret; 76 77 wptr &= AMDGPU_GMC_HOLE_MASK; 78 wptr_mapping = amdgpu_vm_bo_lookup_mapping(wptr_vm, wptr >> PAGE_SHIFT); 79 amdgpu_bo_unreserve(wptr_vm->root.bo); 80 if (!wptr_mapping) { 81 DRM_ERROR("Failed to lookup wptr bo\n"); 82 return -EINVAL; 83 } 84 85 wptr_obj->obj = wptr_mapping->bo_va->base.bo; 86 if (wptr_obj->obj->tbo.base.size > PAGE_SIZE) { 87 DRM_ERROR("Requested GART mapping for wptr bo larger than one page\n"); 88 return -EINVAL; 89 } 90 91 ret = mes_userq_map_gtt_bo_to_gart(wptr_obj->obj); 92 if (ret) { 93 DRM_ERROR("Failed to map wptr bo to GART\n"); 94 return ret; 95 } 96 97 ret = amdgpu_bo_reserve(wptr_obj->obj, true); 98 if (ret) { 99 DRM_ERROR("Failed to reserve wptr bo\n"); 100 return ret; 101 } 102 103 /* TODO use eviction fence instead of pinning. */ 104 ret = amdgpu_bo_pin(wptr_obj->obj, AMDGPU_GEM_DOMAIN_GTT); 105 if (ret) { 106 drm_file_err(uq_mgr->file, "[Usermode queues] Failed to pin wptr bo\n"); 107 goto unresv_bo; 108 } 109 110 queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(wptr_obj->obj); 111 amdgpu_bo_unreserve(wptr_obj->obj); 112 113 return 0; 114 115 unresv_bo: 116 amdgpu_bo_unreserve(wptr_obj->obj); 117 return ret; 118 119 } 120 121 static int convert_to_mes_priority(int priority) 122 { 123 switch (priority) { 124 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_LOW: 125 default: 126 return AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 127 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_LOW: 128 return AMDGPU_MES_PRIORITY_LEVEL_LOW; 129 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_HIGH: 130 return AMDGPU_MES_PRIORITY_LEVEL_MEDIUM; 131 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_HIGH: 132 return AMDGPU_MES_PRIORITY_LEVEL_HIGH; 133 } 134 } 135 136 static int mes_userq_map(struct amdgpu_usermode_queue *queue) 137 { 138 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 139 struct amdgpu_device *adev = uq_mgr->adev; 140 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 141 struct amdgpu_mqd_prop *userq_props = queue->userq_prop; 142 struct mes_add_queue_input queue_input; 143 int r; 144 145 memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input)); 146 147 queue_input.process_va_start = 0; 148 queue_input.process_va_end = adev->vm_manager.max_pfn - 1; 149 150 /* set process quantum to 10 ms and gang quantum to 1 ms as default */ 151 queue_input.process_quantum = 100000; 152 queue_input.gang_quantum = 10000; 153 queue_input.paging = false; 154 155 queue_input.process_context_addr = ctx->gpu_addr; 156 queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ; 157 queue_input.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 158 queue_input.gang_global_priority_level = convert_to_mes_priority(queue->priority); 159 160 queue_input.process_id = queue->vm->pasid; 161 queue_input.queue_type = queue->queue_type; 162 queue_input.mqd_addr = queue->mqd.gpu_addr; 163 queue_input.wptr_addr = userq_props->wptr_gpu_addr; 164 queue_input.queue_size = userq_props->queue_size >> 2; 165 queue_input.doorbell_offset = userq_props->doorbell_index; 166 queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo); 167 queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr; 168 169 amdgpu_mes_lock(&adev->mes); 170 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input); 171 amdgpu_mes_unlock(&adev->mes); 172 if (r) { 173 DRM_ERROR("Failed to map queue in HW, err (%d)\n", r); 174 return r; 175 } 176 177 DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index); 178 return 0; 179 } 180 181 static int mes_userq_unmap(struct amdgpu_usermode_queue *queue) 182 { 183 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 184 struct amdgpu_device *adev = uq_mgr->adev; 185 struct mes_remove_queue_input queue_input; 186 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 187 int r; 188 189 memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); 190 queue_input.doorbell_offset = queue->doorbell_index; 191 queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ; 192 193 amdgpu_mes_lock(&adev->mes); 194 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); 195 amdgpu_mes_unlock(&adev->mes); 196 if (r) 197 DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r); 198 return r; 199 } 200 201 static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr, 202 struct amdgpu_usermode_queue *queue, 203 struct drm_amdgpu_userq_in *mqd_user) 204 { 205 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 206 int r, size; 207 208 /* 209 * The FW expects at least one page space allocated for 210 * process ctx and gang ctx each. Create an object 211 * for the same. 212 */ 213 size = AMDGPU_USERQ_PROC_CTX_SZ + AMDGPU_USERQ_GANG_CTX_SZ; 214 r = amdgpu_userq_create_object(uq_mgr, ctx, size); 215 if (r) { 216 DRM_ERROR("Failed to allocate ctx space bo for userqueue, err:%d\n", r); 217 return r; 218 } 219 220 return 0; 221 } 222 223 static int mes_userq_detect_and_reset(struct amdgpu_device *adev, 224 int queue_type) 225 { 226 int db_array_size = amdgpu_mes_get_hung_queue_db_array_size(adev); 227 struct mes_detect_and_reset_queue_input input; 228 struct amdgpu_usermode_queue *queue; 229 unsigned int hung_db_num = 0; 230 unsigned long queue_id; 231 u32 db_array[8]; 232 bool found_hung_queue = false; 233 int r, i; 234 235 if (db_array_size > 8) { 236 dev_err(adev->dev, "DB array size (%d vs 8) too small\n", 237 db_array_size); 238 return -EINVAL; 239 } 240 241 memset(&input, 0x0, sizeof(struct mes_detect_and_reset_queue_input)); 242 243 input.queue_type = queue_type; 244 245 amdgpu_mes_lock(&adev->mes); 246 r = amdgpu_mes_detect_and_reset_hung_queues(adev, queue_type, false, 247 &hung_db_num, db_array, 0); 248 amdgpu_mes_unlock(&adev->mes); 249 if (r) { 250 dev_err(adev->dev, "Failed to detect and reset queues, err (%d)\n", r); 251 } else if (hung_db_num) { 252 xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) { 253 if (queue->queue_type == queue_type) { 254 for (i = 0; i < hung_db_num; i++) { 255 if (queue->doorbell_index == db_array[i]) { 256 queue->state = AMDGPU_USERQ_STATE_HUNG; 257 found_hung_queue = true; 258 atomic_inc(&adev->gpu_reset_counter); 259 amdgpu_userq_fence_driver_force_completion(queue); 260 drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, NULL); 261 } 262 } 263 } 264 } 265 } 266 267 if (found_hung_queue) { 268 /* Resume scheduling after hang recovery */ 269 r = amdgpu_mes_resume(adev); 270 } 271 272 return r; 273 } 274 275 static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, 276 struct drm_amdgpu_userq_in *args_in) 277 { 278 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 279 struct amdgpu_device *adev = uq_mgr->adev; 280 struct amdgpu_mqd *mqd_hw_default = &adev->mqds[queue->queue_type]; 281 struct drm_amdgpu_userq_in *mqd_user = args_in; 282 struct amdgpu_mqd_prop *userq_props; 283 int r; 284 285 /* Structure to initialize MQD for userqueue using generic MQD init function */ 286 userq_props = kzalloc_obj(struct amdgpu_mqd_prop); 287 if (!userq_props) { 288 DRM_ERROR("Failed to allocate memory for userq_props\n"); 289 return -ENOMEM; 290 } 291 292 r = amdgpu_userq_create_object(uq_mgr, &queue->mqd, 293 AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size)); 294 if (r) { 295 DRM_ERROR("Failed to create MQD object for userqueue\n"); 296 goto free_props; 297 } 298 299 /* Initialize the MQD BO with user given values */ 300 userq_props->wptr_gpu_addr = mqd_user->wptr_va; 301 userq_props->rptr_gpu_addr = mqd_user->rptr_va; 302 userq_props->queue_size = mqd_user->queue_size; 303 userq_props->hqd_base_gpu_addr = mqd_user->queue_va; 304 userq_props->mqd_gpu_addr = queue->mqd.gpu_addr; 305 userq_props->use_doorbell = true; 306 userq_props->doorbell_index = queue->doorbell_index; 307 userq_props->fence_address = queue->fence_drv->gpu_addr; 308 309 if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) { 310 struct drm_amdgpu_userq_mqd_compute_gfx11 *compute_mqd; 311 312 if (mqd_user->mqd_size != sizeof(*compute_mqd)) { 313 DRM_ERROR("Invalid compute IP MQD size\n"); 314 r = -EINVAL; 315 goto free_mqd; 316 } 317 318 compute_mqd = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); 319 if (IS_ERR(compute_mqd)) { 320 DRM_ERROR("Failed to read user MQD\n"); 321 r = -ENOMEM; 322 goto free_mqd; 323 } 324 325 r = amdgpu_bo_reserve(queue->vm->root.bo, false); 326 if (r) { 327 kfree(compute_mqd); 328 goto free_mqd; 329 } 330 r = amdgpu_userq_input_va_validate(adev, queue, compute_mqd->eop_va, 331 2048); 332 amdgpu_bo_unreserve(queue->vm->root.bo); 333 if (r) { 334 kfree(compute_mqd); 335 goto free_mqd; 336 } 337 338 userq_props->eop_gpu_addr = compute_mqd->eop_va; 339 userq_props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL; 340 userq_props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM; 341 userq_props->hqd_active = false; 342 userq_props->tmz_queue = 343 mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE; 344 kfree(compute_mqd); 345 } else if (queue->queue_type == AMDGPU_HW_IP_GFX) { 346 struct drm_amdgpu_userq_mqd_gfx11 *mqd_gfx_v11; 347 struct amdgpu_gfx_shadow_info shadow_info; 348 349 if (adev->gfx.funcs->get_gfx_shadow_info) { 350 adev->gfx.funcs->get_gfx_shadow_info(adev, &shadow_info, true); 351 } else { 352 r = -EINVAL; 353 goto free_mqd; 354 } 355 356 if (mqd_user->mqd_size != sizeof(*mqd_gfx_v11) || !mqd_user->mqd) { 357 DRM_ERROR("Invalid GFX MQD\n"); 358 r = -EINVAL; 359 goto free_mqd; 360 } 361 362 mqd_gfx_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); 363 if (IS_ERR(mqd_gfx_v11)) { 364 DRM_ERROR("Failed to read user MQD\n"); 365 r = -ENOMEM; 366 goto free_mqd; 367 } 368 369 userq_props->shadow_addr = mqd_gfx_v11->shadow_va; 370 userq_props->csa_addr = mqd_gfx_v11->csa_va; 371 userq_props->tmz_queue = 372 mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE; 373 374 r = amdgpu_bo_reserve(queue->vm->root.bo, false); 375 if (r) { 376 kfree(mqd_gfx_v11); 377 goto free_mqd; 378 } 379 r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->shadow_va, 380 shadow_info.shadow_size); 381 if (r) { 382 amdgpu_bo_unreserve(queue->vm->root.bo); 383 kfree(mqd_gfx_v11); 384 goto free_mqd; 385 } 386 387 r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->csa_va, 388 shadow_info.csa_size); 389 amdgpu_bo_unreserve(queue->vm->root.bo); 390 if (r) { 391 kfree(mqd_gfx_v11); 392 goto free_mqd; 393 } 394 395 kfree(mqd_gfx_v11); 396 } else if (queue->queue_type == AMDGPU_HW_IP_DMA) { 397 struct drm_amdgpu_userq_mqd_sdma_gfx11 *mqd_sdma_v11; 398 399 if (mqd_user->mqd_size != sizeof(*mqd_sdma_v11) || !mqd_user->mqd) { 400 DRM_ERROR("Invalid SDMA MQD\n"); 401 r = -EINVAL; 402 goto free_mqd; 403 } 404 405 mqd_sdma_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); 406 if (IS_ERR(mqd_sdma_v11)) { 407 DRM_ERROR("Failed to read sdma user MQD\n"); 408 r = -ENOMEM; 409 goto free_mqd; 410 } 411 412 r = amdgpu_bo_reserve(queue->vm->root.bo, false); 413 if (r) { 414 kfree(mqd_sdma_v11); 415 goto free_mqd; 416 } 417 r = amdgpu_userq_input_va_validate(adev, queue, mqd_sdma_v11->csa_va, 418 32); 419 amdgpu_bo_unreserve(queue->vm->root.bo); 420 if (r) { 421 kfree(mqd_sdma_v11); 422 goto free_mqd; 423 } 424 425 userq_props->csa_addr = mqd_sdma_v11->csa_va; 426 kfree(mqd_sdma_v11); 427 } 428 429 queue->userq_prop = userq_props; 430 431 r = mqd_hw_default->init_mqd(adev, (void *)queue->mqd.cpu_ptr, userq_props); 432 if (r) { 433 DRM_ERROR("Failed to initialize MQD for userqueue\n"); 434 goto free_mqd; 435 } 436 437 /* Create BO for FW operations */ 438 r = mes_userq_create_ctx_space(uq_mgr, queue, mqd_user); 439 if (r) { 440 DRM_ERROR("Failed to allocate BO for userqueue (%d)", r); 441 goto free_mqd; 442 } 443 444 /* FW expects WPTR BOs to be mapped into GART */ 445 r = mes_userq_create_wptr_mapping(adev, uq_mgr, queue, userq_props->wptr_gpu_addr); 446 if (r) { 447 DRM_ERROR("Failed to create WPTR mapping\n"); 448 goto free_ctx; 449 } 450 451 return 0; 452 453 free_ctx: 454 amdgpu_userq_destroy_object(uq_mgr, &queue->fw_obj); 455 456 free_mqd: 457 amdgpu_userq_destroy_object(uq_mgr, &queue->mqd); 458 459 free_props: 460 kfree(userq_props); 461 462 return r; 463 } 464 465 static void mes_userq_mqd_destroy(struct amdgpu_usermode_queue *queue) 466 { 467 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 468 469 amdgpu_userq_destroy_object(uq_mgr, &queue->fw_obj); 470 kfree(queue->userq_prop); 471 amdgpu_userq_destroy_object(uq_mgr, &queue->mqd); 472 } 473 474 static int mes_userq_preempt(struct amdgpu_usermode_queue *queue) 475 { 476 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 477 struct amdgpu_device *adev = uq_mgr->adev; 478 struct mes_suspend_gang_input queue_input; 479 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 480 signed long timeout = 2100000; /* 2100 ms */ 481 u64 fence_gpu_addr; 482 u32 fence_offset; 483 u64 *fence_ptr; 484 int i, r; 485 486 if (queue->state != AMDGPU_USERQ_STATE_MAPPED) 487 return 0; 488 r = amdgpu_device_wb_get(adev, &fence_offset); 489 if (r) 490 return r; 491 492 fence_gpu_addr = adev->wb.gpu_addr + (fence_offset * 4); 493 fence_ptr = (u64 *)&adev->wb.wb[fence_offset]; 494 *fence_ptr = 0; 495 496 memset(&queue_input, 0x0, sizeof(struct mes_suspend_gang_input)); 497 queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ; 498 queue_input.suspend_fence_addr = fence_gpu_addr; 499 queue_input.suspend_fence_value = 1; 500 amdgpu_mes_lock(&adev->mes); 501 r = adev->mes.funcs->suspend_gang(&adev->mes, &queue_input); 502 amdgpu_mes_unlock(&adev->mes); 503 if (r) { 504 DRM_ERROR("Failed to suspend gang: %d\n", r); 505 goto out; 506 } 507 508 for (i = 0; i < timeout; i++) { 509 if (*fence_ptr == 1) 510 goto out; 511 udelay(1); 512 } 513 r = -ETIMEDOUT; 514 515 out: 516 amdgpu_device_wb_free(adev, fence_offset); 517 return r; 518 } 519 520 static int mes_userq_restore(struct amdgpu_usermode_queue *queue) 521 { 522 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 523 struct amdgpu_device *adev = uq_mgr->adev; 524 struct mes_resume_gang_input queue_input; 525 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 526 int r; 527 528 if (queue->state == AMDGPU_USERQ_STATE_HUNG) 529 return -EINVAL; 530 if (queue->state != AMDGPU_USERQ_STATE_PREEMPTED) 531 return 0; 532 533 memset(&queue_input, 0x0, sizeof(struct mes_resume_gang_input)); 534 queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ; 535 536 amdgpu_mes_lock(&adev->mes); 537 r = adev->mes.funcs->resume_gang(&adev->mes, &queue_input); 538 amdgpu_mes_unlock(&adev->mes); 539 if (r) 540 dev_err(adev->dev, "Failed to resume queue, err (%d)\n", r); 541 return r; 542 } 543 544 const struct amdgpu_userq_funcs userq_mes_funcs = { 545 .mqd_create = mes_userq_mqd_create, 546 .mqd_destroy = mes_userq_mqd_destroy, 547 .unmap = mes_userq_unmap, 548 .map = mes_userq_map, 549 .detect_and_reset = mes_userq_detect_and_reset, 550 .preempt = mes_userq_preempt, 551 .restore = mes_userq_restore, 552 }; 553