1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2024 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 #include <drm/drm_drv.h> 25 #include "amdgpu.h" 26 #include "amdgpu_gfx.h" 27 #include "mes_userqueue.h" 28 #include "amdgpu_userq_fence.h" 29 #include "amdgpu_trace.h" 30 31 #define AMDGPU_USERQ_PROC_CTX_SZ PAGE_SIZE 32 #define AMDGPU_USERQ_GANG_CTX_SZ PAGE_SIZE 33 34 static int 35 mes_userq_create_wptr_mapping(struct amdgpu_device *adev, 36 struct amdgpu_userq_mgr *uq_mgr, 37 struct amdgpu_usermode_queue *queue, 38 uint64_t wptr) 39 { 40 struct amdgpu_bo_va_mapping *wptr_mapping; 41 struct amdgpu_userq_obj *wptr_obj = &queue->wptr_obj; 42 struct amdgpu_bo *obj; 43 struct amdgpu_vm *vm = queue->vm; 44 struct drm_exec exec; 45 int ret; 46 47 wptr &= AMDGPU_GMC_HOLE_MASK; 48 49 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 2); 50 drm_exec_until_all_locked(&exec) { 51 ret = amdgpu_vm_lock_pd(vm, &exec, 1); 52 drm_exec_retry_on_contention(&exec); 53 if (unlikely(ret)) 54 goto fail_lock; 55 56 wptr_mapping = amdgpu_vm_bo_lookup_mapping(vm, wptr); 57 if (!wptr_mapping || !wptr_mapping->bo_va->base.bo) { 58 ret = -EINVAL; 59 goto fail_lock; 60 } 61 62 obj = wptr_mapping->bo_va->base.bo; 63 ret = drm_exec_prepare_obj(&exec, &obj->tbo.base, 64 TTM_NUM_MOVE_FENCES + 1); 65 drm_exec_retry_on_contention(&exec); 66 if (unlikely(ret)) 67 goto fail_lock; 68 } 69 70 wptr_obj->obj = amdgpu_bo_ref(wptr_mapping->bo_va->base.bo); 71 if (wptr_obj->obj->tbo.base.size > PAGE_SIZE) { 72 ret = -EINVAL; 73 goto fail_map; 74 } 75 /* Keep WPTR BO under eviction-fence control instead of pinning. */ 76 ret = amdgpu_evf_mgr_attach_fence(&uq_mgr_to_fpriv(uq_mgr)->evf_mgr, wptr_obj->obj); 77 if (ret) { 78 DRM_ERROR("Failed to attach eviction fence to wptr bo. ret %d\n", ret); 79 goto fail_map; 80 } 81 82 ret = amdgpu_ttm_alloc_gart(&wptr_obj->obj->tbo); 83 if (ret) { 84 DRM_ERROR("Failed to bind wptr bo to GART. ret %d\n", ret); 85 goto fail_map; 86 } 87 88 queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(wptr_obj->obj); 89 90 drm_exec_fini(&exec); 91 return 0; 92 fail_map: 93 amdgpu_bo_unref(&wptr_obj->obj); 94 fail_lock: 95 drm_exec_fini(&exec); 96 return ret; 97 98 } 99 100 static int convert_to_mes_priority(int priority) 101 { 102 switch (priority) { 103 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_LOW: 104 default: 105 return AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 106 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_LOW: 107 return AMDGPU_MES_PRIORITY_LEVEL_LOW; 108 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_HIGH: 109 return AMDGPU_MES_PRIORITY_LEVEL_MEDIUM; 110 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_HIGH: 111 return AMDGPU_MES_PRIORITY_LEVEL_HIGH; 112 } 113 } 114 115 static int mes_userq_map(struct amdgpu_usermode_queue *queue) 116 { 117 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 118 struct amdgpu_device *adev = uq_mgr->adev; 119 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 120 struct amdgpu_mqd_prop *userq_props = queue->userq_prop; 121 struct mes_add_queue_input queue_input; 122 struct amdgpu_mes *mes = &adev->mes; 123 int r; 124 125 memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input)); 126 127 queue_input.process_va_start = 0; 128 queue_input.process_va_end = adev->vm_manager.max_pfn - 1; 129 130 /* set process quantum to 10 ms and gang quantum to 1 ms as default */ 131 queue_input.process_quantum = 100000; 132 queue_input.gang_quantum = 10000; 133 queue_input.paging = false; 134 135 queue_input.process_context_addr = uq_mgr->proc_ctx_obj.gpu_addr; 136 queue_input.gang_context_addr = ctx->gpu_addr; 137 queue_input.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 138 queue_input.gang_global_priority_level = convert_to_mes_priority(queue->priority); 139 140 queue_input.process_id = queue->vm->pasid; 141 queue_input.queue_type = queue->queue_type; 142 queue_input.mqd_addr = queue->mqd.gpu_addr; 143 queue_input.wptr_addr = userq_props->wptr_gpu_addr; 144 queue_input.queue_size = userq_props->queue_size >> 2; 145 queue_input.doorbell_offset = userq_props->doorbell_index; 146 queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo); 147 queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr; 148 149 if (mes->use_rs64mem) { 150 if (!uq_mgr->proc_ctx_allocated) { 151 r = amdgpu_mes_alloc_proc_ctx_index(mes, &uq_mgr->proc_ctx_array_index); 152 if (r) { 153 DRM_ERROR("Failed to allocate userq process index err:%d\n", r); 154 return r; 155 } 156 uq_mgr->proc_ctx_allocated = true; 157 } 158 159 r = amdgpu_mes_alloc_gang_ctx_index(mes, &queue->gang_ctx_array_index); 160 if (r) { 161 DRM_ERROR("Failed to allocate userq gang index err:%d\n", r); 162 return r; 163 } 164 queue_input.process_context_array_index = uq_mgr->proc_ctx_array_index; 165 queue_input.gang_context_array_index = queue->gang_ctx_array_index; 166 } 167 amdgpu_mes_lock(&adev->mes); 168 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input); 169 amdgpu_mes_unlock(&adev->mes); 170 if (r) { 171 DRM_ERROR("Failed to map queue in HW, err (%d)\n", r); 172 return r; 173 } 174 175 amdgpu_mes_userq_queue_mapped(adev); 176 177 DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index); 178 return 0; 179 } 180 181 static int mes_userq_unmap(struct amdgpu_usermode_queue *queue) 182 { 183 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 184 struct amdgpu_device *adev = uq_mgr->adev; 185 struct mes_remove_queue_input queue_input; 186 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 187 struct amdgpu_mes *mes = &adev->mes; 188 int r; 189 190 memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); 191 queue_input.gang_context_array_index = queue->gang_ctx_array_index; 192 queue_input.doorbell_offset = queue->doorbell_index; 193 queue_input.gang_context_addr = ctx->gpu_addr; 194 queue_input.queue_type = queue->queue_type; 195 196 amdgpu_mes_lock(&adev->mes); 197 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); 198 amdgpu_mes_unlock(&adev->mes); 199 if (mes->use_rs64mem) 200 amdgpu_mes_free_gang_ctx_index(mes, queue->gang_ctx_array_index); 201 if (r) { 202 DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r); 203 return r; 204 } 205 206 amdgpu_mes_userq_queue_unmapped(adev); 207 return 0; 208 } 209 210 int mes_userq_reset_hw(struct amdgpu_usermode_queue *queue, 211 struct mes_reset_queue_input *input) 212 { 213 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 214 struct amdgpu_device *adev = uq_mgr->adev; 215 int r; 216 217 amdgpu_mes_lock(&adev->mes); 218 r = adev->mes.funcs->reset_hw_queue(&adev->mes, input); 219 amdgpu_mes_unlock(&adev->mes); 220 if (r) 221 return r; 222 223 r = mes_userq_unmap(queue); 224 if (r) 225 return r; 226 227 /* Prevent the restore worker from resubmitting the guilty job. */ 228 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG); 229 queue->state = AMDGPU_USERQ_STATE_HUNG; 230 return 0; 231 } 232 233 int mes_userq_reset(struct amdgpu_usermode_queue *queue) 234 { 235 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 236 struct amdgpu_device *adev = uq_mgr->adev; 237 struct mes_reset_queue_input queue_input; 238 239 /* already reset by an earlier job's hang-detect; just signal and bail */ 240 if (queue->state == AMDGPU_USERQ_STATE_HUNG) 241 return 0; 242 243 /* A GFX pipe reset also affects the kernel rings on that pipe. */ 244 if (queue->queue_type == AMDGPU_HW_IP_GFX && 245 adev->gfx.funcs->detect_hung_queue && 246 amdgpu_gfx_me_pipe_reset_supported(adev)) 247 return amdgpu_gfx_reset_mes_gfx(adev, NULL, NULL, queue, 0, true); 248 249 memset(&queue_input, 0x0, sizeof(struct mes_reset_queue_input)); 250 queue_input.doorbell_offset = queue->doorbell_index; 251 queue_input.queue_type = queue->queue_type; 252 253 /* 254 * The MES packet reset fails once the hung queue wedges MES. For SDMA the 255 * queue is still on its HW slot, so reset it over MMIO instead: recover 256 * the (instance, queue_id) slot from the doorbell. 257 */ 258 if (queue->queue_type == AMDGPU_HW_IP_DMA && 259 adev->sdma.instance[0].funcs && 260 adev->sdma.instance[0].funcs->detect_hung_queue) { 261 u32 instance, hw_queue_id; 262 263 if (adev->sdma.instance[0].funcs->detect_hung_queue(adev, 264 queue->doorbell_index, &instance, &hw_queue_id)) { 265 queue_input.use_mmio = true; 266 queue_input.me_id = instance; 267 queue_input.queue_id = hw_queue_id; 268 } else { 269 dev_warn(adev->dev, 270 "SDMA userq (doorbell %llu) not on any HW slot; falling back to MES reset\n", 271 queue->doorbell_index); 272 } 273 } 274 275 return mes_userq_reset_hw(queue, &queue_input); 276 } 277 278 int mes_userq_reset_queue(struct amdgpu_device *adev, 279 struct amdgpu_usermode_queue *guilty_uq, 280 int queue_type, 281 unsigned int pipe, 282 unsigned int queue, 283 unsigned int db) 284 { 285 struct amdgpu_usermode_queue *uq; 286 bool use_mmio = adev->gfx.mec.use_mmio_for_reset; 287 unsigned long uq_id; 288 int r; 289 290 xa_for_each(&adev->userq_doorbell_xa, uq_id, uq) { 291 if (uq->queue_type == queue_type) { 292 if (uq == guilty_uq) 293 continue; 294 if (uq->doorbell_index == db) { 295 uq->state = AMDGPU_USERQ_STATE_HUNG; 296 if (use_mmio) 297 r = amdgpu_mes_reset_queue_mmio(adev, queue_type, 0, 1, pipe, queue, 0); 298 else 299 r = amdgpu_mes_reset_user_queue(adev, queue_type, db, 0); 300 if (r) 301 return r; 302 r = mes_userq_unmap(uq); 303 if (r) 304 return r; 305 amdgpu_userq_fence_driver_force_completion(uq); 306 break; 307 } 308 } 309 } 310 return 0; 311 } 312 313 static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr, 314 struct amdgpu_usermode_queue *queue, 315 struct drm_amdgpu_userq_in *mqd_user) 316 { 317 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 318 int r, size; 319 320 /* The FW expects at least one page space allocated for gang ctx. */ 321 size = AMDGPU_USERQ_GANG_CTX_SZ; 322 r = amdgpu_bo_create_kernel(uq_mgr->adev, size, 0, 323 AMDGPU_GEM_DOMAIN_GTT, 324 &ctx->obj, &ctx->gpu_addr, 325 &ctx->cpu_ptr); 326 if (r) { 327 DRM_ERROR("Failed to allocate ctx space bo for userqueue, err:%d\n", r); 328 return r; 329 } 330 331 memset(ctx->cpu_ptr, 0, size); 332 return 0; 333 } 334 335 static int mes_userq_create_proc_ctx_space(struct amdgpu_userq_mgr *uq_mgr) 336 { 337 int r = 0; 338 339 mutex_lock(&uq_mgr->proc_ctx_lock); 340 /* This check is a necessary because amdgpu_bo_create_kernel() 341 * calls helpers like amdgpu_bo_pin() and memset() unconditionally 342 */ 343 if (!uq_mgr->proc_ctx_obj.obj) { 344 r = amdgpu_bo_create_kernel(uq_mgr->adev, AMDGPU_USERQ_PROC_CTX_SZ, 345 0, AMDGPU_GEM_DOMAIN_GTT, 346 &uq_mgr->proc_ctx_obj.obj, 347 &uq_mgr->proc_ctx_obj.gpu_addr, 348 &uq_mgr->proc_ctx_obj.cpu_ptr); 349 350 if (!r) 351 memset(uq_mgr->proc_ctx_obj.cpu_ptr, 0, AMDGPU_USERQ_PROC_CTX_SZ); 352 } 353 354 mutex_unlock(&uq_mgr->proc_ctx_lock); 355 356 return r; 357 } 358 359 static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, 360 struct drm_amdgpu_userq_in *args_in) 361 { 362 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 363 struct amdgpu_device *adev = uq_mgr->adev; 364 struct amdgpu_mqd *mqd_hw_default = &adev->mqds[queue->queue_type]; 365 struct drm_amdgpu_userq_in *mqd_user = args_in; 366 struct amdgpu_mqd_prop *userq_props; 367 int r; 368 369 /* Structure to initialize MQD for userqueue using generic MQD init function */ 370 userq_props = kzalloc_obj(struct amdgpu_mqd_prop); 371 if (!userq_props) { 372 DRM_ERROR("Failed to allocate memory for userq_props\n"); 373 return -ENOMEM; 374 } 375 376 r = amdgpu_bo_create_kernel(adev, 377 AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size), 378 0, AMDGPU_GEM_DOMAIN_GTT, 379 &queue->mqd.obj, &queue->mqd.gpu_addr, 380 &queue->mqd.cpu_ptr); 381 if (r) { 382 DRM_ERROR("Failed to create MQD object for userqueue\n"); 383 goto free_props; 384 } 385 386 memset(queue->mqd.cpu_ptr, 0, 387 AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size)); 388 389 /* Initialize the MQD BO with user given values */ 390 userq_props->wptr_gpu_addr = mqd_user->wptr_va; 391 userq_props->rptr_gpu_addr = mqd_user->rptr_va; 392 userq_props->queue_size = mqd_user->queue_size; 393 userq_props->hqd_base_gpu_addr = mqd_user->queue_va; 394 userq_props->mqd_gpu_addr = queue->mqd.gpu_addr; 395 userq_props->use_doorbell = true; 396 userq_props->doorbell_index = queue->doorbell_index; 397 userq_props->fence_address = queue->fence_drv->gpu_addr; 398 399 if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) { 400 struct drm_amdgpu_userq_mqd_compute_gfx11 *compute_mqd; 401 402 if (mqd_user->mqd_size != sizeof(*compute_mqd)) { 403 DRM_ERROR("Invalid compute IP MQD size\n"); 404 r = -EINVAL; 405 goto free_mqd; 406 } 407 408 compute_mqd = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); 409 if (IS_ERR(compute_mqd)) { 410 r = PTR_ERR(compute_mqd); 411 goto free_mqd; 412 } 413 414 r = amdgpu_bo_reserve(queue->vm->root.bo, false); 415 if (r) { 416 kfree(compute_mqd); 417 goto free_mqd; 418 } 419 r = amdgpu_userq_input_va_validate(adev, queue, 420 compute_mqd->eop_va, 2048, 421 &queue->userq_vas.va.eop); 422 amdgpu_bo_unreserve(queue->vm->root.bo); 423 if (r) { 424 kfree(compute_mqd); 425 goto free_mqd; 426 } 427 428 userq_props->eop_gpu_addr = compute_mqd->eop_va; 429 userq_props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL; 430 userq_props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM; 431 userq_props->hqd_active = false; 432 userq_props->tmz_queue = 433 mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE; 434 kfree(compute_mqd); 435 } else if (queue->queue_type == AMDGPU_HW_IP_GFX) { 436 struct drm_amdgpu_userq_mqd_gfx11 *mqd_gfx_v11; 437 struct amdgpu_gfx_shadow_info shadow_info; 438 439 if (adev->gfx.funcs->get_gfx_shadow_info) { 440 adev->gfx.funcs->get_gfx_shadow_info(adev, &shadow_info, true); 441 } else { 442 r = -EINVAL; 443 goto free_mqd; 444 } 445 446 if (mqd_user->mqd_size != sizeof(*mqd_gfx_v11) || !mqd_user->mqd) { 447 DRM_ERROR("Invalid GFX MQD\n"); 448 r = -EINVAL; 449 goto free_mqd; 450 } 451 452 mqd_gfx_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); 453 if (IS_ERR(mqd_gfx_v11)) { 454 r = PTR_ERR(mqd_gfx_v11); 455 goto free_mqd; 456 } 457 458 userq_props->shadow_addr = mqd_gfx_v11->shadow_va; 459 userq_props->csa_addr = mqd_gfx_v11->csa_va; 460 userq_props->tmz_queue = 461 mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE; 462 463 r = amdgpu_bo_reserve(queue->vm->root.bo, false); 464 if (r) { 465 kfree(mqd_gfx_v11); 466 goto free_mqd; 467 } 468 r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->shadow_va, 469 shadow_info.shadow_size, 470 &queue->userq_vas.va.shadow); 471 if (r) { 472 amdgpu_bo_unreserve(queue->vm->root.bo); 473 kfree(mqd_gfx_v11); 474 goto free_mqd; 475 } 476 477 r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->csa_va, 478 shadow_info.csa_size, 479 &queue->userq_vas.va.csa); 480 amdgpu_bo_unreserve(queue->vm->root.bo); 481 if (r) { 482 kfree(mqd_gfx_v11); 483 goto free_mqd; 484 } 485 486 kfree(mqd_gfx_v11); 487 } else if (queue->queue_type == AMDGPU_HW_IP_DMA) { 488 struct drm_amdgpu_userq_mqd_sdma_gfx11 *mqd_sdma_v11; 489 490 if (mqd_user->mqd_size != sizeof(*mqd_sdma_v11) || !mqd_user->mqd) { 491 DRM_ERROR("Invalid SDMA MQD\n"); 492 r = -EINVAL; 493 goto free_mqd; 494 } 495 496 mqd_sdma_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); 497 if (IS_ERR(mqd_sdma_v11)) { 498 r = PTR_ERR(mqd_sdma_v11); 499 goto free_mqd; 500 } 501 502 r = amdgpu_bo_reserve(queue->vm->root.bo, false); 503 if (r) { 504 kfree(mqd_sdma_v11); 505 goto free_mqd; 506 } 507 r = amdgpu_userq_input_va_validate(adev, queue, mqd_sdma_v11->csa_va, 508 32, 509 &queue->userq_vas.va.csa); 510 amdgpu_bo_unreserve(queue->vm->root.bo); 511 if (r) { 512 kfree(mqd_sdma_v11); 513 goto free_mqd; 514 } 515 516 userq_props->csa_addr = mqd_sdma_v11->csa_va; 517 kfree(mqd_sdma_v11); 518 } 519 520 queue->userq_prop = userq_props; 521 522 r = mqd_hw_default->init_mqd(adev, (void *)queue->mqd.cpu_ptr, userq_props); 523 if (r) { 524 DRM_ERROR("Failed to initialize MQD for userqueue\n"); 525 goto free_mqd; 526 } 527 528 /* Create per-process MES process context BO */ 529 r = mes_userq_create_proc_ctx_space(uq_mgr); 530 if (r) { 531 DRM_ERROR("Failed to allocate MES process context space bo, error: %d\n", r); 532 goto free_mqd; 533 } 534 535 /* Create BO of a gang for FW operations */ 536 r = mes_userq_create_ctx_space(uq_mgr, queue, mqd_user); 537 if (r) { 538 DRM_ERROR("Failed to allocate BO for userqueue (%d)", r); 539 goto free_mqd; 540 } 541 542 /* FW expects WPTR BOs to be mapped into GART */ 543 r = mes_userq_create_wptr_mapping(adev, uq_mgr, queue, userq_props->wptr_gpu_addr); 544 if (r) { 545 DRM_ERROR("Failed to create WPTR mapping\n"); 546 goto free_ctx; 547 } 548 549 return 0; 550 551 free_ctx: 552 amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr, 553 &queue->fw_obj.cpu_ptr); 554 555 free_mqd: 556 amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr, 557 &queue->mqd.cpu_ptr); 558 559 free_props: 560 kfree(userq_props); 561 562 return r; 563 } 564 565 static void mes_userq_mqd_destroy(struct amdgpu_usermode_queue *queue) 566 { 567 568 amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr, 569 &queue->fw_obj.cpu_ptr); 570 kfree(queue->userq_prop); 571 amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr, 572 &queue->mqd.cpu_ptr); 573 574 amdgpu_bo_unref(&queue->wptr_obj.obj); 575 } 576 577 static int mes_userq_preempt(struct amdgpu_usermode_queue *queue) 578 { 579 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 580 struct amdgpu_device *adev = uq_mgr->adev; 581 struct mes_suspend_gang_input queue_input; 582 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 583 signed long timeout = 2100000; /* 2100 ms */ 584 u64 fence_gpu_addr; 585 u32 fence_offset; 586 u64 *fence_ptr; 587 int i, r; 588 589 if (queue->state != AMDGPU_USERQ_STATE_MAPPED) 590 return 0; 591 r = amdgpu_wb_get(adev, &fence_offset); 592 if (r) 593 return r; 594 595 fence_gpu_addr = adev->wb.gpu_addr + (fence_offset * 4); 596 fence_ptr = (u64 *)&adev->wb.wb[fence_offset]; 597 *fence_ptr = 0; 598 599 memset(&queue_input, 0x0, sizeof(struct mes_suspend_gang_input)); 600 queue_input.gang_context_addr = ctx->gpu_addr; 601 queue_input.suspend_fence_addr = fence_gpu_addr; 602 queue_input.suspend_fence_value = 1; 603 amdgpu_mes_lock(&adev->mes); 604 r = adev->mes.funcs->suspend_gang(&adev->mes, &queue_input); 605 amdgpu_mes_unlock(&adev->mes); 606 if (r) { 607 DRM_ERROR("Failed to suspend gang: %d\n", r); 608 goto out; 609 } 610 611 for (i = 0; i < timeout; i++) { 612 if (*fence_ptr == 1) 613 goto out; 614 udelay(1); 615 } 616 r = -ETIMEDOUT; 617 618 out: 619 amdgpu_wb_free(adev, fence_offset); 620 return r; 621 } 622 623 static int mes_userq_restore(struct amdgpu_usermode_queue *queue) 624 { 625 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 626 struct amdgpu_device *adev = uq_mgr->adev; 627 struct mes_resume_gang_input queue_input; 628 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 629 int r; 630 631 if (queue->state == AMDGPU_USERQ_STATE_HUNG) 632 return -EINVAL; 633 if (queue->state != AMDGPU_USERQ_STATE_PREEMPTED) 634 return 0; 635 636 memset(&queue_input, 0x0, sizeof(struct mes_resume_gang_input)); 637 queue_input.gang_context_addr = ctx->gpu_addr; 638 639 amdgpu_mes_lock(&adev->mes); 640 r = adev->mes.funcs->resume_gang(&adev->mes, &queue_input); 641 amdgpu_mes_unlock(&adev->mes); 642 if (r) 643 dev_err(adev->dev, "Failed to resume queue, err (%d)\n", r); 644 return r; 645 } 646 647 const struct amdgpu_userq_funcs userq_mes_funcs = { 648 .mqd_create = mes_userq_mqd_create, 649 .mqd_destroy = mes_userq_mqd_destroy, 650 .unmap = mes_userq_unmap, 651 .map = mes_userq_map, 652 .preempt = mes_userq_preempt, 653 .restore = mes_userq_restore, 654 .reset = mes_userq_reset, 655 }; 656