1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2024 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 #include <drm/drm_drv.h> 25 #include "amdgpu.h" 26 #include "amdgpu_gfx.h" 27 #include "mes_userqueue.h" 28 #include "amdgpu_userq_fence.h" 29 30 #define AMDGPU_USERQ_PROC_CTX_SZ PAGE_SIZE 31 #define AMDGPU_USERQ_GANG_CTX_SZ PAGE_SIZE 32 33 static int 34 mes_userq_create_wptr_mapping(struct amdgpu_device *adev, 35 struct amdgpu_userq_mgr *uq_mgr, 36 struct amdgpu_usermode_queue *queue, 37 uint64_t wptr) 38 { 39 struct amdgpu_bo_va_mapping *wptr_mapping; 40 struct amdgpu_userq_obj *wptr_obj = &queue->wptr_obj; 41 struct amdgpu_bo *obj; 42 struct amdgpu_vm *vm = queue->vm; 43 struct drm_exec exec; 44 int ret; 45 46 wptr &= AMDGPU_GMC_HOLE_MASK; 47 48 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 2); 49 drm_exec_until_all_locked(&exec) { 50 ret = amdgpu_vm_lock_pd(vm, &exec, 1); 51 drm_exec_retry_on_contention(&exec); 52 if (unlikely(ret)) 53 goto fail_lock; 54 55 wptr_mapping = amdgpu_vm_bo_lookup_mapping(vm, wptr >> PAGE_SHIFT); 56 if (!wptr_mapping) { 57 ret = -EINVAL; 58 goto fail_lock; 59 } 60 61 obj = wptr_mapping->bo_va->base.bo; 62 ret = drm_exec_lock_obj(&exec, &obj->tbo.base); 63 drm_exec_retry_on_contention(&exec); 64 if (unlikely(ret)) 65 goto fail_lock; 66 } 67 68 wptr_obj->obj = amdgpu_bo_ref(wptr_mapping->bo_va->base.bo); 69 if (wptr_obj->obj->tbo.base.size > PAGE_SIZE) { 70 ret = -EINVAL; 71 goto fail_map; 72 } 73 /* Keep WPTR BO under eviction-fence control instead of pinning. */ 74 ret = amdgpu_evf_mgr_attach_fence(&uq_mgr_to_fpriv(uq_mgr)->evf_mgr, wptr_obj->obj); 75 if (ret) { 76 DRM_ERROR("Failed to attach eviction fence to wptr bo. ret %d\n", ret); 77 goto fail_map; 78 } 79 80 ret = amdgpu_ttm_alloc_gart(&wptr_obj->obj->tbo); 81 if (ret) { 82 DRM_ERROR("Failed to bind wptr bo to GART. ret %d\n", ret); 83 goto fail_map; 84 } 85 86 queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(wptr_obj->obj); 87 88 drm_exec_fini(&exec); 89 return 0; 90 fail_map: 91 amdgpu_bo_unref(&wptr_obj->obj); 92 fail_lock: 93 drm_exec_fini(&exec); 94 return ret; 95 96 } 97 98 static int convert_to_mes_priority(int priority) 99 { 100 switch (priority) { 101 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_LOW: 102 default: 103 return AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 104 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_LOW: 105 return AMDGPU_MES_PRIORITY_LEVEL_LOW; 106 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_HIGH: 107 return AMDGPU_MES_PRIORITY_LEVEL_MEDIUM; 108 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_HIGH: 109 return AMDGPU_MES_PRIORITY_LEVEL_HIGH; 110 } 111 } 112 113 static int mes_userq_map(struct amdgpu_usermode_queue *queue) 114 { 115 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 116 struct amdgpu_device *adev = uq_mgr->adev; 117 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 118 struct amdgpu_mqd_prop *userq_props = queue->userq_prop; 119 struct mes_add_queue_input queue_input; 120 struct amdgpu_mes *mes = &adev->mes; 121 int r; 122 123 memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input)); 124 125 queue_input.process_va_start = 0; 126 queue_input.process_va_end = adev->vm_manager.max_pfn - 1; 127 128 /* set process quantum to 10 ms and gang quantum to 1 ms as default */ 129 queue_input.process_quantum = 100000; 130 queue_input.gang_quantum = 10000; 131 queue_input.paging = false; 132 133 queue_input.process_context_addr = uq_mgr->proc_ctx_obj.gpu_addr; 134 queue_input.gang_context_addr = ctx->gpu_addr; 135 queue_input.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 136 queue_input.gang_global_priority_level = convert_to_mes_priority(queue->priority); 137 138 queue_input.process_id = queue->vm->pasid; 139 queue_input.queue_type = queue->queue_type; 140 queue_input.mqd_addr = queue->mqd.gpu_addr; 141 queue_input.wptr_addr = userq_props->wptr_gpu_addr; 142 queue_input.queue_size = userq_props->queue_size >> 2; 143 queue_input.doorbell_offset = userq_props->doorbell_index; 144 queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo); 145 queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr; 146 if (mes->use_rs64mem) { 147 amdgpu_mes_alloc_proc_ctx_index(mes, queue); 148 queue_input.process_context_array_index = queue->proc_ctx_array_index; 149 amdgpu_mes_alloc_gang_ctx_index(mes, queue); 150 queue_input.gang_context_array_index = queue->gang_ctx_array_index; 151 } 152 amdgpu_mes_lock(&adev->mes); 153 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input); 154 amdgpu_mes_unlock(&adev->mes); 155 if (r) { 156 DRM_ERROR("Failed to map queue in HW, err (%d)\n", r); 157 return r; 158 } 159 160 DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index); 161 return 0; 162 } 163 164 static int mes_userq_unmap(struct amdgpu_usermode_queue *queue) 165 { 166 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 167 struct amdgpu_device *adev = uq_mgr->adev; 168 struct mes_remove_queue_input queue_input; 169 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 170 struct amdgpu_mes *mes = &adev->mes; 171 int r; 172 173 memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); 174 queue_input.gang_context_array_index = queue->gang_ctx_array_index; 175 queue_input.doorbell_offset = queue->doorbell_index; 176 queue_input.gang_context_addr = ctx->gpu_addr; 177 queue_input.queue_type = queue->queue_type; 178 179 amdgpu_mes_lock(&adev->mes); 180 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); 181 amdgpu_mes_unlock(&adev->mes); 182 if (mes->use_rs64mem) { 183 amdgpu_mes_free_proc_ctx_index(mes, queue); 184 amdgpu_mes_free_gang_ctx_index(mes, queue); 185 } 186 if (r) 187 DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r); 188 return r; 189 } 190 191 int mes_userq_reset(struct amdgpu_usermode_queue *queue) 192 { 193 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 194 struct amdgpu_device *adev = uq_mgr->adev; 195 struct mes_reset_queue_input queue_input; 196 int r; 197 198 /* XXX: add a FW version check for SDMA per queue reset */ 199 memset(&queue_input, 0x0, sizeof(struct mes_reset_queue_input)); 200 queue_input.doorbell_offset = queue->doorbell_index; 201 queue_input.queue_type = queue->queue_type; 202 203 amdgpu_mes_lock(&adev->mes); 204 r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); 205 amdgpu_mes_unlock(&adev->mes); 206 if (r) 207 return r; 208 return mes_userq_unmap(queue); 209 } 210 211 int mes_userq_reset_queue(struct amdgpu_device *adev, 212 struct amdgpu_usermode_queue *guilty_uq, 213 int queue_type, 214 unsigned int pipe, 215 unsigned int queue, 216 unsigned int db) 217 { 218 struct amdgpu_usermode_queue *uq; 219 bool use_mmio = adev->gfx.mec.use_mmio_for_reset; 220 unsigned long uq_id; 221 int r; 222 223 xa_for_each(&adev->userq_doorbell_xa, uq_id, uq) { 224 if (uq->queue_type == queue_type) { 225 if (uq == guilty_uq) 226 continue; 227 if (uq->doorbell_index == db) { 228 uq->state = AMDGPU_USERQ_STATE_HUNG; 229 if (use_mmio) 230 r = amdgpu_mes_reset_queue_mmio(adev, queue_type, 0, 1, pipe, queue, 0); 231 else 232 r = amdgpu_mes_reset_user_queue(adev, queue_type, db, 0); 233 if (r) 234 return r; 235 r = mes_userq_unmap(uq); 236 if (r) 237 return r; 238 amdgpu_userq_fence_driver_force_completion(uq); 239 break; 240 } 241 } 242 } 243 return 0; 244 } 245 246 static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr, 247 struct amdgpu_usermode_queue *queue, 248 struct drm_amdgpu_userq_in *mqd_user) 249 { 250 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 251 int r, size; 252 253 /* The FW expects at least one page space allocated for gang ctx. */ 254 size = AMDGPU_USERQ_GANG_CTX_SZ; 255 r = amdgpu_bo_create_kernel(uq_mgr->adev, size, 0, 256 AMDGPU_GEM_DOMAIN_GTT, 257 &ctx->obj, &ctx->gpu_addr, 258 &ctx->cpu_ptr); 259 if (r) { 260 DRM_ERROR("Failed to allocate ctx space bo for userqueue, err:%d\n", r); 261 return r; 262 } 263 264 memset(ctx->cpu_ptr, 0, size); 265 return 0; 266 } 267 268 static int mes_userq_create_proc_ctx_space(struct amdgpu_userq_mgr *uq_mgr) 269 { 270 int r = 0; 271 272 mutex_lock(&uq_mgr->proc_ctx_lock); 273 /* This check is a necessary because amdgpu_bo_create_kernel() 274 * calls helpers like amdgpu_bo_pin() and memset() unconditionally 275 */ 276 if (!uq_mgr->proc_ctx_obj.obj) { 277 r = amdgpu_bo_create_kernel(uq_mgr->adev, AMDGPU_USERQ_PROC_CTX_SZ, 278 0, AMDGPU_GEM_DOMAIN_GTT, 279 &uq_mgr->proc_ctx_obj.obj, 280 &uq_mgr->proc_ctx_obj.gpu_addr, 281 &uq_mgr->proc_ctx_obj.cpu_ptr); 282 283 if (!r) 284 memset(uq_mgr->proc_ctx_obj.cpu_ptr, 0, AMDGPU_USERQ_PROC_CTX_SZ); 285 } 286 287 mutex_unlock(&uq_mgr->proc_ctx_lock); 288 289 return r; 290 } 291 292 static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, 293 struct drm_amdgpu_userq_in *args_in) 294 { 295 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 296 struct amdgpu_device *adev = uq_mgr->adev; 297 struct amdgpu_mqd *mqd_hw_default = &adev->mqds[queue->queue_type]; 298 struct drm_amdgpu_userq_in *mqd_user = args_in; 299 struct amdgpu_mqd_prop *userq_props; 300 int r; 301 302 /* Structure to initialize MQD for userqueue using generic MQD init function */ 303 userq_props = kzalloc_obj(struct amdgpu_mqd_prop); 304 if (!userq_props) { 305 DRM_ERROR("Failed to allocate memory for userq_props\n"); 306 return -ENOMEM; 307 } 308 309 r = amdgpu_bo_create_kernel(adev, 310 AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size), 311 0, AMDGPU_GEM_DOMAIN_GTT, 312 &queue->mqd.obj, &queue->mqd.gpu_addr, 313 &queue->mqd.cpu_ptr); 314 if (r) { 315 DRM_ERROR("Failed to create MQD object for userqueue\n"); 316 goto free_props; 317 } 318 319 memset(queue->mqd.cpu_ptr, 0, 320 AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size)); 321 322 /* Initialize the MQD BO with user given values */ 323 userq_props->wptr_gpu_addr = mqd_user->wptr_va; 324 userq_props->rptr_gpu_addr = mqd_user->rptr_va; 325 userq_props->queue_size = mqd_user->queue_size; 326 userq_props->hqd_base_gpu_addr = mqd_user->queue_va; 327 userq_props->mqd_gpu_addr = queue->mqd.gpu_addr; 328 userq_props->use_doorbell = true; 329 userq_props->doorbell_index = queue->doorbell_index; 330 userq_props->fence_address = queue->fence_drv->gpu_addr; 331 332 if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) { 333 struct drm_amdgpu_userq_mqd_compute_gfx11 *compute_mqd; 334 335 if (mqd_user->mqd_size != sizeof(*compute_mqd)) { 336 DRM_ERROR("Invalid compute IP MQD size\n"); 337 r = -EINVAL; 338 goto free_mqd; 339 } 340 341 compute_mqd = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); 342 if (IS_ERR(compute_mqd)) { 343 DRM_ERROR("Failed to read user MQD\n"); 344 r = -ENOMEM; 345 goto free_mqd; 346 } 347 348 r = amdgpu_bo_reserve(queue->vm->root.bo, false); 349 if (r) { 350 kfree(compute_mqd); 351 goto free_mqd; 352 } 353 r = amdgpu_userq_input_va_validate(adev, queue, 354 compute_mqd->eop_va, 2048, 355 &queue->userq_vas.va.eop); 356 amdgpu_bo_unreserve(queue->vm->root.bo); 357 if (r) { 358 kfree(compute_mqd); 359 goto free_mqd; 360 } 361 362 userq_props->eop_gpu_addr = compute_mqd->eop_va; 363 userq_props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL; 364 userq_props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM; 365 userq_props->hqd_active = false; 366 userq_props->tmz_queue = 367 mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE; 368 kfree(compute_mqd); 369 } else if (queue->queue_type == AMDGPU_HW_IP_GFX) { 370 struct drm_amdgpu_userq_mqd_gfx11 *mqd_gfx_v11; 371 struct amdgpu_gfx_shadow_info shadow_info; 372 373 if (adev->gfx.funcs->get_gfx_shadow_info) { 374 adev->gfx.funcs->get_gfx_shadow_info(adev, &shadow_info, true); 375 } else { 376 r = -EINVAL; 377 goto free_mqd; 378 } 379 380 if (mqd_user->mqd_size != sizeof(*mqd_gfx_v11) || !mqd_user->mqd) { 381 DRM_ERROR("Invalid GFX MQD\n"); 382 r = -EINVAL; 383 goto free_mqd; 384 } 385 386 mqd_gfx_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); 387 if (IS_ERR(mqd_gfx_v11)) { 388 DRM_ERROR("Failed to read user MQD\n"); 389 r = -ENOMEM; 390 goto free_mqd; 391 } 392 393 userq_props->shadow_addr = mqd_gfx_v11->shadow_va; 394 userq_props->csa_addr = mqd_gfx_v11->csa_va; 395 userq_props->tmz_queue = 396 mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE; 397 398 r = amdgpu_bo_reserve(queue->vm->root.bo, false); 399 if (r) { 400 kfree(mqd_gfx_v11); 401 goto free_mqd; 402 } 403 r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->shadow_va, 404 shadow_info.shadow_size, 405 &queue->userq_vas.va.shadow); 406 if (r) { 407 amdgpu_bo_unreserve(queue->vm->root.bo); 408 kfree(mqd_gfx_v11); 409 goto free_mqd; 410 } 411 412 r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->csa_va, 413 shadow_info.csa_size, 414 &queue->userq_vas.va.csa); 415 amdgpu_bo_unreserve(queue->vm->root.bo); 416 if (r) { 417 kfree(mqd_gfx_v11); 418 goto free_mqd; 419 } 420 421 kfree(mqd_gfx_v11); 422 } else if (queue->queue_type == AMDGPU_HW_IP_DMA) { 423 struct drm_amdgpu_userq_mqd_sdma_gfx11 *mqd_sdma_v11; 424 425 if (mqd_user->mqd_size != sizeof(*mqd_sdma_v11) || !mqd_user->mqd) { 426 DRM_ERROR("Invalid SDMA MQD\n"); 427 r = -EINVAL; 428 goto free_mqd; 429 } 430 431 mqd_sdma_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); 432 if (IS_ERR(mqd_sdma_v11)) { 433 DRM_ERROR("Failed to read sdma user MQD\n"); 434 r = -ENOMEM; 435 goto free_mqd; 436 } 437 438 r = amdgpu_bo_reserve(queue->vm->root.bo, false); 439 if (r) { 440 kfree(mqd_sdma_v11); 441 goto free_mqd; 442 } 443 r = amdgpu_userq_input_va_validate(adev, queue, mqd_sdma_v11->csa_va, 444 32, 445 &queue->userq_vas.va.csa); 446 amdgpu_bo_unreserve(queue->vm->root.bo); 447 if (r) { 448 kfree(mqd_sdma_v11); 449 goto free_mqd; 450 } 451 452 userq_props->csa_addr = mqd_sdma_v11->csa_va; 453 kfree(mqd_sdma_v11); 454 } 455 456 queue->userq_prop = userq_props; 457 458 r = mqd_hw_default->init_mqd(adev, (void *)queue->mqd.cpu_ptr, userq_props); 459 if (r) { 460 DRM_ERROR("Failed to initialize MQD for userqueue\n"); 461 goto free_mqd; 462 } 463 464 /* Create per-process MES process context BO */ 465 r = mes_userq_create_proc_ctx_space(uq_mgr); 466 if (r) { 467 DRM_ERROR("Failed to allocate MES process context space bo, error: %d\n", r); 468 goto free_mqd; 469 } 470 471 /* Create BO of a gang for FW operations */ 472 r = mes_userq_create_ctx_space(uq_mgr, queue, mqd_user); 473 if (r) { 474 DRM_ERROR("Failed to allocate BO for userqueue (%d)", r); 475 goto free_mqd; 476 } 477 478 /* FW expects WPTR BOs to be mapped into GART */ 479 r = mes_userq_create_wptr_mapping(adev, uq_mgr, queue, userq_props->wptr_gpu_addr); 480 if (r) { 481 DRM_ERROR("Failed to create WPTR mapping\n"); 482 goto free_ctx; 483 } 484 485 return 0; 486 487 free_ctx: 488 amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr, 489 &queue->fw_obj.cpu_ptr); 490 491 free_mqd: 492 amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr, 493 &queue->mqd.cpu_ptr); 494 495 free_props: 496 kfree(userq_props); 497 498 return r; 499 } 500 501 static void mes_userq_mqd_destroy(struct amdgpu_usermode_queue *queue) 502 { 503 504 amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr, 505 &queue->fw_obj.cpu_ptr); 506 kfree(queue->userq_prop); 507 amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr, 508 &queue->mqd.cpu_ptr); 509 510 amdgpu_bo_unref(&queue->wptr_obj.obj); 511 } 512 513 static int mes_userq_preempt(struct amdgpu_usermode_queue *queue) 514 { 515 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 516 struct amdgpu_device *adev = uq_mgr->adev; 517 struct mes_suspend_gang_input queue_input; 518 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 519 signed long timeout = 2100000; /* 2100 ms */ 520 u64 fence_gpu_addr; 521 u32 fence_offset; 522 u64 *fence_ptr; 523 int i, r; 524 525 if (queue->state != AMDGPU_USERQ_STATE_MAPPED) 526 return 0; 527 r = amdgpu_wb_get(adev, &fence_offset); 528 if (r) 529 return r; 530 531 fence_gpu_addr = adev->wb.gpu_addr + (fence_offset * 4); 532 fence_ptr = (u64 *)&adev->wb.wb[fence_offset]; 533 *fence_ptr = 0; 534 535 memset(&queue_input, 0x0, sizeof(struct mes_suspend_gang_input)); 536 queue_input.gang_context_addr = ctx->gpu_addr; 537 queue_input.suspend_fence_addr = fence_gpu_addr; 538 queue_input.suspend_fence_value = 1; 539 amdgpu_mes_lock(&adev->mes); 540 r = adev->mes.funcs->suspend_gang(&adev->mes, &queue_input); 541 amdgpu_mes_unlock(&adev->mes); 542 if (r) { 543 DRM_ERROR("Failed to suspend gang: %d\n", r); 544 goto out; 545 } 546 547 for (i = 0; i < timeout; i++) { 548 if (*fence_ptr == 1) 549 goto out; 550 udelay(1); 551 } 552 r = -ETIMEDOUT; 553 554 out: 555 amdgpu_wb_free(adev, fence_offset); 556 return r; 557 } 558 559 static int mes_userq_restore(struct amdgpu_usermode_queue *queue) 560 { 561 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 562 struct amdgpu_device *adev = uq_mgr->adev; 563 struct mes_resume_gang_input queue_input; 564 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 565 int r; 566 567 if (queue->state == AMDGPU_USERQ_STATE_HUNG) 568 return -EINVAL; 569 if (queue->state != AMDGPU_USERQ_STATE_PREEMPTED) 570 return 0; 571 572 memset(&queue_input, 0x0, sizeof(struct mes_resume_gang_input)); 573 queue_input.gang_context_addr = ctx->gpu_addr; 574 575 amdgpu_mes_lock(&adev->mes); 576 r = adev->mes.funcs->resume_gang(&adev->mes, &queue_input); 577 amdgpu_mes_unlock(&adev->mes); 578 if (r) 579 dev_err(adev->dev, "Failed to resume queue, err (%d)\n", r); 580 return r; 581 } 582 583 const struct amdgpu_userq_funcs userq_mes_funcs = { 584 .mqd_create = mes_userq_mqd_create, 585 .mqd_destroy = mes_userq_mqd_destroy, 586 .unmap = mes_userq_unmap, 587 .map = mes_userq_map, 588 .preempt = mes_userq_preempt, 589 .restore = mes_userq_restore, 590 .reset = mes_userq_reset, 591 }; 592