1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2024 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 #include <drm/drm_drv.h> 25 #include "amdgpu.h" 26 #include "amdgpu_gfx.h" 27 #include "mes_userqueue.h" 28 #include "amdgpu_userq_fence.h" 29 #include "amdgpu_trace.h" 30 31 #define AMDGPU_USERQ_PROC_CTX_SZ PAGE_SIZE 32 #define AMDGPU_USERQ_GANG_CTX_SZ PAGE_SIZE 33 34 static int 35 mes_userq_create_wptr_mapping(struct amdgpu_device *adev, 36 struct amdgpu_userq_mgr *uq_mgr, 37 struct amdgpu_usermode_queue *queue, 38 uint64_t wptr) 39 { 40 struct amdgpu_bo_va_mapping *wptr_mapping; 41 struct amdgpu_userq_obj *wptr_obj = &queue->wptr_obj; 42 struct amdgpu_bo *obj; 43 struct amdgpu_vm *vm = queue->vm; 44 struct drm_exec exec; 45 int ret; 46 47 wptr &= AMDGPU_GMC_HOLE_MASK; 48 49 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 2); 50 drm_exec_until_all_locked(&exec) { 51 ret = amdgpu_vm_lock_pd(vm, &exec, 1); 52 drm_exec_retry_on_contention(&exec); 53 if (unlikely(ret)) 54 goto fail_lock; 55 56 wptr_mapping = amdgpu_vm_bo_lookup_mapping(vm, wptr >> PAGE_SHIFT); 57 if (!wptr_mapping) { 58 ret = -EINVAL; 59 goto fail_lock; 60 } 61 62 obj = wptr_mapping->bo_va->base.bo; 63 ret = drm_exec_lock_obj(&exec, &obj->tbo.base); 64 drm_exec_retry_on_contention(&exec); 65 if (unlikely(ret)) 66 goto fail_lock; 67 } 68 69 wptr_obj->obj = amdgpu_bo_ref(wptr_mapping->bo_va->base.bo); 70 if (wptr_obj->obj->tbo.base.size > PAGE_SIZE) { 71 ret = -EINVAL; 72 goto fail_map; 73 } 74 /* Keep WPTR BO under eviction-fence control instead of pinning. */ 75 ret = amdgpu_evf_mgr_attach_fence(&uq_mgr_to_fpriv(uq_mgr)->evf_mgr, wptr_obj->obj); 76 if (ret) { 77 DRM_ERROR("Failed to attach eviction fence to wptr bo. ret %d\n", ret); 78 goto fail_map; 79 } 80 81 ret = amdgpu_ttm_alloc_gart(&wptr_obj->obj->tbo); 82 if (ret) { 83 DRM_ERROR("Failed to bind wptr bo to GART. ret %d\n", ret); 84 goto fail_map; 85 } 86 87 queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(wptr_obj->obj); 88 89 drm_exec_fini(&exec); 90 return 0; 91 fail_map: 92 amdgpu_bo_unref(&wptr_obj->obj); 93 fail_lock: 94 drm_exec_fini(&exec); 95 return ret; 96 97 } 98 99 static int convert_to_mes_priority(int priority) 100 { 101 switch (priority) { 102 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_LOW: 103 default: 104 return AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 105 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_LOW: 106 return AMDGPU_MES_PRIORITY_LEVEL_LOW; 107 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_HIGH: 108 return AMDGPU_MES_PRIORITY_LEVEL_MEDIUM; 109 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_HIGH: 110 return AMDGPU_MES_PRIORITY_LEVEL_HIGH; 111 } 112 } 113 114 static int mes_userq_map(struct amdgpu_usermode_queue *queue) 115 { 116 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 117 struct amdgpu_device *adev = uq_mgr->adev; 118 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 119 struct amdgpu_mqd_prop *userq_props = queue->userq_prop; 120 struct mes_add_queue_input queue_input; 121 struct amdgpu_mes *mes = &adev->mes; 122 int r; 123 124 memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input)); 125 126 queue_input.process_va_start = 0; 127 queue_input.process_va_end = adev->vm_manager.max_pfn - 1; 128 129 /* set process quantum to 10 ms and gang quantum to 1 ms as default */ 130 queue_input.process_quantum = 100000; 131 queue_input.gang_quantum = 10000; 132 queue_input.paging = false; 133 134 queue_input.process_context_addr = uq_mgr->proc_ctx_obj.gpu_addr; 135 queue_input.gang_context_addr = ctx->gpu_addr; 136 queue_input.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL; 137 queue_input.gang_global_priority_level = convert_to_mes_priority(queue->priority); 138 139 queue_input.process_id = queue->vm->pasid; 140 queue_input.queue_type = queue->queue_type; 141 queue_input.mqd_addr = queue->mqd.gpu_addr; 142 queue_input.wptr_addr = userq_props->wptr_gpu_addr; 143 queue_input.queue_size = userq_props->queue_size >> 2; 144 queue_input.doorbell_offset = userq_props->doorbell_index; 145 queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo); 146 queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr; 147 148 if (mes->use_rs64mem) { 149 if (!uq_mgr->proc_ctx_allocated) { 150 r = amdgpu_mes_alloc_proc_ctx_index(mes, &uq_mgr->proc_ctx_array_index); 151 if (r) { 152 DRM_ERROR("Failed to allocate userq process index err:%d\n", r); 153 return r; 154 } 155 uq_mgr->proc_ctx_allocated = true; 156 } 157 158 r = amdgpu_mes_alloc_gang_ctx_index(mes, &queue->gang_ctx_array_index); 159 if (r) { 160 DRM_ERROR("Failed to allocate userq gang index err:%d\n", r); 161 return r; 162 } 163 queue_input.process_context_array_index = uq_mgr->proc_ctx_array_index; 164 queue_input.gang_context_array_index = queue->gang_ctx_array_index; 165 } 166 amdgpu_mes_lock(&adev->mes); 167 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input); 168 amdgpu_mes_unlock(&adev->mes); 169 if (r) { 170 DRM_ERROR("Failed to map queue in HW, err (%d)\n", r); 171 return r; 172 } 173 174 DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index); 175 return 0; 176 } 177 178 static int mes_userq_unmap(struct amdgpu_usermode_queue *queue) 179 { 180 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 181 struct amdgpu_device *adev = uq_mgr->adev; 182 struct mes_remove_queue_input queue_input; 183 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 184 struct amdgpu_mes *mes = &adev->mes; 185 int r; 186 187 memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); 188 queue_input.gang_context_array_index = queue->gang_ctx_array_index; 189 queue_input.doorbell_offset = queue->doorbell_index; 190 queue_input.gang_context_addr = ctx->gpu_addr; 191 queue_input.queue_type = queue->queue_type; 192 193 amdgpu_mes_lock(&adev->mes); 194 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); 195 amdgpu_mes_unlock(&adev->mes); 196 if (mes->use_rs64mem) 197 amdgpu_mes_free_gang_ctx_index(mes, queue->gang_ctx_array_index); 198 if (r) 199 DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r); 200 return r; 201 } 202 203 int mes_userq_reset(struct amdgpu_usermode_queue *queue) 204 { 205 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 206 struct amdgpu_device *adev = uq_mgr->adev; 207 struct mes_reset_queue_input queue_input; 208 int r; 209 210 /* XXX: add a FW version check for SDMA per queue reset */ 211 memset(&queue_input, 0x0, sizeof(struct mes_reset_queue_input)); 212 queue_input.doorbell_offset = queue->doorbell_index; 213 queue_input.queue_type = queue->queue_type; 214 215 amdgpu_mes_lock(&adev->mes); 216 r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); 217 amdgpu_mes_unlock(&adev->mes); 218 if (r) 219 return r; 220 221 /* mes_userq_unmap() does not update queue->state; mark it UNMAPPED so the 222 * destroy path does not issue a second REMOVE_QUEUE for the removed queue. 223 */ 224 r = mes_userq_unmap(queue); 225 if (!r) { 226 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_UNMAPPED); 227 queue->state = AMDGPU_USERQ_STATE_UNMAPPED; 228 } 229 return r; 230 } 231 232 int mes_userq_reset_queue(struct amdgpu_device *adev, 233 struct amdgpu_usermode_queue *guilty_uq, 234 int queue_type, 235 unsigned int pipe, 236 unsigned int queue, 237 unsigned int db) 238 { 239 struct amdgpu_usermode_queue *uq; 240 bool use_mmio = adev->gfx.mec.use_mmio_for_reset; 241 unsigned long uq_id; 242 int r; 243 244 xa_for_each(&adev->userq_doorbell_xa, uq_id, uq) { 245 if (uq->queue_type == queue_type) { 246 if (uq == guilty_uq) 247 continue; 248 if (uq->doorbell_index == db) { 249 uq->state = AMDGPU_USERQ_STATE_HUNG; 250 if (use_mmio) 251 r = amdgpu_mes_reset_queue_mmio(adev, queue_type, 0, 1, pipe, queue, 0); 252 else 253 r = amdgpu_mes_reset_user_queue(adev, queue_type, db, 0); 254 if (r) 255 return r; 256 r = mes_userq_unmap(uq); 257 if (r) 258 return r; 259 amdgpu_userq_fence_driver_force_completion(uq); 260 break; 261 } 262 } 263 } 264 return 0; 265 } 266 267 static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr, 268 struct amdgpu_usermode_queue *queue, 269 struct drm_amdgpu_userq_in *mqd_user) 270 { 271 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 272 int r, size; 273 274 /* The FW expects at least one page space allocated for gang ctx. */ 275 size = AMDGPU_USERQ_GANG_CTX_SZ; 276 r = amdgpu_bo_create_kernel(uq_mgr->adev, size, 0, 277 AMDGPU_GEM_DOMAIN_GTT, 278 &ctx->obj, &ctx->gpu_addr, 279 &ctx->cpu_ptr); 280 if (r) { 281 DRM_ERROR("Failed to allocate ctx space bo for userqueue, err:%d\n", r); 282 return r; 283 } 284 285 memset(ctx->cpu_ptr, 0, size); 286 return 0; 287 } 288 289 static int mes_userq_create_proc_ctx_space(struct amdgpu_userq_mgr *uq_mgr) 290 { 291 int r = 0; 292 293 mutex_lock(&uq_mgr->proc_ctx_lock); 294 /* This check is a necessary because amdgpu_bo_create_kernel() 295 * calls helpers like amdgpu_bo_pin() and memset() unconditionally 296 */ 297 if (!uq_mgr->proc_ctx_obj.obj) { 298 r = amdgpu_bo_create_kernel(uq_mgr->adev, AMDGPU_USERQ_PROC_CTX_SZ, 299 0, AMDGPU_GEM_DOMAIN_GTT, 300 &uq_mgr->proc_ctx_obj.obj, 301 &uq_mgr->proc_ctx_obj.gpu_addr, 302 &uq_mgr->proc_ctx_obj.cpu_ptr); 303 304 if (!r) 305 memset(uq_mgr->proc_ctx_obj.cpu_ptr, 0, AMDGPU_USERQ_PROC_CTX_SZ); 306 } 307 308 mutex_unlock(&uq_mgr->proc_ctx_lock); 309 310 return r; 311 } 312 313 static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, 314 struct drm_amdgpu_userq_in *args_in) 315 { 316 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 317 struct amdgpu_device *adev = uq_mgr->adev; 318 struct amdgpu_mqd *mqd_hw_default = &adev->mqds[queue->queue_type]; 319 struct drm_amdgpu_userq_in *mqd_user = args_in; 320 struct amdgpu_mqd_prop *userq_props; 321 int r; 322 323 /* Structure to initialize MQD for userqueue using generic MQD init function */ 324 userq_props = kzalloc_obj(struct amdgpu_mqd_prop); 325 if (!userq_props) { 326 DRM_ERROR("Failed to allocate memory for userq_props\n"); 327 return -ENOMEM; 328 } 329 330 r = amdgpu_bo_create_kernel(adev, 331 AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size), 332 0, AMDGPU_GEM_DOMAIN_GTT, 333 &queue->mqd.obj, &queue->mqd.gpu_addr, 334 &queue->mqd.cpu_ptr); 335 if (r) { 336 DRM_ERROR("Failed to create MQD object for userqueue\n"); 337 goto free_props; 338 } 339 340 memset(queue->mqd.cpu_ptr, 0, 341 AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size)); 342 343 /* Initialize the MQD BO with user given values */ 344 userq_props->wptr_gpu_addr = mqd_user->wptr_va; 345 userq_props->rptr_gpu_addr = mqd_user->rptr_va; 346 userq_props->queue_size = mqd_user->queue_size; 347 userq_props->hqd_base_gpu_addr = mqd_user->queue_va; 348 userq_props->mqd_gpu_addr = queue->mqd.gpu_addr; 349 userq_props->use_doorbell = true; 350 userq_props->doorbell_index = queue->doorbell_index; 351 userq_props->fence_address = queue->fence_drv->gpu_addr; 352 353 if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) { 354 struct drm_amdgpu_userq_mqd_compute_gfx11 *compute_mqd; 355 356 if (mqd_user->mqd_size != sizeof(*compute_mqd)) { 357 DRM_ERROR("Invalid compute IP MQD size\n"); 358 r = -EINVAL; 359 goto free_mqd; 360 } 361 362 compute_mqd = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); 363 if (IS_ERR(compute_mqd)) { 364 DRM_ERROR("Failed to read user MQD\n"); 365 r = -ENOMEM; 366 goto free_mqd; 367 } 368 369 r = amdgpu_bo_reserve(queue->vm->root.bo, false); 370 if (r) { 371 kfree(compute_mqd); 372 goto free_mqd; 373 } 374 r = amdgpu_userq_input_va_validate(adev, queue, 375 compute_mqd->eop_va, 2048, 376 &queue->userq_vas.va.eop); 377 amdgpu_bo_unreserve(queue->vm->root.bo); 378 if (r) { 379 kfree(compute_mqd); 380 goto free_mqd; 381 } 382 383 userq_props->eop_gpu_addr = compute_mqd->eop_va; 384 userq_props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL; 385 userq_props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM; 386 userq_props->hqd_active = false; 387 userq_props->tmz_queue = 388 mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE; 389 kfree(compute_mqd); 390 } else if (queue->queue_type == AMDGPU_HW_IP_GFX) { 391 struct drm_amdgpu_userq_mqd_gfx11 *mqd_gfx_v11; 392 struct amdgpu_gfx_shadow_info shadow_info; 393 394 if (adev->gfx.funcs->get_gfx_shadow_info) { 395 adev->gfx.funcs->get_gfx_shadow_info(adev, &shadow_info, true); 396 } else { 397 r = -EINVAL; 398 goto free_mqd; 399 } 400 401 if (mqd_user->mqd_size != sizeof(*mqd_gfx_v11) || !mqd_user->mqd) { 402 DRM_ERROR("Invalid GFX MQD\n"); 403 r = -EINVAL; 404 goto free_mqd; 405 } 406 407 mqd_gfx_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); 408 if (IS_ERR(mqd_gfx_v11)) { 409 DRM_ERROR("Failed to read user MQD\n"); 410 r = -ENOMEM; 411 goto free_mqd; 412 } 413 414 userq_props->shadow_addr = mqd_gfx_v11->shadow_va; 415 userq_props->csa_addr = mqd_gfx_v11->csa_va; 416 userq_props->tmz_queue = 417 mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE; 418 419 r = amdgpu_bo_reserve(queue->vm->root.bo, false); 420 if (r) { 421 kfree(mqd_gfx_v11); 422 goto free_mqd; 423 } 424 r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->shadow_va, 425 shadow_info.shadow_size, 426 &queue->userq_vas.va.shadow); 427 if (r) { 428 amdgpu_bo_unreserve(queue->vm->root.bo); 429 kfree(mqd_gfx_v11); 430 goto free_mqd; 431 } 432 433 r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->csa_va, 434 shadow_info.csa_size, 435 &queue->userq_vas.va.csa); 436 amdgpu_bo_unreserve(queue->vm->root.bo); 437 if (r) { 438 kfree(mqd_gfx_v11); 439 goto free_mqd; 440 } 441 442 kfree(mqd_gfx_v11); 443 } else if (queue->queue_type == AMDGPU_HW_IP_DMA) { 444 struct drm_amdgpu_userq_mqd_sdma_gfx11 *mqd_sdma_v11; 445 446 if (mqd_user->mqd_size != sizeof(*mqd_sdma_v11) || !mqd_user->mqd) { 447 DRM_ERROR("Invalid SDMA MQD\n"); 448 r = -EINVAL; 449 goto free_mqd; 450 } 451 452 mqd_sdma_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); 453 if (IS_ERR(mqd_sdma_v11)) { 454 DRM_ERROR("Failed to read sdma user MQD\n"); 455 r = -ENOMEM; 456 goto free_mqd; 457 } 458 459 r = amdgpu_bo_reserve(queue->vm->root.bo, false); 460 if (r) { 461 kfree(mqd_sdma_v11); 462 goto free_mqd; 463 } 464 r = amdgpu_userq_input_va_validate(adev, queue, mqd_sdma_v11->csa_va, 465 32, 466 &queue->userq_vas.va.csa); 467 amdgpu_bo_unreserve(queue->vm->root.bo); 468 if (r) { 469 kfree(mqd_sdma_v11); 470 goto free_mqd; 471 } 472 473 userq_props->csa_addr = mqd_sdma_v11->csa_va; 474 kfree(mqd_sdma_v11); 475 } 476 477 queue->userq_prop = userq_props; 478 479 r = mqd_hw_default->init_mqd(adev, (void *)queue->mqd.cpu_ptr, userq_props); 480 if (r) { 481 DRM_ERROR("Failed to initialize MQD for userqueue\n"); 482 goto free_mqd; 483 } 484 485 /* Create per-process MES process context BO */ 486 r = mes_userq_create_proc_ctx_space(uq_mgr); 487 if (r) { 488 DRM_ERROR("Failed to allocate MES process context space bo, error: %d\n", r); 489 goto free_mqd; 490 } 491 492 /* Create BO of a gang for FW operations */ 493 r = mes_userq_create_ctx_space(uq_mgr, queue, mqd_user); 494 if (r) { 495 DRM_ERROR("Failed to allocate BO for userqueue (%d)", r); 496 goto free_mqd; 497 } 498 499 /* FW expects WPTR BOs to be mapped into GART */ 500 r = mes_userq_create_wptr_mapping(adev, uq_mgr, queue, userq_props->wptr_gpu_addr); 501 if (r) { 502 DRM_ERROR("Failed to create WPTR mapping\n"); 503 goto free_ctx; 504 } 505 506 return 0; 507 508 free_ctx: 509 amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr, 510 &queue->fw_obj.cpu_ptr); 511 512 free_mqd: 513 amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr, 514 &queue->mqd.cpu_ptr); 515 516 free_props: 517 kfree(userq_props); 518 519 return r; 520 } 521 522 static void mes_userq_mqd_destroy(struct amdgpu_usermode_queue *queue) 523 { 524 525 amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr, 526 &queue->fw_obj.cpu_ptr); 527 kfree(queue->userq_prop); 528 amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr, 529 &queue->mqd.cpu_ptr); 530 531 amdgpu_bo_unref(&queue->wptr_obj.obj); 532 } 533 534 static int mes_userq_preempt(struct amdgpu_usermode_queue *queue) 535 { 536 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 537 struct amdgpu_device *adev = uq_mgr->adev; 538 struct mes_suspend_gang_input queue_input; 539 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 540 signed long timeout = 2100000; /* 2100 ms */ 541 u64 fence_gpu_addr; 542 u32 fence_offset; 543 u64 *fence_ptr; 544 int i, r; 545 546 if (queue->state != AMDGPU_USERQ_STATE_MAPPED) 547 return 0; 548 r = amdgpu_wb_get(adev, &fence_offset); 549 if (r) 550 return r; 551 552 fence_gpu_addr = adev->wb.gpu_addr + (fence_offset * 4); 553 fence_ptr = (u64 *)&adev->wb.wb[fence_offset]; 554 *fence_ptr = 0; 555 556 memset(&queue_input, 0x0, sizeof(struct mes_suspend_gang_input)); 557 queue_input.gang_context_addr = ctx->gpu_addr; 558 queue_input.suspend_fence_addr = fence_gpu_addr; 559 queue_input.suspend_fence_value = 1; 560 amdgpu_mes_lock(&adev->mes); 561 r = adev->mes.funcs->suspend_gang(&adev->mes, &queue_input); 562 amdgpu_mes_unlock(&adev->mes); 563 if (r) { 564 DRM_ERROR("Failed to suspend gang: %d\n", r); 565 goto out; 566 } 567 568 for (i = 0; i < timeout; i++) { 569 if (*fence_ptr == 1) 570 goto out; 571 udelay(1); 572 } 573 r = -ETIMEDOUT; 574 575 out: 576 amdgpu_wb_free(adev, fence_offset); 577 return r; 578 } 579 580 static int mes_userq_restore(struct amdgpu_usermode_queue *queue) 581 { 582 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 583 struct amdgpu_device *adev = uq_mgr->adev; 584 struct mes_resume_gang_input queue_input; 585 struct amdgpu_userq_obj *ctx = &queue->fw_obj; 586 int r; 587 588 if (queue->state == AMDGPU_USERQ_STATE_HUNG) 589 return -EINVAL; 590 if (queue->state != AMDGPU_USERQ_STATE_PREEMPTED) 591 return 0; 592 593 memset(&queue_input, 0x0, sizeof(struct mes_resume_gang_input)); 594 queue_input.gang_context_addr = ctx->gpu_addr; 595 596 amdgpu_mes_lock(&adev->mes); 597 r = adev->mes.funcs->resume_gang(&adev->mes, &queue_input); 598 amdgpu_mes_unlock(&adev->mes); 599 if (r) 600 dev_err(adev->dev, "Failed to resume queue, err (%d)\n", r); 601 return r; 602 } 603 604 const struct amdgpu_userq_funcs userq_mes_funcs = { 605 .mqd_create = mes_userq_mqd_create, 606 .mqd_destroy = mes_userq_mqd_destroy, 607 .unmap = mes_userq_unmap, 608 .map = mes_userq_map, 609 .preempt = mes_userq_preempt, 610 .restore = mes_userq_restore, 611 .reset = mes_userq_reset, 612 }; 613