1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2023 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <drm/drm_auth.h> 26 #include <drm/drm_exec.h> 27 #include <linux/pm_runtime.h> 28 #include <linux/overflow.h> 29 #include <drm/drm_drv.h> 30 31 #include "amdgpu.h" 32 #include "amdgpu_reset.h" 33 #include "amdgpu_vm.h" 34 #include "amdgpu_userq.h" 35 #include "amdgpu_hmm.h" 36 #include "amdgpu_userq_fence.h" 37 #include "amdgpu_trace.h" 38 39 u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev) 40 { 41 int i; 42 u32 userq_ip_mask = 0; 43 44 for (i = 0; i < AMDGPU_HW_IP_NUM; i++) { 45 if (adev->userq_funcs[i]) 46 userq_ip_mask |= (1 << i); 47 } 48 49 return userq_ip_mask; 50 } 51 52 static bool amdgpu_userq_is_reset_type_supported(struct amdgpu_device *adev, 53 enum amdgpu_ring_type ring_type, int reset_type) 54 { 55 56 if (ring_type < 0 || ring_type >= AMDGPU_RING_TYPE_MAX) 57 return false; 58 59 switch (ring_type) { 60 case AMDGPU_RING_TYPE_GFX: 61 if (adev->gfx.gfx_supported_reset & reset_type) 62 return true; 63 break; 64 case AMDGPU_RING_TYPE_COMPUTE: 65 if (adev->gfx.compute_supported_reset & reset_type) 66 return true; 67 break; 68 case AMDGPU_RING_TYPE_SDMA: 69 if (adev->sdma.supported_reset & reset_type) 70 return true; 71 break; 72 case AMDGPU_RING_TYPE_VCN_DEC: 73 case AMDGPU_RING_TYPE_VCN_ENC: 74 if (adev->vcn.supported_reset & reset_type) 75 return true; 76 break; 77 case AMDGPU_RING_TYPE_VCN_JPEG: 78 if (adev->jpeg.supported_reset & reset_type) 79 return true; 80 break; 81 default: 82 break; 83 } 84 return false; 85 } 86 87 static void amdgpu_userq_mgr_reset_work(struct work_struct *work) 88 { 89 struct amdgpu_userq_mgr *uq_mgr = 90 container_of(work, struct amdgpu_userq_mgr, 91 reset_work); 92 struct amdgpu_device *adev = uq_mgr->adev; 93 struct amdgpu_reset_context reset_context; 94 95 if (unlikely(adev->debug_disable_gpu_ring_reset)) { 96 dev_err(adev->dev, "userq reset disabled by debug mask\n"); 97 return; 98 } 99 100 /* 101 * If GPU recovery feature is disabled system-wide, 102 * skip all reset detection logic 103 */ 104 if (!amdgpu_gpu_recovery) 105 return; 106 107 memset(&reset_context, 0, sizeof(reset_context)); 108 109 reset_context.method = AMD_RESET_METHOD_NONE; 110 reset_context.reset_req_dev = adev; 111 reset_context.src = AMDGPU_RESET_SRC_USERQ; 112 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 113 /*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/ 114 115 amdgpu_device_gpu_recover(adev, NULL, &reset_context); 116 } 117 118 static void amdgpu_userq_hang_detect_work(struct work_struct *work) 119 { 120 struct amdgpu_usermode_queue *queue = 121 container_of(work, struct amdgpu_usermode_queue, 122 hang_detect_work.work); 123 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 124 struct amdgpu_device *adev = uq_mgr->adev; 125 const struct amdgpu_userq_funcs *userq_funcs = 126 adev->userq_funcs[queue->queue_type]; 127 struct drm_wedge_task_info *info = NULL; 128 struct amdgpu_task_info *ti = NULL; 129 bool gpu_reset = false; 130 131 if (unlikely(adev->debug_disable_gpu_ring_reset)) { 132 dev_err(adev->dev, "userq reset disabled by debug mask\n"); 133 return; 134 } 135 136 /* 137 * If GPU recovery feature is disabled system-wide, 138 * skip all reset detection logic 139 */ 140 if (!amdgpu_gpu_recovery) 141 return; 142 143 if (queue->vm && queue->vm->pasid) { 144 ti = amdgpu_vm_get_task_info_pasid(adev, queue->vm->pasid); 145 if (ti) { 146 amdgpu_vm_print_task_info(adev, ti); 147 info = &ti->task; 148 } 149 } 150 151 if (amdgpu_userq_is_reset_type_supported(adev, queue->queue_type, 152 AMDGPU_RESET_TYPE_PER_QUEUE)) { 153 int r; 154 155 if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) 156 r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL, 157 queue, NULL, NULL); 158 else 159 r = userq_funcs->reset(queue); 160 if (r) { 161 gpu_reset = true; 162 } else { 163 atomic_inc(&adev->gpu_reset_counter); 164 amdgpu_userq_fence_driver_force_completion(queue); 165 drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, info); 166 } 167 } else { 168 gpu_reset = true; 169 } 170 amdgpu_vm_put_task_info(ti); 171 172 /* 173 * Don't schedule the work here! Scheduling or queue work from one reset 174 * handler to another is illegal if you don't take extra precautions! 175 */ 176 if (gpu_reset) 177 amdgpu_userq_mgr_reset_work(&queue->userq_mgr->reset_work); 178 } 179 180 /* 181 * Start hang detection for a user queue fence. A delayed work will be scheduled 182 * to reset the queues when the fence doesn't signal in time. 183 */ 184 void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue) 185 { 186 struct amdgpu_device *adev; 187 unsigned long timeout_jiffies; 188 189 adev = queue->userq_mgr->adev; 190 /* Determine timeout based on queue type */ 191 switch (queue->queue_type) { 192 case AMDGPU_RING_TYPE_GFX: 193 timeout_jiffies = adev->gfx_timeout; 194 break; 195 case AMDGPU_RING_TYPE_COMPUTE: 196 timeout_jiffies = adev->compute_timeout; 197 break; 198 case AMDGPU_RING_TYPE_SDMA: 199 timeout_jiffies = adev->sdma_timeout; 200 break; 201 default: 202 timeout_jiffies = adev->gfx_timeout; 203 break; 204 } 205 206 queue_delayed_work(adev->reset_domain->wq, &queue->hang_detect_work, 207 timeout_jiffies); 208 } 209 210 void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell) 211 { 212 struct xarray *xa = &adev->userq_doorbell_xa; 213 struct amdgpu_usermode_queue *queue; 214 unsigned long flags; 215 int r; 216 217 xa_lock_irqsave(xa, flags); 218 queue = xa_load(xa, doorbell); 219 if (queue) { 220 r = amdgpu_userq_fence_driver_process(queue->fence_drv); 221 /* 222 * We are in interrupt context here, this *can't* wait for 223 * reset work to finish. 224 */ 225 if (r >= 0) 226 cancel_delayed_work(&queue->hang_detect_work); 227 228 /* Restart the timer when there are still fences pending */ 229 if (r == 1) 230 amdgpu_userq_start_hang_detect_work(queue); 231 } 232 xa_unlock_irqrestore(xa, flags); 233 } 234 235 int amdgpu_userq_input_va_validate(struct amdgpu_device *adev, 236 struct amdgpu_usermode_queue *queue, 237 u64 addr, u64 expected_size, 238 u64 *va_out) 239 { 240 struct amdgpu_bo_va_mapping *va_map; 241 struct amdgpu_vm *vm = queue->vm; 242 u64 start_addr; 243 u64 end_addr; 244 245 /* Caller must hold vm->root.bo reservation */ 246 dma_resv_assert_held(queue->vm->root.bo->tbo.base.resv); 247 248 if (!expected_size) 249 return -EINVAL; 250 251 start_addr = addr & AMDGPU_GMC_HOLE_MASK; 252 if (check_add_overflow(start_addr, expected_size - 1, &end_addr)) 253 return -EINVAL; 254 255 va_map = amdgpu_vm_bo_lookup_mapping(vm, start_addr); 256 if (!va_map || !va_map->bo_va->base.bo) 257 return -EINVAL; 258 259 /* Lookup guarantees start_addr is mapped; ensure full span is covered. */ 260 if ((end_addr >> AMDGPU_GPU_PAGE_SHIFT) <= va_map->last) { 261 va_map->bo_va->userq_va_mapped = true; 262 *va_out = start_addr; 263 return 0; 264 } 265 266 return -EINVAL; 267 } 268 269 static bool amdgpu_userq_buffer_va_mapped(struct amdgpu_vm *vm, u64 addr) 270 { 271 struct amdgpu_bo_va_mapping *mapping; 272 bool r; 273 274 dma_resv_assert_held(vm->root.bo->tbo.base.resv); 275 276 mapping = amdgpu_vm_bo_lookup_mapping(vm, addr); 277 if (!IS_ERR_OR_NULL(mapping) && mapping->bo_va->userq_va_mapped) 278 r = true; 279 else 280 r = false; 281 282 return r; 283 } 284 285 static bool amdgpu_userq_buffer_vas_mapped(struct amdgpu_usermode_queue *queue) 286 { 287 int i; 288 bool mapped; 289 290 for (i = 0; i < ARRAY_SIZE(queue->userq_vas.va_array); i++) { 291 if (!queue->userq_vas.va_array[i]) 292 continue; 293 294 mapped = amdgpu_userq_buffer_va_mapped(queue->vm, 295 queue->userq_vas.va_array[i]); 296 dev_dbg(queue->userq_mgr->adev->dev, 297 "validate the userq mapping:%p va:%llx r:%d\n", 298 queue, queue->userq_vas.va_array[i], mapped); 299 300 if (!mapped) 301 return false; 302 } 303 304 return true; 305 } 306 307 308 309 static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue) 310 { 311 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 312 struct amdgpu_device *adev = uq_mgr->adev; 313 const struct amdgpu_userq_funcs *userq_funcs = 314 adev->userq_funcs[queue->queue_type]; 315 int r; 316 317 if (queue->state == AMDGPU_USERQ_STATE_MAPPED) { 318 trace_amdgpu_userq_state_start(queue); 319 320 r = userq_funcs->preempt(queue); 321 if (r) { 322 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG); 323 queue->state = AMDGPU_USERQ_STATE_HUNG; 324 return r; 325 } else { 326 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_PREEMPTED); 327 queue->state = AMDGPU_USERQ_STATE_PREEMPTED; 328 } 329 } 330 return 0; 331 } 332 333 static int amdgpu_userq_restore_helper(struct amdgpu_usermode_queue *queue) 334 { 335 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 336 struct amdgpu_device *adev = uq_mgr->adev; 337 const struct amdgpu_userq_funcs *userq_funcs = 338 adev->userq_funcs[queue->queue_type]; 339 int r = 0; 340 341 if (queue->state == AMDGPU_USERQ_STATE_PREEMPTED) { 342 trace_amdgpu_userq_state_start(queue); 343 344 r = userq_funcs->restore(queue); 345 if (r) { 346 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG); 347 queue->state = AMDGPU_USERQ_STATE_HUNG; 348 } else { 349 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED); 350 queue->state = AMDGPU_USERQ_STATE_MAPPED; 351 } 352 } 353 354 return r; 355 } 356 357 static int amdgpu_userq_unmap_helper(struct amdgpu_usermode_queue *queue) 358 { 359 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 360 struct amdgpu_device *adev = uq_mgr->adev; 361 const struct amdgpu_userq_funcs *userq_funcs = 362 adev->userq_funcs[queue->queue_type]; 363 int r; 364 365 /* The reset changes the queue state and the MES is unusable meanwhile */ 366 guard(rwsem_read)(&adev->reset_domain->sem); 367 if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) || 368 (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) { 369 trace_amdgpu_userq_state_start(queue); 370 371 r = userq_funcs->unmap(queue); 372 if (r) { 373 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG); 374 queue->state = AMDGPU_USERQ_STATE_HUNG; 375 return r; 376 } else { 377 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_UNMAPPED); 378 queue->state = AMDGPU_USERQ_STATE_UNMAPPED; 379 } 380 } 381 382 return 0; 383 } 384 385 static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue) 386 { 387 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 388 struct amdgpu_device *adev = uq_mgr->adev; 389 const struct amdgpu_userq_funcs *userq_funcs = 390 adev->userq_funcs[queue->queue_type]; 391 int r; 392 393 /* The reset changes the queue state and the MES is unusable meanwhile */ 394 guard(rwsem_read)(&adev->reset_domain->sem); 395 if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) { 396 trace_amdgpu_userq_state_start(queue); 397 398 r = userq_funcs->map(queue); 399 if (r) { 400 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG); 401 queue->state = AMDGPU_USERQ_STATE_HUNG; 402 return r; 403 } else { 404 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED); 405 queue->state = AMDGPU_USERQ_STATE_MAPPED; 406 } 407 } 408 409 return 0; 410 } 411 412 static void amdgpu_userq_wait_for_last_fence(struct amdgpu_usermode_queue *queue) 413 { 414 struct dma_fence *f = queue->last_fence; 415 416 if (!f) 417 return; 418 419 dma_fence_wait(f, false); 420 } 421 422 static void amdgpu_userq_detach_doorbell(struct amdgpu_usermode_queue *queue) 423 { 424 struct amdgpu_device *adev = queue->userq_mgr->adev; 425 426 down_read(&adev->reset_domain->sem); 427 xa_erase_irq(&adev->userq_doorbell_xa, queue->doorbell_index); 428 up_read(&adev->reset_domain->sem); 429 } 430 431 /** 432 * amdgpu_userq_ensure_ev_fence - ensure a valid, unsignaled eviction fence exists 433 * @uq_mgr: the usermode queue manager for this process 434 * @evf_mgr: the eviction fence manager to check and rearm 435 * 436 * Ensures that a valid and not yet signaled eviction fence is attached to the 437 * usermode queue before any queue operations proceed. If it is signalled, then 438 * rearm a new eviction fence. 439 */ 440 void 441 amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr, 442 struct amdgpu_eviction_fence_mgr *evf_mgr) 443 { 444 struct dma_fence *ev_fence; 445 446 retry: 447 /* Flush any pending resume work to create ev_fence */ 448 flush_delayed_work(&uq_mgr->resume_work); 449 450 mutex_lock(&uq_mgr->userq_mutex); 451 ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr); 452 if (dma_fence_is_signaled(ev_fence)) { 453 dma_fence_put(ev_fence); 454 mutex_unlock(&uq_mgr->userq_mutex); 455 /* 456 * Looks like there was no pending resume work, 457 * add one now to create a valid eviction fence 458 */ 459 schedule_delayed_work(&uq_mgr->resume_work, 0); 460 goto retry; 461 } 462 dma_fence_put(ev_fence); 463 } 464 465 466 467 static int 468 amdgpu_userq_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr, 469 struct amdgpu_db_info *db_info, 470 struct drm_file *filp, 471 u64 *index) 472 { 473 u64 doorbell_index; 474 struct drm_gem_object *gobj; 475 struct amdgpu_userq_obj *db_obj = db_info->db_obj; 476 struct amdgpu_bo *abo; 477 int r, db_size; 478 479 gobj = drm_gem_object_lookup(filp, db_info->doorbell_handle); 480 if (gobj == NULL) { 481 drm_file_err(uq_mgr->file, "Can't find GEM object for doorbell\n"); 482 return -EINVAL; 483 } 484 485 /* 486 * Pinning a regular BO into the doorbell domain would discard its 487 * contents, possibly those of a buffer shared by another client. 488 */ 489 abo = gem_to_amdgpu_bo(gobj); 490 if (!(abo->preferred_domains & AMDGPU_GEM_DOMAIN_DOORBELL)) { 491 drm_gem_object_put(gobj); 492 return -EINVAL; 493 } 494 495 db_obj->obj = amdgpu_bo_ref(abo); 496 drm_gem_object_put(gobj); 497 498 r = amdgpu_bo_reserve(db_obj->obj, true); 499 if (r) { 500 drm_file_err(uq_mgr->file, "[Usermode queues] Failed to pin doorbell object\n"); 501 goto unref_bo; 502 } 503 504 /* Pin the BO before generating the index, unpin in queue destroy */ 505 r = amdgpu_bo_pin(db_obj->obj, AMDGPU_GEM_DOMAIN_DOORBELL); 506 if (r) { 507 drm_file_err(uq_mgr->file, "[Usermode queues] Failed to pin doorbell object\n"); 508 goto unresv_bo; 509 } 510 511 switch (db_info->queue_type) { 512 case AMDGPU_HW_IP_GFX: 513 case AMDGPU_HW_IP_COMPUTE: 514 case AMDGPU_HW_IP_DMA: 515 db_size = sizeof(u64); 516 break; 517 default: 518 drm_file_err(uq_mgr->file, "[Usermode queues] IP %d not support\n", 519 db_info->queue_type); 520 r = -EINVAL; 521 goto unpin_bo; 522 } 523 524 /* Validate doorbell_offset is within the doorbell BO */ 525 if ((u64)db_info->doorbell_offset * db_size + db_size > 526 amdgpu_bo_size(db_obj->obj)) { 527 r = -EINVAL; 528 goto unpin_bo; 529 } 530 531 doorbell_index = amdgpu_doorbell_index_on_bar(uq_mgr->adev, db_obj->obj, 532 db_info->doorbell_offset, db_size); 533 drm_dbg_driver(adev_to_drm(uq_mgr->adev), 534 "[Usermode queues] doorbell index=%lld\n", doorbell_index); 535 amdgpu_bo_unreserve(db_obj->obj); 536 *index = doorbell_index; 537 return 0; 538 539 unpin_bo: 540 amdgpu_bo_unpin(db_obj->obj); 541 unresv_bo: 542 amdgpu_bo_unreserve(db_obj->obj); 543 unref_bo: 544 amdgpu_bo_unref(&db_obj->obj); 545 return r; 546 } 547 548 static int 549 amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_queue *queue) 550 { 551 struct amdgpu_device *adev = uq_mgr->adev; 552 const struct amdgpu_userq_funcs *uq_funcs = adev->userq_funcs[queue->queue_type]; 553 int r = 0; 554 555 trace_amdgpu_userq_destroy_start(queue); 556 557 cancel_delayed_work_sync(&uq_mgr->resume_work); 558 /* Cancel before taking userq_mutex: cancel_delayed_work_sync() waits 559 * for any running instance, which itself takes userq_mutex. 560 */ 561 cancel_delayed_work_sync(&queue->hang_detect_work); 562 563 mutex_lock(&uq_mgr->userq_mutex); 564 amdgpu_userq_wait_for_last_fence(queue); 565 566 amdgpu_userq_detach_doorbell(queue); 567 568 #if defined(CONFIG_DEBUG_FS) 569 debugfs_remove_recursive(queue->debugfs_queue); 570 #endif 571 r = amdgpu_userq_unmap_helper(queue); 572 atomic_dec(&uq_mgr->userq_count[queue->queue_type]); 573 amdgpu_userq_fence_driver_free(queue); 574 queue->fence_drv = NULL; 575 mutex_unlock(&uq_mgr->userq_mutex); 576 577 /* 578 * A failed unmap means MES could not remove the hung queue and is now 579 * unresponsive. Recover the GPU here so the wedged MES does not fail 580 * the next, unrelated queue submission and trigger a reset attributed 581 * to an innocent workload. 582 */ 583 if (r) 584 queue_work(adev->reset_domain->wq, &uq_mgr->reset_work); 585 586 uq_funcs->mqd_destroy(queue); 587 queue->userq_mgr = NULL; 588 589 amdgpu_bo_reserve(queue->db_obj.obj, true); 590 amdgpu_bo_unpin(queue->db_obj.obj); 591 amdgpu_bo_unreserve(queue->db_obj.obj); 592 amdgpu_bo_unref(&queue->db_obj.obj); 593 594 trace_amdgpu_userq_destroy_end(queue, r); 595 kfree(queue); 596 597 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 598 599 return r; 600 } 601 602 static void amdgpu_userq_kref_destroy(struct kref *kref) 603 { 604 int r; 605 struct amdgpu_usermode_queue *queue = 606 container_of(kref, struct amdgpu_usermode_queue, refcount); 607 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; 608 609 r = amdgpu_userq_destroy(uq_mgr, queue); 610 if (r) 611 drm_file_err(uq_mgr->file, "Failed to destroy usermode queue %d\n", r); 612 } 613 614 struct amdgpu_usermode_queue *amdgpu_userq_get(struct amdgpu_userq_mgr *uq_mgr, u32 qid) 615 { 616 struct amdgpu_usermode_queue *queue; 617 618 xa_lock(&uq_mgr->userq_xa); 619 queue = xa_load(&uq_mgr->userq_xa, qid); 620 if (queue) 621 kref_get(&queue->refcount); 622 xa_unlock(&uq_mgr->userq_xa); 623 624 return queue; 625 } 626 627 void amdgpu_userq_put(struct amdgpu_usermode_queue *queue) 628 { 629 if (queue) 630 kref_put(&queue->refcount, amdgpu_userq_kref_destroy); 631 } 632 633 static int amdgpu_userq_priority_permit(struct drm_file *filp, 634 int priority) 635 { 636 if (priority < AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_HIGH) 637 return 0; 638 639 if (capable(CAP_SYS_NICE)) 640 return 0; 641 642 if (drm_is_current_master(filp)) 643 return 0; 644 645 return -EACCES; 646 } 647 648 static int 649 amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) 650 { 651 struct amdgpu_fpriv *fpriv = filp->driver_priv; 652 struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr; 653 struct amdgpu_device *adev = uq_mgr->adev; 654 const struct amdgpu_userq_funcs *uq_funcs; 655 struct amdgpu_usermode_queue *queue; 656 struct amdgpu_db_info db_info; 657 uint64_t index; 658 int priority; 659 u32 qid; 660 int r; 661 662 priority = 663 (args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK) 664 >> AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT; 665 r = amdgpu_userq_priority_permit(filp, priority); 666 if (r) 667 return r; 668 669 r = pm_runtime_resume_and_get(adev_to_drm(adev)->dev); 670 if (r < 0) { 671 drm_file_err(uq_mgr->file, "pm_runtime_resume_and_get() failed for userqueue create\n"); 672 return r; 673 } 674 675 uq_funcs = adev->userq_funcs[args->in.ip_type]; 676 if (!uq_funcs) { 677 switch (args->in.ip_type) { 678 case AMDGPU_HW_IP_GFX: 679 case AMDGPU_HW_IP_COMPUTE: 680 dev_warn_once(adev->dev, 681 "Usermode queues for GFX/COMPUTE is not supported by the fw " 682 "on this ASIC (me: %u, pfp: %u, mec: %u, mes: %u)\n", 683 adev->gfx.me_fw_version, adev->gfx.pfp_fw_version, 684 adev->gfx.mec_fw_version, adev->mes.fw_version[0]); 685 break; 686 case AMDGPU_HW_IP_DMA: 687 dev_warn_once(adev->dev, 688 "Usermode queues for SDMA is not supported by the fw " 689 "on this ASIC (sdma: %u)\n", 690 adev->sdma.instance[0].fw_version); 691 break; 692 default: 693 break; 694 } 695 r = -EINVAL; 696 goto err_pm_runtime; 697 } 698 699 queue = kzalloc_obj(struct amdgpu_usermode_queue); 700 if (!queue) { 701 r = -ENOMEM; 702 goto err_pm_runtime; 703 } 704 705 kref_init(&queue->refcount); 706 queue->doorbell_handle = args->in.doorbell_handle; 707 queue->queue_type = args->in.ip_type; 708 queue->vm = &fpriv->vm; 709 queue->priority = priority; 710 queue->xcp_id = (fpriv->xcp_id != AMDGPU_XCP_NO_PARTITION) ? 711 fpriv->xcp_id : 0; 712 queue->userq_mgr = uq_mgr; 713 INIT_DELAYED_WORK(&queue->hang_detect_work, 714 amdgpu_userq_hang_detect_work); 715 716 r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv); 717 if (r) 718 goto free_queue; 719 720 xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC); 721 mutex_init(&queue->fence_drv_lock); 722 /* Make sure the queue can actually run with those virtual addresses. */ 723 r = amdgpu_bo_reserve(fpriv->vm.root.bo, false); 724 if (r) 725 goto free_fence_drv; 726 727 if (amdgpu_userq_input_va_validate(adev, queue, args->in.queue_va, 728 args->in.queue_size, 729 &queue->userq_vas.va.queue_rb) || 730 amdgpu_userq_input_va_validate(adev, queue, args->in.rptr_va, 731 sizeof(u64), 732 &queue->userq_vas.va.rptr) || 733 amdgpu_userq_input_va_validate(adev, queue, args->in.wptr_va, 734 sizeof(u64), 735 &queue->userq_vas.va.wptr)) { 736 r = -EINVAL; 737 amdgpu_bo_unreserve(fpriv->vm.root.bo); 738 goto free_fence_drv; 739 } 740 amdgpu_bo_unreserve(fpriv->vm.root.bo); 741 742 /* Convert relative doorbell offset into absolute doorbell index */ 743 db_info.queue_type = queue->queue_type; 744 db_info.doorbell_handle = queue->doorbell_handle; 745 db_info.db_obj = &queue->db_obj; 746 db_info.doorbell_offset = args->in.doorbell_offset; 747 r = amdgpu_userq_get_doorbell_index(uq_mgr, &db_info, filp, &index); 748 if (r) { 749 drm_file_err(uq_mgr->file, "Failed to get doorbell for queue\n"); 750 goto free_fence_drv; 751 } 752 753 queue->doorbell_index = index; 754 queue->doorbell_offset = (u32)args->in.doorbell_offset; 755 trace_amdgpu_userq_create_start(queue); 756 r = uq_funcs->mqd_create(queue, &args->in); 757 if (r) { 758 drm_file_err(uq_mgr->file, "Failed to create Queue\n"); 759 goto clean_doorbell_bo; 760 } 761 762 /* Update VM owner at userq submit-time for page-fault attribution. */ 763 amdgpu_vm_set_task_info(&fpriv->vm); 764 765 r = xa_insert_irq(&adev->userq_doorbell_xa, index, queue, 766 GFP_KERNEL); 767 if (r) 768 goto clean_mqd; 769 770 amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr); 771 772 /* don't map the queue if scheduling is halted */ 773 if (!adev->userq_halt_for_enforce_isolation || 774 ((queue->queue_type != AMDGPU_HW_IP_GFX) && 775 (queue->queue_type != AMDGPU_HW_IP_COMPUTE))) { 776 r = amdgpu_userq_map_helper(queue); 777 if (r) { 778 drm_file_err(uq_mgr->file, "Failed to map Queue\n"); 779 trace_amdgpu_userq_create_end(queue, r); 780 mutex_unlock(&uq_mgr->userq_mutex); 781 goto erase_doorbell; 782 } 783 } 784 785 atomic_inc(&uq_mgr->userq_count[queue->queue_type]); 786 mutex_unlock(&uq_mgr->userq_mutex); 787 788 r = xa_alloc(&uq_mgr->userq_xa, &qid, queue, 789 XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT), 790 GFP_KERNEL); 791 if (r) { 792 /* 793 * This drops the last reference which should take care of 794 * all cleanup. 795 */ 796 trace_amdgpu_userq_create_end(queue, r); 797 amdgpu_userq_put(queue); 798 return r; 799 } 800 801 amdgpu_debugfs_userq_init(filp, queue, qid); 802 trace_amdgpu_userq_create_end(queue, 0); 803 args->out.queue_id = qid; 804 return 0; 805 806 erase_doorbell: 807 xa_erase_irq(&adev->userq_doorbell_xa, index); 808 clean_mqd: 809 uq_funcs->mqd_destroy(queue); 810 clean_doorbell_bo: 811 amdgpu_bo_reserve(queue->db_obj.obj, true); 812 amdgpu_bo_unpin(queue->db_obj.obj); 813 amdgpu_bo_unreserve(queue->db_obj.obj); 814 amdgpu_bo_unref(&queue->db_obj.obj); 815 free_fence_drv: 816 amdgpu_userq_fence_driver_free(queue); 817 free_queue: 818 trace_amdgpu_userq_create_end(queue, r); 819 kfree(queue); 820 err_pm_runtime: 821 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 822 return r; 823 } 824 825 static int amdgpu_userq_input_args_validate(struct drm_device *dev, 826 union drm_amdgpu_userq *args, 827 struct drm_file *filp) 828 { 829 struct amdgpu_device *adev = drm_to_adev(dev); 830 831 switch (args->in.op) { 832 case AMDGPU_USERQ_OP_CREATE: 833 if (args->in.flags & ~(AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK | 834 AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE)) 835 return -EINVAL; 836 /* Usermode queues are only supported for GFX IP as of now */ 837 if (args->in.ip_type != AMDGPU_HW_IP_GFX && 838 args->in.ip_type != AMDGPU_HW_IP_DMA && 839 args->in.ip_type != AMDGPU_HW_IP_COMPUTE) { 840 drm_file_err(filp, "Usermode queue doesn't support IP type %u\n", 841 args->in.ip_type); 842 return -EINVAL; 843 } 844 845 if ((args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE) && 846 (args->in.ip_type != AMDGPU_HW_IP_GFX) && 847 (args->in.ip_type != AMDGPU_HW_IP_COMPUTE) && 848 !amdgpu_is_tmz(adev)) { 849 drm_file_err(filp, "Secure only supported on GFX/Compute queues\n"); 850 return -EINVAL; 851 } 852 853 if (args->in.queue_va == AMDGPU_BO_INVALID_OFFSET || 854 args->in.queue_va == 0 || 855 args->in.queue_size == 0) { 856 drm_file_err(filp, "invalidate userq queue va or size\n"); 857 return -EINVAL; 858 } 859 860 if (!is_power_of_2(args->in.queue_size)) { 861 drm_file_err(filp, "Queue size must be a power of 2\n"); 862 return -EINVAL; 863 } 864 865 if (args->in.queue_size < AMDGPU_GPU_PAGE_SIZE) { 866 drm_file_err(filp, "Queue size smaller than AMDGPU_GPU_PAGE_SIZE\n"); 867 return -EINVAL; 868 } 869 870 if (!args->in.wptr_va || !args->in.rptr_va) { 871 drm_file_err(filp, "invalidate userq queue rptr or wptr\n"); 872 return -EINVAL; 873 } 874 875 if (!IS_ALIGNED(args->in.wptr_va, sizeof(u64)) || 876 !IS_ALIGNED(args->in.rptr_va, sizeof(u64))) { 877 drm_file_err(filp, "user queue rptr or wptr is not 8-byte aligned\n"); 878 return -EINVAL; 879 } 880 break; 881 case AMDGPU_USERQ_OP_FREE: 882 if (args->in.ip_type || 883 args->in.doorbell_handle || 884 args->in.doorbell_offset || 885 args->in.flags || 886 args->in.queue_va || 887 args->in.queue_size || 888 args->in.rptr_va || 889 args->in.wptr_va || 890 args->in.mqd || 891 args->in.mqd_size) 892 return -EINVAL; 893 break; 894 default: 895 return -EINVAL; 896 } 897 898 return 0; 899 } 900 901 bool amdgpu_userq_enabled(struct drm_device *dev) 902 { 903 struct amdgpu_device *adev = drm_to_adev(dev); 904 int i; 905 906 for (i = 0; i < AMDGPU_HW_IP_NUM; i++) { 907 if (adev->userq_funcs[i]) 908 return true; 909 } 910 911 return false; 912 } 913 914 int amdgpu_userq_ioctl(struct drm_device *dev, void *data, 915 struct drm_file *filp) 916 { 917 union drm_amdgpu_userq *args = data; 918 struct amdgpu_fpriv *fpriv = filp->driver_priv; 919 struct amdgpu_usermode_queue *queue; 920 int r = 0; 921 922 if (!amdgpu_userq_enabled(dev)) 923 return -ENOTSUPP; 924 925 if (amdgpu_userq_input_args_validate(dev, args, filp) < 0) 926 return -EINVAL; 927 928 switch (args->in.op) { 929 case AMDGPU_USERQ_OP_CREATE: 930 r = amdgpu_userq_create(filp, args); 931 if (r) 932 drm_file_err(filp, "Failed to create usermode queue\n"); 933 break; 934 935 case AMDGPU_USERQ_OP_FREE: { 936 xa_lock(&fpriv->userq_mgr.userq_xa); 937 queue = __xa_erase(&fpriv->userq_mgr.userq_xa, args->in.queue_id); 938 xa_unlock(&fpriv->userq_mgr.userq_xa); 939 if (!queue) 940 return -ENOENT; 941 942 amdgpu_userq_put(queue); 943 break; 944 } 945 946 default: 947 drm_dbg_driver(dev, "Invalid user queue op specified: %d\n", args->in.op); 948 return -EINVAL; 949 } 950 951 return r; 952 } 953 954 static int 955 amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr) 956 { 957 struct amdgpu_usermode_queue *queue; 958 unsigned long queue_id; 959 int ret = 0, r; 960 961 mutex_lock(&uq_mgr->userq_mutex); 962 /* Resume all the queues for this process */ 963 xa_for_each(&uq_mgr->userq_xa, queue_id, queue) { 964 965 if (!amdgpu_userq_buffer_vas_mapped(queue)) { 966 drm_file_err(uq_mgr->file, 967 "trying restore queue without va mapping\n"); 968 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_INVALID_VA); 969 queue->state = AMDGPU_USERQ_STATE_INVALID_VA; 970 continue; 971 } 972 973 r = amdgpu_userq_map_helper(queue); 974 if (r) 975 ret = r; 976 } 977 mutex_unlock(&uq_mgr->userq_mutex); 978 979 if (ret) 980 drm_file_err(uq_mgr->file, 981 "Failed to map all the queues, restore failed ret=%d\n", ret); 982 return ret; 983 } 984 985 static int amdgpu_userq_validate_vm(void *param, struct amdgpu_bo *bo) 986 { 987 struct ttm_operation_ctx ctx = { false, false }; 988 989 amdgpu_bo_placement_from_domain(bo, bo->allowed_domains); 990 return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 991 } 992 993 /* 994 * Handle all individual BOs which were moved, lock and validate them and 995 * update the PTs. An evicted individual BO lands on the needs_update list, 996 * the evicted one only holds imported DMA-bufs of compute VMs. 997 */ 998 static int 999 amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec, 1000 struct amdgpu_vm *vm) 1001 { 1002 struct ttm_operation_ctx ctx = { false, false }; 1003 struct amdgpu_bo_va *bo_va; 1004 struct amdgpu_bo *bo, *tmp; 1005 int ret; 1006 1007 spin_lock(&vm->individual_lock); 1008 while (!list_empty(&vm->individual.needs_update)) { 1009 bo_va = list_first_entry(&vm->individual.needs_update, 1010 struct amdgpu_bo_va, 1011 base.vm_status); 1012 bo = bo_va->base.bo; 1013 tmp = amdgpu_bo_ref(bo); 1014 spin_unlock(&vm->individual_lock); 1015 1016 ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 1017 TTM_NUM_MOVE_FENCES + 1); 1018 amdgpu_bo_unref(&tmp); 1019 if (unlikely(ret)) 1020 return ret; 1021 1022 amdgpu_bo_placement_from_domain(bo, bo->allowed_domains); 1023 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 1024 if (ret) 1025 return ret; 1026 1027 /* This moves the bo_va to the idle list */ 1028 ret = amdgpu_vm_bo_update(adev, bo_va, false); 1029 if (ret) 1030 return ret; 1031 1032 spin_lock(&vm->individual_lock); 1033 } 1034 spin_unlock(&vm->individual_lock); 1035 1036 return 0; 1037 } 1038 1039 /* Make sure the whole VM is ready to be used */ 1040 static int 1041 amdgpu_userq_vm_validate_and_restore_queue(struct amdgpu_userq_mgr *uq_mgr) 1042 { 1043 struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); 1044 bool invalidated = false, new_addition = false; 1045 struct ttm_operation_ctx ctx = { true, false }; 1046 struct amdgpu_device *adev = uq_mgr->adev; 1047 struct amdgpu_hmm_range *range; 1048 struct amdgpu_vm *vm = &fpriv->vm; 1049 unsigned long key, tmp_key; 1050 struct amdgpu_bo_va *bo_va; 1051 struct amdgpu_usermode_queue *queue; 1052 struct amdgpu_bo *bo; 1053 struct drm_exec exec; 1054 struct xarray xa; 1055 int ret; 1056 1057 xa_init(&xa); 1058 1059 retry_lock: 1060 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0); 1061 drm_exec_until_all_locked(&exec) { 1062 /* 1063 * Rearm adds one BOOKKEEP fence and validation may queue move fences 1064 * on the root PD BO, so reserve caller-side slots accordingly. 1065 */ 1066 ret = amdgpu_vm_lock_pd(vm, &exec, TTM_NUM_MOVE_FENCES + 1); 1067 drm_exec_retry_on_contention(&exec); 1068 if (unlikely(ret)) 1069 goto unlock_all; 1070 1071 ret = amdgpu_vm_lock_individual(vm, &exec, TTM_NUM_MOVE_FENCES + 1); 1072 drm_exec_retry_on_contention(&exec); 1073 if (unlikely(ret)) 1074 goto unlock_all; 1075 1076 /* This validates PDs, PTs and per VM BOs */ 1077 ret = amdgpu_vm_validate(adev, vm, NULL, 1078 amdgpu_userq_validate_vm, 1079 NULL); 1080 if (unlikely(ret)) 1081 goto unlock_all; 1082 1083 /* This locks and validates the remaining evicted BOs */ 1084 ret = amdgpu_userq_bo_validate(adev, &exec, vm); 1085 drm_exec_retry_on_contention(&exec); 1086 if (unlikely(ret)) 1087 goto unlock_all; 1088 1089 /* 1090 * WPTR BOs are VM-mapped, but each BO has its own reservation 1091 * object. Lock them into this drm_exec ww context so the later 1092 * amdgpu_bo_gpu_offset() reads are done with the BO resv locked. 1093 */ 1094 xa_for_each(&uq_mgr->userq_xa, tmp_key, queue) { 1095 struct ttm_operation_ctx wptr_ctx = { false, false }; 1096 1097 bo = queue->wptr_obj.obj; 1098 if (!bo) 1099 continue; 1100 1101 ret = drm_exec_prepare_obj(&exec, &bo->tbo.base, 1102 TTM_NUM_MOVE_FENCES + 1); 1103 drm_exec_retry_on_contention(&exec); 1104 if (unlikely(ret)) 1105 goto unlock_all; 1106 1107 amdgpu_bo_placement_from_domain(bo, bo->allowed_domains); 1108 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &wptr_ctx); 1109 if (unlikely(ret)) 1110 goto unlock_all; 1111 } 1112 } 1113 1114 if (invalidated) { 1115 xa_for_each(&xa, tmp_key, range) { 1116 bo = range->bo; 1117 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 1118 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 1119 if (ret) 1120 goto unlock_all; 1121 1122 amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, range); 1123 1124 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 1125 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 1126 if (ret) 1127 goto unlock_all; 1128 } 1129 invalidated = false; 1130 } 1131 1132 ret = amdgpu_vm_handle_moved(adev, vm, NULL); 1133 if (ret) 1134 goto unlock_all; 1135 1136 /* 1137 * PRT/sparse mappings are kept off the vm_bo state lists, so 1138 * amdgpu_vm_handle_moved() does not touch them. Refresh their PTEs 1139 * explicitly here (as the CS path does) so sparse mappings survive a 1140 * VRAM-lost reset. 1141 */ 1142 ret = amdgpu_vm_bo_update(adev, fpriv->prt_va, false); 1143 if (ret) 1144 goto unlock_all; 1145 1146 key = 0; 1147 /* Validate User Ptr BOs */ 1148 list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status) { 1149 bo = bo_va->base.bo; 1150 if (!bo) 1151 continue; 1152 1153 if (!amdgpu_ttm_tt_is_userptr(bo->tbo.ttm)) 1154 continue; 1155 1156 range = xa_load(&xa, key); 1157 if (range && range->bo != bo) { 1158 xa_erase(&xa, key); 1159 amdgpu_hmm_range_free(range); 1160 range = NULL; 1161 } 1162 1163 if (!range) { 1164 range = amdgpu_hmm_range_alloc(bo); 1165 if (!range) { 1166 ret = -ENOMEM; 1167 goto unlock_all; 1168 } 1169 1170 xa_store(&xa, key, range, GFP_KERNEL); 1171 new_addition = true; 1172 } 1173 key++; 1174 } 1175 1176 if (new_addition) { 1177 drm_exec_fini(&exec); 1178 xa_for_each(&xa, tmp_key, range) { 1179 if (!range) 1180 continue; 1181 bo = range->bo; 1182 ret = amdgpu_ttm_tt_get_user_pages(bo, range); 1183 if (ret) 1184 goto free_ranges; 1185 } 1186 1187 invalidated = true; 1188 new_addition = false; 1189 goto retry_lock; 1190 } 1191 1192 ret = amdgpu_vm_update_pdes(adev, vm, false); 1193 if (ret) 1194 goto unlock_all; 1195 1196 /* 1197 * We need to wait for all VM updates to finish before restarting the 1198 * queues. Using the idle list like that is now ok since everything is 1199 * locked in place. 1200 */ 1201 list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status) 1202 dma_fence_wait(bo_va->last_pt_update, false); 1203 /* 1204 * The PRT bo_va is kept off the state lists, so its PTE update fence 1205 * lands in prt_va->last_pt_update rather than vm->last_update; wait on 1206 * it explicitly (as the CS path syncs it) before restarting queues. 1207 */ 1208 dma_fence_wait(fpriv->prt_va->last_pt_update, false); 1209 dma_fence_wait(vm->last_update, false); 1210 1211 xa_for_each(&uq_mgr->userq_xa, tmp_key, queue) { 1212 bo = queue->wptr_obj.obj; 1213 if (!bo) { 1214 ret = -EINVAL; 1215 goto unlock_all; 1216 } 1217 1218 ret = amdgpu_ttm_alloc_gart(&bo->tbo); 1219 if (unlikely(ret)) { 1220 drm_file_err(uq_mgr->file, 1221 "failed to bind wptr bo to gart on resume, qid=%lu ret=%d\n", 1222 tmp_key, ret); 1223 goto unlock_all; 1224 } 1225 1226 queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(bo); 1227 } 1228 1229 ret = amdgpu_evf_mgr_rearm(&fpriv->evf_mgr, &exec); 1230 if (ret) { 1231 drm_file_err(uq_mgr->file, "Failed to replace eviction fence\n"); 1232 goto unlock_all; 1233 } 1234 1235 ret = amdgpu_userq_restore_all(uq_mgr); 1236 1237 unlock_all: 1238 drm_exec_fini(&exec); 1239 free_ranges: 1240 xa_for_each(&xa, tmp_key, range) { 1241 if (!range) 1242 continue; 1243 bo = range->bo; 1244 amdgpu_hmm_range_free(range); 1245 } 1246 xa_destroy(&xa); 1247 return ret; 1248 } 1249 1250 static void amdgpu_userq_restore_worker(struct work_struct *work) 1251 { 1252 struct amdgpu_userq_mgr *uq_mgr = work_to_uq_mgr(work, resume_work.work); 1253 struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); 1254 struct dma_fence *ev_fence; 1255 int ret; 1256 1257 ev_fence = amdgpu_evf_mgr_get_fence(&fpriv->evf_mgr); 1258 if (!dma_fence_is_signaled(ev_fence)) 1259 goto put_fence; 1260 1261 ret = amdgpu_userq_vm_validate_and_restore_queue(uq_mgr); 1262 if (ret) { 1263 drm_file_err(uq_mgr->file, "Failed to validate BOs to restore ret=%d\n", ret); 1264 goto put_fence; 1265 } 1266 1267 put_fence: 1268 dma_fence_put(ev_fence); 1269 } 1270 1271 void amdgpu_userq_process_reset_irq(struct amdgpu_device *adev, 1272 u32 pasid, u32 doorbell_offset) 1273 { 1274 struct xarray *xa = &adev->userq_doorbell_xa; 1275 struct amdgpu_usermode_queue *queue; 1276 unsigned long flags, idx; 1277 1278 xa_lock_irqsave(xa, flags); 1279 xa_for_each(xa, idx, queue) { 1280 if (queue->vm && queue->vm->pasid == pasid && 1281 queue->doorbell_offset == doorbell_offset) { 1282 amdgpu_userq_start_hang_detect_work(queue); 1283 break; 1284 } 1285 } 1286 xa_unlock_irqrestore(xa, flags); 1287 } 1288 1289 static int 1290 amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr) 1291 { 1292 struct amdgpu_usermode_queue *queue; 1293 unsigned long queue_id; 1294 int ret = 0, r; 1295 1296 /* Try to unmap all the queues in this process ctx */ 1297 xa_for_each(&uq_mgr->userq_xa, queue_id, queue) { 1298 r = amdgpu_userq_unmap_helper(queue); 1299 if (r) 1300 ret = r; 1301 } 1302 1303 if (ret) { 1304 drm_file_err(uq_mgr->file, 1305 "Couldn't unmap all the queues, eviction failed ret=%d\n", ret); 1306 amdgpu_reset_domain_schedule(uq_mgr->adev->reset_domain, 1307 &uq_mgr->reset_work); 1308 flush_work(&uq_mgr->reset_work); 1309 } 1310 return ret; 1311 } 1312 1313 void 1314 amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr) 1315 { 1316 struct amdgpu_usermode_queue *queue; 1317 unsigned long queue_id; 1318 1319 xa_for_each(&uq_mgr->userq_xa, queue_id, queue) { 1320 struct dma_fence *f = queue->last_fence; 1321 1322 if (!f) 1323 continue; 1324 1325 dma_fence_wait(f, false); 1326 } 1327 } 1328 1329 void 1330 amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr) 1331 { 1332 amdgpu_userq_evict_all(uq_mgr); 1333 } 1334 1335 int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv, 1336 struct amdgpu_device *adev) 1337 { 1338 mutex_init(&userq_mgr->userq_mutex); 1339 xa_init_flags(&userq_mgr->userq_xa, XA_FLAGS_ALLOC); 1340 userq_mgr->adev = adev; 1341 userq_mgr->file = file_priv; 1342 userq_mgr->proc_ctx_allocated = false; 1343 mutex_init(&userq_mgr->proc_ctx_lock); 1344 1345 INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userq_restore_worker); 1346 INIT_WORK(&userq_mgr->reset_work, amdgpu_userq_mgr_reset_work); 1347 return 0; 1348 } 1349 1350 void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev) 1351 { 1352 struct xarray *xa = &adev->userq_doorbell_xa; 1353 struct amdgpu_usermode_queue *queue; 1354 unsigned long flags, queue_id; 1355 1356 xa_lock_irqsave(xa, flags); 1357 xa_for_each(xa, queue_id, queue) { 1358 cancel_delayed_work(&queue->hang_detect_work); 1359 cancel_work(&queue->userq_mgr->reset_work); 1360 } 1361 xa_unlock_irqrestore(xa, flags); 1362 } 1363 1364 void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr) 1365 { 1366 cancel_delayed_work_sync(&userq_mgr->resume_work); 1367 } 1368 1369 void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr) 1370 { 1371 struct amdgpu_mes *mes = &userq_mgr->adev->mes; 1372 struct amdgpu_usermode_queue *queue; 1373 unsigned long queue_id = 0; 1374 1375 for (;;) { 1376 xa_lock(&userq_mgr->userq_xa); 1377 queue = xa_find(&userq_mgr->userq_xa, &queue_id, ULONG_MAX, 1378 XA_PRESENT); 1379 if (queue) 1380 __xa_erase(&userq_mgr->userq_xa, queue_id); 1381 xa_unlock(&userq_mgr->userq_xa); 1382 1383 if (!queue) 1384 break; 1385 1386 amdgpu_userq_put(queue); 1387 } 1388 1389 xa_destroy(&userq_mgr->userq_xa); 1390 1391 /* 1392 * Drain any in-flight reset_work. By this point all queues are freed 1393 * and userq_count is 0, so if reset_work starts now it exits early. 1394 * We still need to wait in case it was already executing gpu_recover. 1395 */ 1396 cancel_work_sync(&userq_mgr->reset_work); 1397 1398 if (userq_mgr->proc_ctx_allocated) { 1399 amdgpu_mes_free_proc_ctx_index(mes, userq_mgr->proc_ctx_array_index); 1400 userq_mgr->proc_ctx_allocated = false; 1401 } 1402 amdgpu_bo_free_kernel(&userq_mgr->proc_ctx_obj.obj, 1403 &userq_mgr->proc_ctx_obj.gpu_addr, 1404 &userq_mgr->proc_ctx_obj.cpu_ptr); 1405 1406 mutex_destroy(&userq_mgr->proc_ctx_lock); 1407 mutex_destroy(&userq_mgr->userq_mutex); 1408 } 1409 1410 int amdgpu_userq_suspend(struct amdgpu_device *adev) 1411 { 1412 struct amdgpu_usermode_queue *queue; 1413 struct amdgpu_userq_mgr *uqm; 1414 unsigned long queue_id; 1415 int r; 1416 1417 xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) { 1418 uqm = queue->userq_mgr; 1419 cancel_delayed_work_sync(&uqm->resume_work); 1420 guard(mutex)(&uqm->userq_mutex); 1421 if (adev->in_s0ix) 1422 r = amdgpu_userq_preempt_helper(queue); 1423 else 1424 r = amdgpu_userq_unmap_helper(queue); 1425 if (r) 1426 return r; 1427 } 1428 return 0; 1429 } 1430 1431 int amdgpu_userq_resume(struct amdgpu_device *adev) 1432 { 1433 struct amdgpu_usermode_queue *queue; 1434 struct amdgpu_userq_mgr *uqm; 1435 unsigned long queue_id; 1436 int r; 1437 1438 xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) { 1439 uqm = queue->userq_mgr; 1440 guard(mutex)(&uqm->userq_mutex); 1441 if (adev->in_s0ix) 1442 r = amdgpu_userq_restore_helper(queue); 1443 else 1444 r = amdgpu_userq_map_helper(queue); 1445 if (r) 1446 return r; 1447 } 1448 1449 return 0; 1450 } 1451 1452 int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev, 1453 u32 idx) 1454 { 1455 u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev); 1456 struct amdgpu_usermode_queue *queue; 1457 struct amdgpu_userq_mgr *uqm; 1458 unsigned long queue_id; 1459 int ret = 0, r; 1460 1461 /* only need to stop gfx/compute */ 1462 if (!(ip_mask & ((1 << AMDGPU_HW_IP_GFX) | (1 << AMDGPU_HW_IP_COMPUTE)))) 1463 return 0; 1464 1465 if (adev->userq_halt_for_enforce_isolation) 1466 dev_warn(adev->dev, "userq scheduling already stopped!\n"); 1467 adev->userq_halt_for_enforce_isolation = true; 1468 xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) { 1469 uqm = queue->userq_mgr; 1470 cancel_delayed_work_sync(&uqm->resume_work); 1471 mutex_lock(&uqm->userq_mutex); 1472 if (((queue->queue_type == AMDGPU_HW_IP_GFX) || 1473 (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) && 1474 (queue->xcp_id == idx)) { 1475 r = amdgpu_userq_preempt_helper(queue); 1476 if (r) 1477 ret = r; 1478 } 1479 mutex_unlock(&uqm->userq_mutex); 1480 } 1481 1482 return ret; 1483 } 1484 1485 int amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device *adev, 1486 u32 idx) 1487 { 1488 u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev); 1489 struct amdgpu_usermode_queue *queue; 1490 struct amdgpu_userq_mgr *uqm; 1491 unsigned long queue_id; 1492 int ret = 0, r; 1493 1494 /* only need to stop gfx/compute */ 1495 if (!(ip_mask & ((1 << AMDGPU_HW_IP_GFX) | (1 << AMDGPU_HW_IP_COMPUTE)))) 1496 return 0; 1497 1498 if (!adev->userq_halt_for_enforce_isolation) 1499 dev_warn(adev->dev, "userq scheduling already started!\n"); 1500 1501 adev->userq_halt_for_enforce_isolation = false; 1502 1503 xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) { 1504 uqm = queue->userq_mgr; 1505 mutex_lock(&uqm->userq_mutex); 1506 if (((queue->queue_type == AMDGPU_HW_IP_GFX) || 1507 (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) && 1508 (queue->xcp_id == idx)) { 1509 r = amdgpu_userq_restore_helper(queue); 1510 if (r) 1511 ret = r; 1512 } 1513 mutex_unlock(&uqm->userq_mutex); 1514 } 1515 1516 return ret; 1517 } 1518 1519 void amdgpu_userq_gem_va_unmap_validate(struct amdgpu_device *adev, 1520 struct amdgpu_bo_va_mapping *mapping) 1521 { 1522 u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev); 1523 struct amdgpu_bo_va *bo_va = mapping->bo_va; 1524 struct dma_resv *resv = bo_va->base.bo->tbo.base.resv; 1525 1526 if (!ip_mask) 1527 return; 1528 1529 /** 1530 * The userq VA mapping reservation should include the eviction fence. 1531 * Note: The eviction fence may be attached to different BOs and this 1532 * unmap is only for one kind of userq VAs, so at this point suppose 1533 * the eviction fence is always unsignaled. 1534 */ 1535 dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP, 1536 false, MAX_SCHEDULE_TIMEOUT); 1537 } 1538 1539 void amdgpu_userq_pre_reset(struct amdgpu_device *adev) 1540 { 1541 const struct amdgpu_userq_funcs *userq_funcs; 1542 struct amdgpu_usermode_queue *queue; 1543 unsigned long queue_id; 1544 1545 /* TODO: We probably need a new lock for the queue state */ 1546 xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) { 1547 if (queue->state == AMDGPU_USERQ_STATE_MAPPED) { 1548 trace_amdgpu_userq_state_start(queue); 1549 userq_funcs = adev->userq_funcs[queue->queue_type]; 1550 userq_funcs->unmap(queue); 1551 /* just mark all queues as hung at this point. 1552 * if unmap succeeds, we could map again 1553 * in amdgpu_userq_post_reset() if vram is not lost 1554 */ 1555 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG); 1556 queue->state = AMDGPU_USERQ_STATE_HUNG; 1557 } 1558 /* Force-complete any pending fence regardless of queue state so 1559 * that eviction/suspend and queue teardown waiters don't block 1560 * forever on a fence that will never signal after the reset. 1561 */ 1562 amdgpu_userq_fence_driver_force_completion(queue); 1563 } 1564 } 1565 1566 int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost) 1567 { 1568 /* if any queue state is AMDGPU_USERQ_STATE_UNMAPPED 1569 * at this point, we should be able to map it again 1570 * and continue if vram is not lost. 1571 */ 1572 struct amdgpu_usermode_queue *queue; 1573 const struct amdgpu_userq_funcs *userq_funcs; 1574 unsigned long queue_id; 1575 int ret = 0, r; 1576 1577 xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) { 1578 if (queue->state == AMDGPU_USERQ_STATE_HUNG && !vram_lost) { 1579 trace_amdgpu_userq_state_start(queue); 1580 1581 userq_funcs = adev->userq_funcs[queue->queue_type]; 1582 /* Re-map queue */ 1583 r = userq_funcs->map(queue); 1584 if (r) { 1585 dev_err(adev->dev, "Failed to remap queue %ld\n", queue_id); 1586 ret = r; 1587 continue; 1588 } 1589 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED); 1590 queue->state = AMDGPU_USERQ_STATE_MAPPED; 1591 } 1592 } 1593 1594 return ret; 1595 } 1596