1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2023 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <drm/drm_auth.h> 26 #include <drm/drm_exec.h> 27 #include <linux/pm_runtime.h> 28 29 #include "amdgpu.h" 30 #include "amdgpu_vm.h" 31 #include "amdgpu_userq.h" 32 #include "amdgpu_userq_fence.h" 33 34 u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev) 35 { 36 int i; 37 u32 userq_ip_mask = 0; 38 39 for (i = 0; i < AMDGPU_HW_IP_NUM; i++) { 40 if (adev->userq_funcs[i]) 41 userq_ip_mask |= (1 << i); 42 } 43 44 return userq_ip_mask; 45 } 46 47 static int 48 amdgpu_userq_unmap_helper(struct amdgpu_userq_mgr *uq_mgr, 49 struct amdgpu_usermode_queue *queue) 50 { 51 struct amdgpu_device *adev = uq_mgr->adev; 52 const struct amdgpu_userq_funcs *userq_funcs = 53 adev->userq_funcs[queue->queue_type]; 54 int r = 0; 55 56 if (queue->state == AMDGPU_USERQ_STATE_MAPPED) { 57 r = userq_funcs->unmap(uq_mgr, queue); 58 if (r) 59 queue->state = AMDGPU_USERQ_STATE_HUNG; 60 else 61 queue->state = AMDGPU_USERQ_STATE_UNMAPPED; 62 } 63 return r; 64 } 65 66 static int 67 amdgpu_userq_map_helper(struct amdgpu_userq_mgr *uq_mgr, 68 struct amdgpu_usermode_queue *queue) 69 { 70 struct amdgpu_device *adev = uq_mgr->adev; 71 const struct amdgpu_userq_funcs *userq_funcs = 72 adev->userq_funcs[queue->queue_type]; 73 int r = 0; 74 75 if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) { 76 r = userq_funcs->map(uq_mgr, queue); 77 if (r) { 78 queue->state = AMDGPU_USERQ_STATE_HUNG; 79 } else { 80 queue->state = AMDGPU_USERQ_STATE_MAPPED; 81 } 82 } 83 return r; 84 } 85 86 static void 87 amdgpu_userq_wait_for_last_fence(struct amdgpu_userq_mgr *uq_mgr, 88 struct amdgpu_usermode_queue *queue) 89 { 90 struct dma_fence *f = queue->last_fence; 91 int ret; 92 93 if (f && !dma_fence_is_signaled(f)) { 94 ret = dma_fence_wait_timeout(f, true, msecs_to_jiffies(100)); 95 if (ret <= 0) 96 drm_file_err(uq_mgr->file, "Timed out waiting for fence=%llu:%llu\n", 97 f->context, f->seqno); 98 } 99 } 100 101 static void 102 amdgpu_userq_cleanup(struct amdgpu_userq_mgr *uq_mgr, 103 struct amdgpu_usermode_queue *queue, 104 int queue_id) 105 { 106 struct amdgpu_device *adev = uq_mgr->adev; 107 const struct amdgpu_userq_funcs *uq_funcs = adev->userq_funcs[queue->queue_type]; 108 109 uq_funcs->mqd_destroy(uq_mgr, queue); 110 amdgpu_userq_fence_driver_free(queue); 111 idr_remove(&uq_mgr->userq_idr, queue_id); 112 kfree(queue); 113 } 114 115 int 116 amdgpu_userq_active(struct amdgpu_userq_mgr *uq_mgr) 117 { 118 struct amdgpu_usermode_queue *queue; 119 int queue_id; 120 int ret = 0; 121 122 mutex_lock(&uq_mgr->userq_mutex); 123 /* Resume all the queues for this process */ 124 idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) 125 ret += queue->state == AMDGPU_USERQ_STATE_MAPPED; 126 127 mutex_unlock(&uq_mgr->userq_mutex); 128 return ret; 129 } 130 131 static struct amdgpu_usermode_queue * 132 amdgpu_userq_find(struct amdgpu_userq_mgr *uq_mgr, int qid) 133 { 134 return idr_find(&uq_mgr->userq_idr, qid); 135 } 136 137 void 138 amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr, 139 struct amdgpu_eviction_fence_mgr *evf_mgr) 140 { 141 struct amdgpu_eviction_fence *ev_fence; 142 143 retry: 144 /* Flush any pending resume work to create ev_fence */ 145 flush_delayed_work(&uq_mgr->resume_work); 146 147 mutex_lock(&uq_mgr->userq_mutex); 148 spin_lock(&evf_mgr->ev_fence_lock); 149 ev_fence = evf_mgr->ev_fence; 150 spin_unlock(&evf_mgr->ev_fence_lock); 151 if (!ev_fence || dma_fence_is_signaled(&ev_fence->base)) { 152 mutex_unlock(&uq_mgr->userq_mutex); 153 /* 154 * Looks like there was no pending resume work, 155 * add one now to create a valid eviction fence 156 */ 157 schedule_delayed_work(&uq_mgr->resume_work, 0); 158 goto retry; 159 } 160 } 161 162 int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr, 163 struct amdgpu_userq_obj *userq_obj, 164 int size) 165 { 166 struct amdgpu_device *adev = uq_mgr->adev; 167 struct amdgpu_bo_param bp; 168 int r; 169 170 memset(&bp, 0, sizeof(bp)); 171 bp.byte_align = PAGE_SIZE; 172 bp.domain = AMDGPU_GEM_DOMAIN_GTT; 173 bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | 174 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 175 bp.type = ttm_bo_type_kernel; 176 bp.size = size; 177 bp.resv = NULL; 178 bp.bo_ptr_size = sizeof(struct amdgpu_bo); 179 180 r = amdgpu_bo_create(adev, &bp, &userq_obj->obj); 181 if (r) { 182 drm_file_err(uq_mgr->file, "Failed to allocate BO for userqueue (%d)", r); 183 return r; 184 } 185 186 r = amdgpu_bo_reserve(userq_obj->obj, true); 187 if (r) { 188 drm_file_err(uq_mgr->file, "Failed to reserve BO to map (%d)", r); 189 goto free_obj; 190 } 191 192 r = amdgpu_ttm_alloc_gart(&(userq_obj->obj)->tbo); 193 if (r) { 194 drm_file_err(uq_mgr->file, "Failed to alloc GART for userqueue object (%d)", r); 195 goto unresv; 196 } 197 198 r = amdgpu_bo_kmap(userq_obj->obj, &userq_obj->cpu_ptr); 199 if (r) { 200 drm_file_err(uq_mgr->file, "Failed to map BO for userqueue (%d)", r); 201 goto unresv; 202 } 203 204 userq_obj->gpu_addr = amdgpu_bo_gpu_offset(userq_obj->obj); 205 amdgpu_bo_unreserve(userq_obj->obj); 206 memset(userq_obj->cpu_ptr, 0, size); 207 return 0; 208 209 unresv: 210 amdgpu_bo_unreserve(userq_obj->obj); 211 212 free_obj: 213 amdgpu_bo_unref(&userq_obj->obj); 214 return r; 215 } 216 217 void amdgpu_userq_destroy_object(struct amdgpu_userq_mgr *uq_mgr, 218 struct amdgpu_userq_obj *userq_obj) 219 { 220 amdgpu_bo_kunmap(userq_obj->obj); 221 amdgpu_bo_unref(&userq_obj->obj); 222 } 223 224 uint64_t 225 amdgpu_userq_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr, 226 struct amdgpu_db_info *db_info, 227 struct drm_file *filp) 228 { 229 uint64_t index; 230 struct drm_gem_object *gobj; 231 struct amdgpu_userq_obj *db_obj = db_info->db_obj; 232 int r, db_size; 233 234 gobj = drm_gem_object_lookup(filp, db_info->doorbell_handle); 235 if (gobj == NULL) { 236 drm_file_err(uq_mgr->file, "Can't find GEM object for doorbell\n"); 237 return -EINVAL; 238 } 239 240 db_obj->obj = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); 241 drm_gem_object_put(gobj); 242 243 r = amdgpu_bo_reserve(db_obj->obj, true); 244 if (r) { 245 drm_file_err(uq_mgr->file, "[Usermode queues] Failed to pin doorbell object\n"); 246 goto unref_bo; 247 } 248 249 /* Pin the BO before generating the index, unpin in queue destroy */ 250 r = amdgpu_bo_pin(db_obj->obj, AMDGPU_GEM_DOMAIN_DOORBELL); 251 if (r) { 252 drm_file_err(uq_mgr->file, "[Usermode queues] Failed to pin doorbell object\n"); 253 goto unresv_bo; 254 } 255 256 switch (db_info->queue_type) { 257 case AMDGPU_HW_IP_GFX: 258 case AMDGPU_HW_IP_COMPUTE: 259 case AMDGPU_HW_IP_DMA: 260 db_size = sizeof(u64); 261 break; 262 263 case AMDGPU_HW_IP_VCN_ENC: 264 db_size = sizeof(u32); 265 db_info->doorbell_offset += AMDGPU_NAVI10_DOORBELL64_VCN0_1 << 1; 266 break; 267 268 case AMDGPU_HW_IP_VPE: 269 db_size = sizeof(u32); 270 db_info->doorbell_offset += AMDGPU_NAVI10_DOORBELL64_VPE << 1; 271 break; 272 273 default: 274 drm_file_err(uq_mgr->file, "[Usermode queues] IP %d not support\n", 275 db_info->queue_type); 276 r = -EINVAL; 277 goto unpin_bo; 278 } 279 280 index = amdgpu_doorbell_index_on_bar(uq_mgr->adev, db_obj->obj, 281 db_info->doorbell_offset, db_size); 282 drm_dbg_driver(adev_to_drm(uq_mgr->adev), 283 "[Usermode queues] doorbell index=%lld\n", index); 284 amdgpu_bo_unreserve(db_obj->obj); 285 return index; 286 287 unpin_bo: 288 amdgpu_bo_unpin(db_obj->obj); 289 unresv_bo: 290 amdgpu_bo_unreserve(db_obj->obj); 291 unref_bo: 292 amdgpu_bo_unref(&db_obj->obj); 293 return r; 294 } 295 296 static int 297 amdgpu_userq_destroy(struct drm_file *filp, int queue_id) 298 { 299 struct amdgpu_fpriv *fpriv = filp->driver_priv; 300 struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr; 301 struct amdgpu_device *adev = uq_mgr->adev; 302 struct amdgpu_usermode_queue *queue; 303 int r = 0; 304 305 cancel_delayed_work_sync(&uq_mgr->resume_work); 306 mutex_lock(&uq_mgr->userq_mutex); 307 308 queue = amdgpu_userq_find(uq_mgr, queue_id); 309 if (!queue) { 310 drm_dbg_driver(adev_to_drm(uq_mgr->adev), "Invalid queue id to destroy\n"); 311 mutex_unlock(&uq_mgr->userq_mutex); 312 return -EINVAL; 313 } 314 amdgpu_userq_wait_for_last_fence(uq_mgr, queue); 315 r = amdgpu_bo_reserve(queue->db_obj.obj, true); 316 if (!r) { 317 amdgpu_bo_unpin(queue->db_obj.obj); 318 amdgpu_bo_unreserve(queue->db_obj.obj); 319 } 320 amdgpu_bo_unref(&queue->db_obj.obj); 321 322 #if defined(CONFIG_DEBUG_FS) 323 debugfs_remove_recursive(queue->debugfs_queue); 324 #endif 325 r = amdgpu_userq_unmap_helper(uq_mgr, queue); 326 amdgpu_userq_cleanup(uq_mgr, queue, queue_id); 327 mutex_unlock(&uq_mgr->userq_mutex); 328 329 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); 330 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 331 332 return r; 333 } 334 335 static int amdgpu_userq_priority_permit(struct drm_file *filp, 336 int priority) 337 { 338 if (priority < AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_HIGH) 339 return 0; 340 341 if (capable(CAP_SYS_NICE)) 342 return 0; 343 344 if (drm_is_current_master(filp)) 345 return 0; 346 347 return -EACCES; 348 } 349 350 #if defined(CONFIG_DEBUG_FS) 351 static int amdgpu_mqd_info_read(struct seq_file *m, void *unused) 352 { 353 struct amdgpu_usermode_queue *queue = m->private; 354 struct amdgpu_bo *bo; 355 int r; 356 357 if (!queue || !queue->mqd.obj) 358 return -EINVAL; 359 360 bo = amdgpu_bo_ref(queue->mqd.obj); 361 r = amdgpu_bo_reserve(bo, true); 362 if (r) { 363 amdgpu_bo_unref(&bo); 364 return -EINVAL; 365 } 366 367 seq_printf(m, "queue_type %d\n", queue->queue_type); 368 seq_printf(m, "mqd_gpu_address: 0x%llx\n", amdgpu_bo_gpu_offset(queue->mqd.obj)); 369 370 amdgpu_bo_unreserve(bo); 371 amdgpu_bo_unref(&bo); 372 373 return 0; 374 } 375 376 static int amdgpu_mqd_info_open(struct inode *inode, struct file *file) 377 { 378 return single_open(file, amdgpu_mqd_info_read, inode->i_private); 379 } 380 381 static const struct file_operations amdgpu_mqd_info_fops = { 382 .owner = THIS_MODULE, 383 .open = amdgpu_mqd_info_open, 384 .read = seq_read, 385 .llseek = seq_lseek, 386 .release = single_release, 387 }; 388 #endif 389 390 static int 391 amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) 392 { 393 struct amdgpu_fpriv *fpriv = filp->driver_priv; 394 struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr; 395 struct amdgpu_device *adev = uq_mgr->adev; 396 const struct amdgpu_userq_funcs *uq_funcs; 397 struct amdgpu_usermode_queue *queue; 398 struct amdgpu_db_info db_info; 399 char *queue_name; 400 bool skip_map_queue; 401 uint64_t index; 402 int qid, r = 0; 403 int priority = 404 (args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK) >> 405 AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT; 406 407 /* Usermode queues are only supported for GFX IP as of now */ 408 if (args->in.ip_type != AMDGPU_HW_IP_GFX && 409 args->in.ip_type != AMDGPU_HW_IP_DMA && 410 args->in.ip_type != AMDGPU_HW_IP_COMPUTE) { 411 drm_file_err(uq_mgr->file, "Usermode queue doesn't support IP type %u\n", 412 args->in.ip_type); 413 return -EINVAL; 414 } 415 416 r = amdgpu_userq_priority_permit(filp, priority); 417 if (r) 418 return r; 419 420 if ((args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE) && 421 (args->in.ip_type != AMDGPU_HW_IP_GFX) && 422 (args->in.ip_type != AMDGPU_HW_IP_COMPUTE) && 423 !amdgpu_is_tmz(adev)) { 424 drm_file_err(uq_mgr->file, "Secure only supported on GFX/Compute queues\n"); 425 return -EINVAL; 426 } 427 428 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 429 if (r < 0) { 430 drm_file_err(uq_mgr->file, "pm_runtime_get_sync() failed for userqueue create\n"); 431 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 432 return r; 433 } 434 435 /* 436 * There could be a situation that we are creating a new queue while 437 * the other queues under this UQ_mgr are suspended. So if there is any 438 * resume work pending, wait for it to get done. 439 * 440 * This will also make sure we have a valid eviction fence ready to be used. 441 */ 442 mutex_lock(&adev->userq_mutex); 443 amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr); 444 445 uq_funcs = adev->userq_funcs[args->in.ip_type]; 446 if (!uq_funcs) { 447 drm_file_err(uq_mgr->file, "Usermode queue is not supported for this IP (%u)\n", 448 args->in.ip_type); 449 r = -EINVAL; 450 goto unlock; 451 } 452 453 queue = kzalloc(sizeof(struct amdgpu_usermode_queue), GFP_KERNEL); 454 if (!queue) { 455 drm_file_err(uq_mgr->file, "Failed to allocate memory for queue\n"); 456 r = -ENOMEM; 457 goto unlock; 458 } 459 queue->doorbell_handle = args->in.doorbell_handle; 460 queue->queue_type = args->in.ip_type; 461 queue->vm = &fpriv->vm; 462 queue->priority = priority; 463 464 db_info.queue_type = queue->queue_type; 465 db_info.doorbell_handle = queue->doorbell_handle; 466 db_info.db_obj = &queue->db_obj; 467 db_info.doorbell_offset = args->in.doorbell_offset; 468 469 /* Convert relative doorbell offset into absolute doorbell index */ 470 index = amdgpu_userq_get_doorbell_index(uq_mgr, &db_info, filp); 471 if (index == (uint64_t)-EINVAL) { 472 drm_file_err(uq_mgr->file, "Failed to get doorbell for queue\n"); 473 kfree(queue); 474 goto unlock; 475 } 476 477 queue->doorbell_index = index; 478 xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC); 479 r = amdgpu_userq_fence_driver_alloc(adev, queue); 480 if (r) { 481 drm_file_err(uq_mgr->file, "Failed to alloc fence driver\n"); 482 goto unlock; 483 } 484 485 r = uq_funcs->mqd_create(uq_mgr, &args->in, queue); 486 if (r) { 487 drm_file_err(uq_mgr->file, "Failed to create Queue\n"); 488 amdgpu_userq_fence_driver_free(queue); 489 kfree(queue); 490 goto unlock; 491 } 492 493 494 qid = idr_alloc(&uq_mgr->userq_idr, queue, 1, AMDGPU_MAX_USERQ_COUNT, GFP_KERNEL); 495 if (qid < 0) { 496 drm_file_err(uq_mgr->file, "Failed to allocate a queue id\n"); 497 amdgpu_userq_fence_driver_free(queue); 498 uq_funcs->mqd_destroy(uq_mgr, queue); 499 kfree(queue); 500 r = -ENOMEM; 501 goto unlock; 502 } 503 504 /* don't map the queue if scheduling is halted */ 505 if (adev->userq_halt_for_enforce_isolation && 506 ((queue->queue_type == AMDGPU_HW_IP_GFX) || 507 (queue->queue_type == AMDGPU_HW_IP_COMPUTE))) 508 skip_map_queue = true; 509 else 510 skip_map_queue = false; 511 if (!skip_map_queue) { 512 r = amdgpu_userq_map_helper(uq_mgr, queue); 513 if (r) { 514 drm_file_err(uq_mgr->file, "Failed to map Queue\n"); 515 idr_remove(&uq_mgr->userq_idr, qid); 516 amdgpu_userq_fence_driver_free(queue); 517 uq_funcs->mqd_destroy(uq_mgr, queue); 518 kfree(queue); 519 goto unlock; 520 } 521 } 522 523 queue_name = kasprintf(GFP_KERNEL, "queue-%d", qid); 524 if (!queue_name) { 525 r = -ENOMEM; 526 goto unlock; 527 } 528 529 #if defined(CONFIG_DEBUG_FS) 530 /* Queue dentry per client to hold MQD information */ 531 queue->debugfs_queue = debugfs_create_dir(queue_name, filp->debugfs_client); 532 debugfs_create_file("mqd_info", 0444, queue->debugfs_queue, queue, &amdgpu_mqd_info_fops); 533 #endif 534 kfree(queue_name); 535 536 args->out.queue_id = qid; 537 538 unlock: 539 mutex_unlock(&uq_mgr->userq_mutex); 540 mutex_unlock(&adev->userq_mutex); 541 542 return r; 543 } 544 545 int amdgpu_userq_ioctl(struct drm_device *dev, void *data, 546 struct drm_file *filp) 547 { 548 union drm_amdgpu_userq *args = data; 549 int r; 550 551 switch (args->in.op) { 552 case AMDGPU_USERQ_OP_CREATE: 553 if (args->in.flags & ~(AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK | 554 AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE)) 555 return -EINVAL; 556 r = amdgpu_userq_create(filp, args); 557 if (r) 558 drm_file_err(filp, "Failed to create usermode queue\n"); 559 break; 560 561 case AMDGPU_USERQ_OP_FREE: 562 if (args->in.ip_type || 563 args->in.doorbell_handle || 564 args->in.doorbell_offset || 565 args->in.flags || 566 args->in.queue_va || 567 args->in.queue_size || 568 args->in.rptr_va || 569 args->in.wptr_va || 570 args->in.wptr_va || 571 args->in.mqd || 572 args->in.mqd_size) 573 return -EINVAL; 574 r = amdgpu_userq_destroy(filp, args->in.queue_id); 575 if (r) 576 drm_file_err(filp, "Failed to destroy usermode queue\n"); 577 break; 578 579 default: 580 drm_dbg_driver(dev, "Invalid user queue op specified: %d\n", args->in.op); 581 return -EINVAL; 582 } 583 584 return r; 585 } 586 587 static int 588 amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr) 589 { 590 struct amdgpu_usermode_queue *queue; 591 int queue_id; 592 int ret = 0, r; 593 594 /* Resume all the queues for this process */ 595 idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) { 596 r = amdgpu_userq_map_helper(uq_mgr, queue); 597 if (r) 598 ret = r; 599 } 600 601 if (ret) 602 drm_file_err(uq_mgr->file, "Failed to map all the queues\n"); 603 return ret; 604 } 605 606 static int 607 amdgpu_userq_validate_vm_bo(void *_unused, struct amdgpu_bo *bo) 608 { 609 struct ttm_operation_ctx ctx = { false, false }; 610 int ret; 611 612 amdgpu_bo_placement_from_domain(bo, bo->allowed_domains); 613 614 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 615 if (ret) 616 DRM_ERROR("Fail to validate\n"); 617 618 return ret; 619 } 620 621 static int 622 amdgpu_userq_validate_bos(struct amdgpu_userq_mgr *uq_mgr) 623 { 624 struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); 625 struct amdgpu_vm *vm = &fpriv->vm; 626 struct amdgpu_device *adev = uq_mgr->adev; 627 struct amdgpu_bo_va *bo_va; 628 struct ww_acquire_ctx *ticket; 629 struct drm_exec exec; 630 struct amdgpu_bo *bo; 631 struct dma_resv *resv; 632 bool clear, unlock; 633 int ret = 0; 634 635 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0); 636 drm_exec_until_all_locked(&exec) { 637 ret = amdgpu_vm_lock_pd(vm, &exec, 2); 638 drm_exec_retry_on_contention(&exec); 639 if (unlikely(ret)) { 640 drm_file_err(uq_mgr->file, "Failed to lock PD\n"); 641 goto unlock_all; 642 } 643 644 /* Lock the done list */ 645 list_for_each_entry(bo_va, &vm->done, base.vm_status) { 646 bo = bo_va->base.bo; 647 if (!bo) 648 continue; 649 650 ret = drm_exec_lock_obj(&exec, &bo->tbo.base); 651 drm_exec_retry_on_contention(&exec); 652 if (unlikely(ret)) 653 goto unlock_all; 654 } 655 } 656 657 spin_lock(&vm->status_lock); 658 while (!list_empty(&vm->moved)) { 659 bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va, 660 base.vm_status); 661 spin_unlock(&vm->status_lock); 662 663 /* Per VM BOs never need to bo cleared in the page tables */ 664 ret = amdgpu_vm_bo_update(adev, bo_va, false); 665 if (ret) 666 goto unlock_all; 667 spin_lock(&vm->status_lock); 668 } 669 670 ticket = &exec.ticket; 671 while (!list_empty(&vm->invalidated)) { 672 bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va, 673 base.vm_status); 674 resv = bo_va->base.bo->tbo.base.resv; 675 spin_unlock(&vm->status_lock); 676 677 bo = bo_va->base.bo; 678 ret = amdgpu_userq_validate_vm_bo(NULL, bo); 679 if (ret) { 680 drm_file_err(uq_mgr->file, "Failed to validate BO\n"); 681 goto unlock_all; 682 } 683 684 /* Try to reserve the BO to avoid clearing its ptes */ 685 if (!adev->debug_vm && dma_resv_trylock(resv)) { 686 clear = false; 687 unlock = true; 688 /* The caller is already holding the reservation lock */ 689 } else if (dma_resv_locking_ctx(resv) == ticket) { 690 clear = false; 691 unlock = false; 692 /* Somebody else is using the BO right now */ 693 } else { 694 clear = true; 695 unlock = false; 696 } 697 698 ret = amdgpu_vm_bo_update(adev, bo_va, clear); 699 700 if (unlock) 701 dma_resv_unlock(resv); 702 if (ret) 703 goto unlock_all; 704 705 spin_lock(&vm->status_lock); 706 } 707 spin_unlock(&vm->status_lock); 708 709 ret = amdgpu_eviction_fence_replace_fence(&fpriv->evf_mgr, &exec); 710 if (ret) 711 drm_file_err(uq_mgr->file, "Failed to replace eviction fence\n"); 712 713 unlock_all: 714 drm_exec_fini(&exec); 715 return ret; 716 } 717 718 static void amdgpu_userq_restore_worker(struct work_struct *work) 719 { 720 struct amdgpu_userq_mgr *uq_mgr = work_to_uq_mgr(work, resume_work.work); 721 struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); 722 int ret; 723 724 flush_delayed_work(&fpriv->evf_mgr.suspend_work); 725 726 mutex_lock(&uq_mgr->userq_mutex); 727 728 ret = amdgpu_userq_validate_bos(uq_mgr); 729 if (ret) { 730 drm_file_err(uq_mgr->file, "Failed to validate BOs to restore\n"); 731 goto unlock; 732 } 733 734 ret = amdgpu_userq_restore_all(uq_mgr); 735 if (ret) { 736 drm_file_err(uq_mgr->file, "Failed to restore all queues\n"); 737 goto unlock; 738 } 739 740 unlock: 741 mutex_unlock(&uq_mgr->userq_mutex); 742 } 743 744 static int 745 amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr) 746 { 747 struct amdgpu_usermode_queue *queue; 748 int queue_id; 749 int ret = 0, r; 750 751 /* Try to unmap all the queues in this process ctx */ 752 idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) { 753 r = amdgpu_userq_unmap_helper(uq_mgr, queue); 754 if (r) 755 ret = r; 756 } 757 758 if (ret) 759 drm_file_err(uq_mgr->file, "Couldn't unmap all the queues\n"); 760 return ret; 761 } 762 763 static int 764 amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr) 765 { 766 struct amdgpu_usermode_queue *queue; 767 int queue_id, ret; 768 769 idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) { 770 struct dma_fence *f = queue->last_fence; 771 772 if (!f || dma_fence_is_signaled(f)) 773 continue; 774 ret = dma_fence_wait_timeout(f, true, msecs_to_jiffies(100)); 775 if (ret <= 0) { 776 drm_file_err(uq_mgr->file, "Timed out waiting for fence=%llu:%llu\n", 777 f->context, f->seqno); 778 return -ETIMEDOUT; 779 } 780 } 781 782 return 0; 783 } 784 785 void 786 amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr, 787 struct amdgpu_eviction_fence *ev_fence) 788 { 789 int ret; 790 struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); 791 struct amdgpu_eviction_fence_mgr *evf_mgr = &fpriv->evf_mgr; 792 793 /* Wait for any pending userqueue fence work to finish */ 794 ret = amdgpu_userq_wait_for_signal(uq_mgr); 795 if (ret) { 796 drm_file_err(uq_mgr->file, "Not evicting userqueue, timeout waiting for work\n"); 797 return; 798 } 799 800 ret = amdgpu_userq_evict_all(uq_mgr); 801 if (ret) { 802 drm_file_err(uq_mgr->file, "Failed to evict userqueue\n"); 803 return; 804 } 805 806 /* Signal current eviction fence */ 807 amdgpu_eviction_fence_signal(evf_mgr, ev_fence); 808 809 if (evf_mgr->fd_closing) { 810 cancel_delayed_work_sync(&uq_mgr->resume_work); 811 return; 812 } 813 814 /* Schedule a resume work */ 815 schedule_delayed_work(&uq_mgr->resume_work, 0); 816 } 817 818 int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv, 819 struct amdgpu_device *adev) 820 { 821 mutex_init(&userq_mgr->userq_mutex); 822 idr_init_base(&userq_mgr->userq_idr, 1); 823 userq_mgr->adev = adev; 824 userq_mgr->file = file_priv; 825 826 mutex_lock(&adev->userq_mutex); 827 list_add(&userq_mgr->list, &adev->userq_mgr_list); 828 mutex_unlock(&adev->userq_mutex); 829 830 INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userq_restore_worker); 831 return 0; 832 } 833 834 void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr) 835 { 836 struct amdgpu_device *adev = userq_mgr->adev; 837 struct amdgpu_usermode_queue *queue; 838 struct amdgpu_userq_mgr *uqm, *tmp; 839 uint32_t queue_id; 840 841 cancel_delayed_work_sync(&userq_mgr->resume_work); 842 843 mutex_lock(&adev->userq_mutex); 844 mutex_lock(&userq_mgr->userq_mutex); 845 idr_for_each_entry(&userq_mgr->userq_idr, queue, queue_id) { 846 amdgpu_userq_wait_for_last_fence(userq_mgr, queue); 847 amdgpu_userq_unmap_helper(userq_mgr, queue); 848 amdgpu_userq_cleanup(userq_mgr, queue, queue_id); 849 } 850 851 list_for_each_entry_safe(uqm, tmp, &adev->userq_mgr_list, list) { 852 if (uqm == userq_mgr) { 853 list_del(&uqm->list); 854 break; 855 } 856 } 857 idr_destroy(&userq_mgr->userq_idr); 858 mutex_unlock(&userq_mgr->userq_mutex); 859 mutex_unlock(&adev->userq_mutex); 860 mutex_destroy(&userq_mgr->userq_mutex); 861 } 862 863 int amdgpu_userq_suspend(struct amdgpu_device *adev) 864 { 865 u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev); 866 struct amdgpu_usermode_queue *queue; 867 struct amdgpu_userq_mgr *uqm, *tmp; 868 int queue_id; 869 int ret = 0, r; 870 871 if (!ip_mask) 872 return 0; 873 874 mutex_lock(&adev->userq_mutex); 875 list_for_each_entry_safe(uqm, tmp, &adev->userq_mgr_list, list) { 876 cancel_delayed_work_sync(&uqm->resume_work); 877 mutex_lock(&uqm->userq_mutex); 878 idr_for_each_entry(&uqm->userq_idr, queue, queue_id) { 879 r = amdgpu_userq_unmap_helper(uqm, queue); 880 if (r) 881 ret = r; 882 } 883 mutex_unlock(&uqm->userq_mutex); 884 } 885 mutex_unlock(&adev->userq_mutex); 886 return ret; 887 } 888 889 int amdgpu_userq_resume(struct amdgpu_device *adev) 890 { 891 u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev); 892 struct amdgpu_usermode_queue *queue; 893 struct amdgpu_userq_mgr *uqm, *tmp; 894 int queue_id; 895 int ret = 0, r; 896 897 if (!ip_mask) 898 return 0; 899 900 mutex_lock(&adev->userq_mutex); 901 list_for_each_entry_safe(uqm, tmp, &adev->userq_mgr_list, list) { 902 mutex_lock(&uqm->userq_mutex); 903 idr_for_each_entry(&uqm->userq_idr, queue, queue_id) { 904 r = amdgpu_userq_map_helper(uqm, queue); 905 if (r) 906 ret = r; 907 } 908 mutex_unlock(&uqm->userq_mutex); 909 } 910 mutex_unlock(&adev->userq_mutex); 911 return ret; 912 } 913 914 int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev, 915 u32 idx) 916 { 917 u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev); 918 struct amdgpu_usermode_queue *queue; 919 struct amdgpu_userq_mgr *uqm, *tmp; 920 int queue_id; 921 int ret = 0, r; 922 923 /* only need to stop gfx/compute */ 924 if (!(ip_mask & ((1 << AMDGPU_HW_IP_GFX) | (1 << AMDGPU_HW_IP_COMPUTE)))) 925 return 0; 926 927 mutex_lock(&adev->userq_mutex); 928 if (adev->userq_halt_for_enforce_isolation) 929 dev_warn(adev->dev, "userq scheduling already stopped!\n"); 930 adev->userq_halt_for_enforce_isolation = true; 931 list_for_each_entry_safe(uqm, tmp, &adev->userq_mgr_list, list) { 932 cancel_delayed_work_sync(&uqm->resume_work); 933 mutex_lock(&uqm->userq_mutex); 934 idr_for_each_entry(&uqm->userq_idr, queue, queue_id) { 935 if (((queue->queue_type == AMDGPU_HW_IP_GFX) || 936 (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) && 937 (queue->xcp_id == idx)) { 938 r = amdgpu_userq_unmap_helper(uqm, queue); 939 if (r) 940 ret = r; 941 } 942 } 943 mutex_unlock(&uqm->userq_mutex); 944 } 945 mutex_unlock(&adev->userq_mutex); 946 return ret; 947 } 948 949 int amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device *adev, 950 u32 idx) 951 { 952 u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev); 953 struct amdgpu_usermode_queue *queue; 954 struct amdgpu_userq_mgr *uqm, *tmp; 955 int queue_id; 956 int ret = 0, r; 957 958 /* only need to stop gfx/compute */ 959 if (!(ip_mask & ((1 << AMDGPU_HW_IP_GFX) | (1 << AMDGPU_HW_IP_COMPUTE)))) 960 return 0; 961 962 mutex_lock(&adev->userq_mutex); 963 if (!adev->userq_halt_for_enforce_isolation) 964 dev_warn(adev->dev, "userq scheduling already started!\n"); 965 adev->userq_halt_for_enforce_isolation = false; 966 list_for_each_entry_safe(uqm, tmp, &adev->userq_mgr_list, list) { 967 mutex_lock(&uqm->userq_mutex); 968 idr_for_each_entry(&uqm->userq_idr, queue, queue_id) { 969 if (((queue->queue_type == AMDGPU_HW_IP_GFX) || 970 (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) && 971 (queue->xcp_id == idx)) { 972 r = amdgpu_userq_map_helper(uqm, queue); 973 if (r) 974 ret = r; 975 } 976 } 977 mutex_unlock(&uqm->userq_mutex); 978 } 979 mutex_unlock(&adev->userq_mutex); 980 return ret; 981 } 982