1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2023 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/kref.h> 26 #include <linux/slab.h> 27 #include <linux/dma-fence-unwrap.h> 28 29 #include <drm/drm_exec.h> 30 #include <drm/drm_syncobj.h> 31 32 #include "amdgpu.h" 33 #include "amdgpu_userq_fence.h" 34 35 static const struct dma_fence_ops amdgpu_userq_fence_ops; 36 static struct kmem_cache *amdgpu_userq_fence_slab; 37 38 int amdgpu_userq_fence_slab_init(void) 39 { 40 amdgpu_userq_fence_slab = kmem_cache_create("amdgpu_userq_fence", 41 sizeof(struct amdgpu_userq_fence), 42 0, 43 SLAB_HWCACHE_ALIGN, 44 NULL); 45 if (!amdgpu_userq_fence_slab) 46 return -ENOMEM; 47 48 return 0; 49 } 50 51 void amdgpu_userq_fence_slab_fini(void) 52 { 53 rcu_barrier(); 54 kmem_cache_destroy(amdgpu_userq_fence_slab); 55 } 56 57 static inline struct amdgpu_userq_fence *to_amdgpu_userq_fence(struct dma_fence *f) 58 { 59 if (!f || f->ops != &amdgpu_userq_fence_ops) 60 return NULL; 61 62 return container_of(f, struct amdgpu_userq_fence, base); 63 } 64 65 static u64 amdgpu_userq_fence_read(struct amdgpu_userq_fence_driver *fence_drv) 66 { 67 return le64_to_cpu(*fence_drv->cpu_addr); 68 } 69 70 static void 71 amdgpu_userq_fence_write(struct amdgpu_userq_fence_driver *fence_drv, 72 u64 seq) 73 { 74 if (fence_drv->cpu_addr) 75 *fence_drv->cpu_addr = cpu_to_le64(seq); 76 } 77 78 int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev, 79 struct amdgpu_usermode_queue *userq) 80 { 81 struct amdgpu_userq_fence_driver *fence_drv; 82 unsigned long flags; 83 int r; 84 85 fence_drv = kzalloc_obj(*fence_drv, GFP_KERNEL); 86 if (!fence_drv) 87 return -ENOMEM; 88 89 /* Acquire seq64 memory */ 90 r = amdgpu_seq64_alloc(adev, &fence_drv->va, &fence_drv->gpu_addr, 91 &fence_drv->cpu_addr); 92 if (r) 93 goto free_fence_drv; 94 95 memset(fence_drv->cpu_addr, 0, sizeof(u64)); 96 97 kref_init(&fence_drv->refcount); 98 INIT_LIST_HEAD(&fence_drv->fences); 99 spin_lock_init(&fence_drv->fence_list_lock); 100 101 fence_drv->adev = adev; 102 fence_drv->context = dma_fence_context_alloc(1); 103 get_task_comm(fence_drv->timeline_name, current); 104 105 xa_lock_irqsave(&adev->userq_xa, flags); 106 r = xa_err(__xa_store(&adev->userq_xa, userq->doorbell_index, 107 fence_drv, GFP_KERNEL)); 108 xa_unlock_irqrestore(&adev->userq_xa, flags); 109 if (r) 110 goto free_seq64; 111 112 userq->fence_drv = fence_drv; 113 114 return 0; 115 116 free_seq64: 117 amdgpu_seq64_free(adev, fence_drv->va); 118 free_fence_drv: 119 kfree(fence_drv); 120 121 return r; 122 } 123 124 static void amdgpu_userq_walk_and_drop_fence_drv(struct xarray *xa) 125 { 126 struct amdgpu_userq_fence_driver *fence_drv; 127 unsigned long index; 128 129 if (xa_empty(xa)) 130 return; 131 132 xa_lock(xa); 133 xa_for_each(xa, index, fence_drv) { 134 __xa_erase(xa, index); 135 amdgpu_userq_fence_driver_put(fence_drv); 136 } 137 138 xa_unlock(xa); 139 } 140 141 void 142 amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq) 143 { 144 dma_fence_put(userq->last_fence); 145 146 amdgpu_userq_walk_and_drop_fence_drv(&userq->fence_drv_xa); 147 xa_destroy(&userq->fence_drv_xa); 148 /* Drop the fence_drv reference held by user queue */ 149 amdgpu_userq_fence_driver_put(userq->fence_drv); 150 } 151 152 void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv) 153 { 154 struct amdgpu_userq_fence *userq_fence, *tmp; 155 struct dma_fence *fence; 156 unsigned long flags; 157 u64 rptr; 158 int i; 159 160 if (!fence_drv) 161 return; 162 163 spin_lock_irqsave(&fence_drv->fence_list_lock, flags); 164 rptr = amdgpu_userq_fence_read(fence_drv); 165 166 list_for_each_entry_safe(userq_fence, tmp, &fence_drv->fences, link) { 167 fence = &userq_fence->base; 168 169 if (rptr < fence->seqno) 170 break; 171 172 dma_fence_signal(fence); 173 174 for (i = 0; i < userq_fence->fence_drv_array_count; i++) 175 amdgpu_userq_fence_driver_put(userq_fence->fence_drv_array[i]); 176 177 list_del(&userq_fence->link); 178 dma_fence_put(fence); 179 } 180 spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags); 181 } 182 183 void amdgpu_userq_fence_driver_destroy(struct kref *ref) 184 { 185 struct amdgpu_userq_fence_driver *fence_drv = container_of(ref, 186 struct amdgpu_userq_fence_driver, 187 refcount); 188 struct amdgpu_userq_fence_driver *xa_fence_drv; 189 struct amdgpu_device *adev = fence_drv->adev; 190 struct amdgpu_userq_fence *fence, *tmp; 191 struct xarray *xa = &adev->userq_xa; 192 unsigned long index, flags; 193 struct dma_fence *f; 194 195 spin_lock_irqsave(&fence_drv->fence_list_lock, flags); 196 list_for_each_entry_safe(fence, tmp, &fence_drv->fences, link) { 197 f = &fence->base; 198 199 if (!dma_fence_is_signaled(f)) { 200 dma_fence_set_error(f, -ECANCELED); 201 dma_fence_signal(f); 202 } 203 204 list_del(&fence->link); 205 dma_fence_put(f); 206 } 207 spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags); 208 209 xa_lock_irqsave(xa, flags); 210 xa_for_each(xa, index, xa_fence_drv) 211 if (xa_fence_drv == fence_drv) 212 __xa_erase(xa, index); 213 xa_unlock_irqrestore(xa, flags); 214 215 /* Free seq64 memory */ 216 amdgpu_seq64_free(adev, fence_drv->va); 217 kfree(fence_drv); 218 } 219 220 void amdgpu_userq_fence_driver_get(struct amdgpu_userq_fence_driver *fence_drv) 221 { 222 kref_get(&fence_drv->refcount); 223 } 224 225 void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv) 226 { 227 kref_put(&fence_drv->refcount, amdgpu_userq_fence_driver_destroy); 228 } 229 230 static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence) 231 { 232 *userq_fence = kmem_cache_alloc(amdgpu_userq_fence_slab, GFP_ATOMIC); 233 return *userq_fence ? 0 : -ENOMEM; 234 } 235 236 static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq, 237 struct amdgpu_userq_fence *userq_fence, 238 u64 seq, struct dma_fence **f) 239 { 240 struct amdgpu_userq_fence_driver *fence_drv; 241 struct dma_fence *fence; 242 unsigned long flags; 243 244 fence_drv = userq->fence_drv; 245 if (!fence_drv) 246 return -EINVAL; 247 248 spin_lock_init(&userq_fence->lock); 249 INIT_LIST_HEAD(&userq_fence->link); 250 fence = &userq_fence->base; 251 userq_fence->fence_drv = fence_drv; 252 253 dma_fence_init64(fence, &amdgpu_userq_fence_ops, &userq_fence->lock, 254 fence_drv->context, seq); 255 256 amdgpu_userq_fence_driver_get(fence_drv); 257 dma_fence_get(fence); 258 259 if (!xa_empty(&userq->fence_drv_xa)) { 260 struct amdgpu_userq_fence_driver *stored_fence_drv; 261 unsigned long index, count = 0; 262 int i = 0; 263 264 xa_lock(&userq->fence_drv_xa); 265 xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv) 266 count++; 267 268 userq_fence->fence_drv_array = 269 kvmalloc_objs(struct amdgpu_userq_fence_driver *, count, 270 GFP_ATOMIC); 271 272 if (userq_fence->fence_drv_array) { 273 xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv) { 274 userq_fence->fence_drv_array[i] = stored_fence_drv; 275 __xa_erase(&userq->fence_drv_xa, index); 276 i++; 277 } 278 } 279 280 userq_fence->fence_drv_array_count = i; 281 xa_unlock(&userq->fence_drv_xa); 282 } else { 283 userq_fence->fence_drv_array = NULL; 284 userq_fence->fence_drv_array_count = 0; 285 } 286 287 /* Check if hardware has already processed the job */ 288 spin_lock_irqsave(&fence_drv->fence_list_lock, flags); 289 if (!dma_fence_is_signaled(fence)) 290 list_add_tail(&userq_fence->link, &fence_drv->fences); 291 else 292 dma_fence_put(fence); 293 294 spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags); 295 296 *f = fence; 297 298 return 0; 299 } 300 301 static const char *amdgpu_userq_fence_get_driver_name(struct dma_fence *f) 302 { 303 return "amdgpu_userq_fence"; 304 } 305 306 static const char *amdgpu_userq_fence_get_timeline_name(struct dma_fence *f) 307 { 308 struct amdgpu_userq_fence *fence = to_amdgpu_userq_fence(f); 309 310 return fence->fence_drv->timeline_name; 311 } 312 313 static bool amdgpu_userq_fence_signaled(struct dma_fence *f) 314 { 315 struct amdgpu_userq_fence *fence = to_amdgpu_userq_fence(f); 316 struct amdgpu_userq_fence_driver *fence_drv = fence->fence_drv; 317 u64 rptr, wptr; 318 319 rptr = amdgpu_userq_fence_read(fence_drv); 320 wptr = fence->base.seqno; 321 322 if (rptr >= wptr) 323 return true; 324 325 return false; 326 } 327 328 static void amdgpu_userq_fence_free(struct rcu_head *rcu) 329 { 330 struct dma_fence *fence = container_of(rcu, struct dma_fence, rcu); 331 struct amdgpu_userq_fence *userq_fence = to_amdgpu_userq_fence(fence); 332 struct amdgpu_userq_fence_driver *fence_drv = userq_fence->fence_drv; 333 334 /* Release the fence driver reference */ 335 amdgpu_userq_fence_driver_put(fence_drv); 336 337 kvfree(userq_fence->fence_drv_array); 338 kmem_cache_free(amdgpu_userq_fence_slab, userq_fence); 339 } 340 341 static void amdgpu_userq_fence_release(struct dma_fence *f) 342 { 343 call_rcu(&f->rcu, amdgpu_userq_fence_free); 344 } 345 346 static const struct dma_fence_ops amdgpu_userq_fence_ops = { 347 .get_driver_name = amdgpu_userq_fence_get_driver_name, 348 .get_timeline_name = amdgpu_userq_fence_get_timeline_name, 349 .signaled = amdgpu_userq_fence_signaled, 350 .release = amdgpu_userq_fence_release, 351 }; 352 353 /** 354 * amdgpu_userq_fence_read_wptr - Read the userq wptr value 355 * 356 * @adev: amdgpu_device pointer 357 * @queue: user mode queue structure pointer 358 * @wptr: write pointer value 359 * 360 * Read the wptr value from userq's MQD. The userq signal IOCTL 361 * creates a dma_fence for the shared buffers that expects the 362 * RPTR value written to seq64 memory >= WPTR. 363 * 364 * Returns wptr value on success, error on failure. 365 */ 366 static int amdgpu_userq_fence_read_wptr(struct amdgpu_device *adev, 367 struct amdgpu_usermode_queue *queue, 368 u64 *wptr) 369 { 370 struct amdgpu_bo_va_mapping *mapping; 371 struct amdgpu_bo *bo; 372 u64 addr, *ptr; 373 int r; 374 375 r = amdgpu_bo_reserve(queue->vm->root.bo, false); 376 if (r) 377 return r; 378 379 addr = queue->userq_prop->wptr_gpu_addr; 380 addr &= AMDGPU_GMC_HOLE_MASK; 381 382 mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, addr >> PAGE_SHIFT); 383 if (!mapping) { 384 amdgpu_bo_unreserve(queue->vm->root.bo); 385 DRM_ERROR("Failed to lookup amdgpu_bo_va_mapping\n"); 386 return -EINVAL; 387 } 388 389 bo = amdgpu_bo_ref(mapping->bo_va->base.bo); 390 amdgpu_bo_unreserve(queue->vm->root.bo); 391 r = amdgpu_bo_reserve(bo, true); 392 if (r) { 393 amdgpu_bo_unref(&bo); 394 DRM_ERROR("Failed to reserve userqueue wptr bo"); 395 return r; 396 } 397 398 r = amdgpu_bo_kmap(bo, (void **)&ptr); 399 if (r) { 400 DRM_ERROR("Failed mapping the userqueue wptr bo"); 401 goto map_error; 402 } 403 404 *wptr = le64_to_cpu(*ptr); 405 406 amdgpu_bo_kunmap(bo); 407 amdgpu_bo_unreserve(bo); 408 amdgpu_bo_unref(&bo); 409 410 return 0; 411 412 map_error: 413 amdgpu_bo_unreserve(bo); 414 amdgpu_bo_unref(&bo); 415 416 return r; 417 } 418 419 static void amdgpu_userq_fence_cleanup(struct dma_fence *fence) 420 { 421 dma_fence_put(fence); 422 } 423 424 static void 425 amdgpu_userq_fence_driver_set_error(struct amdgpu_userq_fence *fence, 426 int error) 427 { 428 struct amdgpu_userq_fence_driver *fence_drv = fence->fence_drv; 429 unsigned long flags; 430 struct dma_fence *f; 431 432 spin_lock_irqsave(&fence_drv->fence_list_lock, flags); 433 434 f = rcu_dereference_protected(&fence->base, 435 lockdep_is_held(&fence_drv->fence_list_lock)); 436 if (f && !dma_fence_is_signaled_locked(f)) 437 dma_fence_set_error(f, error); 438 spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags); 439 } 440 441 void 442 amdgpu_userq_fence_driver_force_completion(struct amdgpu_usermode_queue *userq) 443 { 444 struct dma_fence *f = userq->last_fence; 445 446 if (f) { 447 struct amdgpu_userq_fence *fence = to_amdgpu_userq_fence(f); 448 struct amdgpu_userq_fence_driver *fence_drv = fence->fence_drv; 449 u64 wptr = fence->base.seqno; 450 451 amdgpu_userq_fence_driver_set_error(fence, -ECANCELED); 452 amdgpu_userq_fence_write(fence_drv, wptr); 453 amdgpu_userq_fence_driver_process(fence_drv); 454 455 } 456 } 457 458 int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data, 459 struct drm_file *filp) 460 { 461 struct amdgpu_device *adev = drm_to_adev(dev); 462 struct amdgpu_fpriv *fpriv = filp->driver_priv; 463 struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr; 464 struct drm_amdgpu_userq_signal *args = data; 465 struct drm_gem_object **gobj_write = NULL; 466 struct drm_gem_object **gobj_read = NULL; 467 struct amdgpu_usermode_queue *queue; 468 struct amdgpu_userq_fence *userq_fence; 469 struct drm_syncobj **syncobj = NULL; 470 u32 *bo_handles_write, num_write_bo_handles; 471 u32 *syncobj_handles, num_syncobj_handles; 472 u32 *bo_handles_read, num_read_bo_handles; 473 int r, i, entry, rentry, wentry; 474 struct dma_fence *fence; 475 struct drm_exec exec; 476 u64 wptr; 477 478 if (!amdgpu_userq_enabled(dev)) 479 return -ENOTSUPP; 480 481 num_syncobj_handles = args->num_syncobj_handles; 482 syncobj_handles = memdup_user(u64_to_user_ptr(args->syncobj_handles), 483 size_mul(sizeof(u32), num_syncobj_handles)); 484 if (IS_ERR(syncobj_handles)) 485 return PTR_ERR(syncobj_handles); 486 487 /* Array of pointers to the looked up syncobjs */ 488 syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj), GFP_KERNEL); 489 if (!syncobj) { 490 r = -ENOMEM; 491 goto free_syncobj_handles; 492 } 493 494 for (entry = 0; entry < num_syncobj_handles; entry++) { 495 syncobj[entry] = drm_syncobj_find(filp, syncobj_handles[entry]); 496 if (!syncobj[entry]) { 497 r = -ENOENT; 498 goto free_syncobj; 499 } 500 } 501 502 num_read_bo_handles = args->num_bo_read_handles; 503 bo_handles_read = memdup_user(u64_to_user_ptr(args->bo_read_handles), 504 sizeof(u32) * num_read_bo_handles); 505 if (IS_ERR(bo_handles_read)) { 506 r = PTR_ERR(bo_handles_read); 507 goto free_syncobj; 508 } 509 510 /* Array of pointers to the GEM read objects */ 511 gobj_read = kmalloc_array(num_read_bo_handles, sizeof(*gobj_read), GFP_KERNEL); 512 if (!gobj_read) { 513 r = -ENOMEM; 514 goto free_bo_handles_read; 515 } 516 517 for (rentry = 0; rentry < num_read_bo_handles; rentry++) { 518 gobj_read[rentry] = drm_gem_object_lookup(filp, bo_handles_read[rentry]); 519 if (!gobj_read[rentry]) { 520 r = -ENOENT; 521 goto put_gobj_read; 522 } 523 } 524 525 num_write_bo_handles = args->num_bo_write_handles; 526 bo_handles_write = memdup_user(u64_to_user_ptr(args->bo_write_handles), 527 sizeof(u32) * num_write_bo_handles); 528 if (IS_ERR(bo_handles_write)) { 529 r = PTR_ERR(bo_handles_write); 530 goto put_gobj_read; 531 } 532 533 /* Array of pointers to the GEM write objects */ 534 gobj_write = kmalloc_array(num_write_bo_handles, sizeof(*gobj_write), GFP_KERNEL); 535 if (!gobj_write) { 536 r = -ENOMEM; 537 goto free_bo_handles_write; 538 } 539 540 for (wentry = 0; wentry < num_write_bo_handles; wentry++) { 541 gobj_write[wentry] = drm_gem_object_lookup(filp, bo_handles_write[wentry]); 542 if (!gobj_write[wentry]) { 543 r = -ENOENT; 544 goto put_gobj_write; 545 } 546 } 547 548 /* Retrieve the user queue */ 549 queue = xa_load(&userq_mgr->userq_xa, args->queue_id); 550 if (!queue) { 551 r = -ENOENT; 552 goto put_gobj_write; 553 } 554 555 r = amdgpu_userq_fence_read_wptr(adev, queue, &wptr); 556 if (r) 557 goto put_gobj_write; 558 559 r = amdgpu_userq_fence_alloc(&userq_fence); 560 if (r) 561 goto put_gobj_write; 562 563 /* We are here means UQ is active, make sure the eviction fence is valid */ 564 amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr); 565 566 /* Create a new fence */ 567 r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence); 568 if (r) { 569 mutex_unlock(&userq_mgr->userq_mutex); 570 kmem_cache_free(amdgpu_userq_fence_slab, userq_fence); 571 goto put_gobj_write; 572 } 573 574 dma_fence_put(queue->last_fence); 575 queue->last_fence = dma_fence_get(fence); 576 amdgpu_userq_start_hang_detect_work(queue); 577 mutex_unlock(&userq_mgr->userq_mutex); 578 579 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 580 (num_read_bo_handles + num_write_bo_handles)); 581 582 /* Lock all BOs with retry handling */ 583 drm_exec_until_all_locked(&exec) { 584 r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1); 585 drm_exec_retry_on_contention(&exec); 586 if (r) { 587 amdgpu_userq_fence_cleanup(fence); 588 goto exec_fini; 589 } 590 591 r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1); 592 drm_exec_retry_on_contention(&exec); 593 if (r) { 594 amdgpu_userq_fence_cleanup(fence); 595 goto exec_fini; 596 } 597 } 598 599 for (i = 0; i < num_read_bo_handles; i++) { 600 if (!gobj_read || !gobj_read[i]->resv) 601 continue; 602 603 dma_resv_add_fence(gobj_read[i]->resv, fence, 604 DMA_RESV_USAGE_READ); 605 } 606 607 for (i = 0; i < num_write_bo_handles; i++) { 608 if (!gobj_write || !gobj_write[i]->resv) 609 continue; 610 611 dma_resv_add_fence(gobj_write[i]->resv, fence, 612 DMA_RESV_USAGE_WRITE); 613 } 614 615 /* Add the created fence to syncobj/BO's */ 616 for (i = 0; i < num_syncobj_handles; i++) 617 drm_syncobj_replace_fence(syncobj[i], fence); 618 619 /* drop the reference acquired in fence creation function */ 620 dma_fence_put(fence); 621 622 exec_fini: 623 drm_exec_fini(&exec); 624 put_gobj_write: 625 while (wentry-- > 0) 626 drm_gem_object_put(gobj_write[wentry]); 627 kfree(gobj_write); 628 free_bo_handles_write: 629 kfree(bo_handles_write); 630 put_gobj_read: 631 while (rentry-- > 0) 632 drm_gem_object_put(gobj_read[rentry]); 633 kfree(gobj_read); 634 free_bo_handles_read: 635 kfree(bo_handles_read); 636 free_syncobj: 637 while (entry-- > 0) 638 if (syncobj[entry]) 639 drm_syncobj_put(syncobj[entry]); 640 kfree(syncobj); 641 free_syncobj_handles: 642 kfree(syncobj_handles); 643 644 return r; 645 } 646 647 int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, 648 struct drm_file *filp) 649 { 650 u32 *syncobj_handles, *timeline_points, *timeline_handles, *bo_handles_read, *bo_handles_write; 651 u32 num_syncobj, num_read_bo_handles, num_write_bo_handles; 652 struct drm_amdgpu_userq_fence_info *fence_info = NULL; 653 struct drm_amdgpu_userq_wait *wait_info = data; 654 struct amdgpu_fpriv *fpriv = filp->driver_priv; 655 struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr; 656 struct amdgpu_usermode_queue *waitq; 657 struct drm_gem_object **gobj_write; 658 struct drm_gem_object **gobj_read; 659 struct dma_fence **fences = NULL; 660 u16 num_points, num_fences = 0; 661 int r, i, rentry, wentry, cnt; 662 struct drm_exec exec; 663 664 if (!amdgpu_userq_enabled(dev)) 665 return -ENOTSUPP; 666 667 num_read_bo_handles = wait_info->num_bo_read_handles; 668 bo_handles_read = memdup_user(u64_to_user_ptr(wait_info->bo_read_handles), 669 size_mul(sizeof(u32), num_read_bo_handles)); 670 if (IS_ERR(bo_handles_read)) 671 return PTR_ERR(bo_handles_read); 672 673 num_write_bo_handles = wait_info->num_bo_write_handles; 674 bo_handles_write = memdup_user(u64_to_user_ptr(wait_info->bo_write_handles), 675 size_mul(sizeof(u32), num_write_bo_handles)); 676 if (IS_ERR(bo_handles_write)) { 677 r = PTR_ERR(bo_handles_write); 678 goto free_bo_handles_read; 679 } 680 681 num_syncobj = wait_info->num_syncobj_handles; 682 syncobj_handles = memdup_user(u64_to_user_ptr(wait_info->syncobj_handles), 683 size_mul(sizeof(u32), num_syncobj)); 684 if (IS_ERR(syncobj_handles)) { 685 r = PTR_ERR(syncobj_handles); 686 goto free_bo_handles_write; 687 } 688 689 num_points = wait_info->num_syncobj_timeline_handles; 690 timeline_handles = memdup_user(u64_to_user_ptr(wait_info->syncobj_timeline_handles), 691 sizeof(u32) * num_points); 692 if (IS_ERR(timeline_handles)) { 693 r = PTR_ERR(timeline_handles); 694 goto free_syncobj_handles; 695 } 696 697 timeline_points = memdup_user(u64_to_user_ptr(wait_info->syncobj_timeline_points), 698 sizeof(u32) * num_points); 699 if (IS_ERR(timeline_points)) { 700 r = PTR_ERR(timeline_points); 701 goto free_timeline_handles; 702 } 703 704 gobj_read = kmalloc_array(num_read_bo_handles, sizeof(*gobj_read), GFP_KERNEL); 705 if (!gobj_read) { 706 r = -ENOMEM; 707 goto free_timeline_points; 708 } 709 710 for (rentry = 0; rentry < num_read_bo_handles; rentry++) { 711 gobj_read[rentry] = drm_gem_object_lookup(filp, bo_handles_read[rentry]); 712 if (!gobj_read[rentry]) { 713 r = -ENOENT; 714 goto put_gobj_read; 715 } 716 } 717 718 gobj_write = kmalloc_array(num_write_bo_handles, sizeof(*gobj_write), GFP_KERNEL); 719 if (!gobj_write) { 720 r = -ENOMEM; 721 goto put_gobj_read; 722 } 723 724 for (wentry = 0; wentry < num_write_bo_handles; wentry++) { 725 gobj_write[wentry] = drm_gem_object_lookup(filp, bo_handles_write[wentry]); 726 if (!gobj_write[wentry]) { 727 r = -ENOENT; 728 goto put_gobj_write; 729 } 730 } 731 732 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 733 (num_read_bo_handles + num_write_bo_handles)); 734 735 /* Lock all BOs with retry handling */ 736 drm_exec_until_all_locked(&exec) { 737 r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1); 738 drm_exec_retry_on_contention(&exec); 739 if (r) { 740 drm_exec_fini(&exec); 741 goto put_gobj_write; 742 } 743 744 r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1); 745 drm_exec_retry_on_contention(&exec); 746 if (r) { 747 drm_exec_fini(&exec); 748 goto put_gobj_write; 749 } 750 } 751 752 if (!wait_info->num_fences) { 753 if (num_points) { 754 struct dma_fence_unwrap iter; 755 struct dma_fence *fence; 756 struct dma_fence *f; 757 758 for (i = 0; i < num_points; i++) { 759 r = drm_syncobj_find_fence(filp, timeline_handles[i], 760 timeline_points[i], 761 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT, 762 &fence); 763 if (r) 764 goto exec_fini; 765 766 dma_fence_unwrap_for_each(f, &iter, fence) 767 num_fences++; 768 769 dma_fence_put(fence); 770 } 771 } 772 773 /* Count syncobj's fence */ 774 for (i = 0; i < num_syncobj; i++) { 775 struct dma_fence *fence; 776 777 r = drm_syncobj_find_fence(filp, syncobj_handles[i], 778 0, 779 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT, 780 &fence); 781 if (r) 782 goto exec_fini; 783 784 num_fences++; 785 dma_fence_put(fence); 786 } 787 788 /* Count GEM objects fence */ 789 for (i = 0; i < num_read_bo_handles; i++) { 790 struct dma_resv_iter resv_cursor; 791 struct dma_fence *fence; 792 793 dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv, 794 DMA_RESV_USAGE_READ, fence) 795 num_fences++; 796 } 797 798 for (i = 0; i < num_write_bo_handles; i++) { 799 struct dma_resv_iter resv_cursor; 800 struct dma_fence *fence; 801 802 dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv, 803 DMA_RESV_USAGE_WRITE, fence) 804 num_fences++; 805 } 806 807 /* 808 * Passing num_fences = 0 means that userspace doesn't want to 809 * retrieve userq_fence_info. If num_fences = 0 we skip filling 810 * userq_fence_info and return the actual number of fences on 811 * args->num_fences. 812 */ 813 wait_info->num_fences = num_fences; 814 } else { 815 /* Array of fence info */ 816 fence_info = kmalloc_array(wait_info->num_fences, sizeof(*fence_info), GFP_KERNEL); 817 if (!fence_info) { 818 r = -ENOMEM; 819 goto exec_fini; 820 } 821 822 /* Array of fences */ 823 fences = kmalloc_array(wait_info->num_fences, sizeof(*fences), GFP_KERNEL); 824 if (!fences) { 825 r = -ENOMEM; 826 goto free_fence_info; 827 } 828 829 /* Retrieve GEM read objects fence */ 830 for (i = 0; i < num_read_bo_handles; i++) { 831 struct dma_resv_iter resv_cursor; 832 struct dma_fence *fence; 833 834 dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv, 835 DMA_RESV_USAGE_READ, fence) { 836 if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) { 837 r = -EINVAL; 838 goto free_fences; 839 } 840 841 fences[num_fences++] = fence; 842 dma_fence_get(fence); 843 } 844 } 845 846 /* Retrieve GEM write objects fence */ 847 for (i = 0; i < num_write_bo_handles; i++) { 848 struct dma_resv_iter resv_cursor; 849 struct dma_fence *fence; 850 851 dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv, 852 DMA_RESV_USAGE_WRITE, fence) { 853 if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) { 854 r = -EINVAL; 855 goto free_fences; 856 } 857 858 fences[num_fences++] = fence; 859 dma_fence_get(fence); 860 } 861 } 862 863 if (num_points) { 864 struct dma_fence_unwrap iter; 865 struct dma_fence *fence; 866 struct dma_fence *f; 867 868 for (i = 0; i < num_points; i++) { 869 r = drm_syncobj_find_fence(filp, timeline_handles[i], 870 timeline_points[i], 871 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT, 872 &fence); 873 if (r) 874 goto free_fences; 875 876 dma_fence_unwrap_for_each(f, &iter, fence) { 877 if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) { 878 r = -EINVAL; 879 goto free_fences; 880 } 881 882 dma_fence_get(f); 883 fences[num_fences++] = f; 884 } 885 886 dma_fence_put(fence); 887 } 888 } 889 890 /* Retrieve syncobj's fence */ 891 for (i = 0; i < num_syncobj; i++) { 892 struct dma_fence *fence; 893 894 r = drm_syncobj_find_fence(filp, syncobj_handles[i], 895 0, 896 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT, 897 &fence); 898 if (r) 899 goto free_fences; 900 901 if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) { 902 r = -EINVAL; 903 goto free_fences; 904 } 905 906 fences[num_fences++] = fence; 907 } 908 909 /* 910 * Keep only the latest fences to reduce the number of values 911 * given back to userspace. 912 */ 913 num_fences = dma_fence_dedup_array(fences, num_fences); 914 915 waitq = xa_load(&userq_mgr->userq_xa, wait_info->waitq_id); 916 if (!waitq) { 917 r = -EINVAL; 918 goto free_fences; 919 } 920 921 for (i = 0, cnt = 0; i < num_fences; i++) { 922 struct amdgpu_userq_fence_driver *fence_drv; 923 struct amdgpu_userq_fence *userq_fence; 924 u32 index; 925 926 userq_fence = to_amdgpu_userq_fence(fences[i]); 927 if (!userq_fence) { 928 /* 929 * Just waiting on other driver fences should 930 * be good for now 931 */ 932 r = dma_fence_wait(fences[i], true); 933 if (r) { 934 dma_fence_put(fences[i]); 935 goto free_fences; 936 } 937 938 dma_fence_put(fences[i]); 939 continue; 940 } 941 942 fence_drv = userq_fence->fence_drv; 943 /* 944 * We need to make sure the user queue release their reference 945 * to the fence drivers at some point before queue destruction. 946 * Otherwise, we would gather those references until we don't 947 * have any more space left and crash. 948 */ 949 r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv, 950 xa_limit_32b, GFP_KERNEL); 951 if (r) 952 goto free_fences; 953 954 amdgpu_userq_fence_driver_get(fence_drv); 955 956 /* Store drm syncobj's gpu va address and value */ 957 fence_info[cnt].va = fence_drv->va; 958 fence_info[cnt].value = fences[i]->seqno; 959 960 dma_fence_put(fences[i]); 961 /* Increment the actual userq fence count */ 962 cnt++; 963 } 964 965 wait_info->num_fences = cnt; 966 /* Copy userq fence info to user space */ 967 if (copy_to_user(u64_to_user_ptr(wait_info->out_fences), 968 fence_info, wait_info->num_fences * sizeof(*fence_info))) { 969 r = -EFAULT; 970 goto free_fences; 971 } 972 973 kfree(fences); 974 kfree(fence_info); 975 } 976 977 drm_exec_fini(&exec); 978 for (i = 0; i < num_read_bo_handles; i++) 979 drm_gem_object_put(gobj_read[i]); 980 kfree(gobj_read); 981 982 for (i = 0; i < num_write_bo_handles; i++) 983 drm_gem_object_put(gobj_write[i]); 984 kfree(gobj_write); 985 986 kfree(timeline_points); 987 kfree(timeline_handles); 988 kfree(syncobj_handles); 989 kfree(bo_handles_write); 990 kfree(bo_handles_read); 991 992 return 0; 993 994 free_fences: 995 while (num_fences-- > 0) 996 dma_fence_put(fences[num_fences]); 997 kfree(fences); 998 free_fence_info: 999 kfree(fence_info); 1000 exec_fini: 1001 drm_exec_fini(&exec); 1002 put_gobj_write: 1003 while (wentry-- > 0) 1004 drm_gem_object_put(gobj_write[wentry]); 1005 kfree(gobj_write); 1006 put_gobj_read: 1007 while (rentry-- > 0) 1008 drm_gem_object_put(gobj_read[rentry]); 1009 kfree(gobj_read); 1010 free_timeline_points: 1011 kfree(timeline_points); 1012 free_timeline_handles: 1013 kfree(timeline_handles); 1014 free_syncobj_handles: 1015 kfree(syncobj_handles); 1016 free_bo_handles_write: 1017 kfree(bo_handles_write); 1018 free_bo_handles_read: 1019 kfree(bo_handles_read); 1020 1021 return r; 1022 } 1023