1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2023 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/kref.h> 26 #include <linux/slab.h> 27 #include <linux/dma-fence-unwrap.h> 28 29 #include <drm/drm_exec.h> 30 #include <drm/drm_syncobj.h> 31 32 #include "amdgpu.h" 33 #include "amdgpu_trace.h" 34 35 #define AMDGPU_USERQ_MAX_HANDLES (1U << 16) 36 37 static const struct dma_fence_ops amdgpu_userq_fence_ops; 38 39 static inline struct amdgpu_userq_fence *to_amdgpu_userq_fence(struct dma_fence *f) 40 { 41 if (!f || f->ops != &amdgpu_userq_fence_ops) 42 return NULL; 43 44 return container_of(f, struct amdgpu_userq_fence, base); 45 } 46 47 static u64 amdgpu_userq_fence_read(struct amdgpu_userq_fence_driver *fence_drv) 48 { 49 return le64_to_cpu(*fence_drv->cpu_addr); 50 } 51 52 static void 53 amdgpu_userq_fence_write(struct amdgpu_userq_fence_driver *fence_drv, 54 u64 seq) 55 { 56 if (fence_drv->cpu_addr) 57 *fence_drv->cpu_addr = cpu_to_le64(seq); 58 } 59 60 int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev, 61 struct amdgpu_userq_fence_driver **fence_drv_req) 62 { 63 struct amdgpu_userq_fence_driver *fence_drv; 64 int r; 65 66 if (!fence_drv_req) 67 return -EINVAL; 68 *fence_drv_req = NULL; 69 70 fence_drv = kzalloc_obj(*fence_drv); 71 if (!fence_drv) 72 return -ENOMEM; 73 74 /* Acquire seq64 memory */ 75 r = amdgpu_seq64_alloc(adev, &fence_drv->va, &fence_drv->gpu_addr, 76 &fence_drv->cpu_addr); 77 if (r) 78 goto free_fence_drv; 79 80 memset(fence_drv->cpu_addr, 0, sizeof(u64)); 81 82 kref_init(&fence_drv->refcount); 83 INIT_LIST_HEAD(&fence_drv->fences); 84 spin_lock_init(&fence_drv->fence_list_lock); 85 86 fence_drv->adev = adev; 87 fence_drv->context = dma_fence_context_alloc(1); 88 get_task_comm(fence_drv->timeline_name, current); 89 90 *fence_drv_req = fence_drv; 91 92 return 0; 93 94 free_fence_drv: 95 kfree(fence_drv); 96 97 return r; 98 } 99 100 static void amdgpu_userq_walk_and_drop_fence_drv(struct xarray *xa) 101 { 102 struct amdgpu_userq_fence_driver *fence_drv; 103 unsigned long index; 104 105 if (xa_empty(xa)) 106 return; 107 108 xa_lock(xa); 109 xa_for_each(xa, index, fence_drv) { 110 __xa_erase(xa, index); 111 amdgpu_userq_fence_driver_put(fence_drv); 112 } 113 114 xa_unlock(xa); 115 } 116 117 void 118 amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq) 119 { 120 dma_fence_put(userq->last_fence); 121 userq->last_fence = NULL; 122 amdgpu_userq_walk_and_drop_fence_drv(&userq->fence_drv_xa); 123 xa_destroy(&userq->fence_drv_xa); 124 mutex_destroy(&userq->fence_drv_lock); 125 /* Drop the queue's ownership reference to fence_drv explicitly */ 126 amdgpu_userq_fence_driver_put(userq->fence_drv); 127 } 128 129 static void 130 amdgpu_userq_fence_put_fence_drv_array(struct amdgpu_userq_fence *userq_fence) 131 { 132 unsigned long i; 133 for (i = 0; i < userq_fence->fence_drv_array_count; i++) 134 amdgpu_userq_fence_driver_put(userq_fence->fence_drv_array[i]); 135 userq_fence->fence_drv_array_count = 0; 136 } 137 138 /* 139 * Returns: 140 * -ENOENT when no fences were processes 141 * 1 when more fences are pending 142 * 0 when no fences are pending any more 143 */ 144 int 145 amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv) 146 { 147 struct amdgpu_userq_fence *userq_fence, *tmp; 148 LIST_HEAD(to_be_signaled); 149 struct dma_fence *fence; 150 unsigned long flags; 151 u64 fence_val; 152 153 spin_lock_irqsave(&fence_drv->fence_list_lock, flags); 154 fence_val = amdgpu_userq_fence_read(fence_drv); 155 156 list_for_each_entry(userq_fence, &fence_drv->fences, link) { 157 if (fence_val < userq_fence->base.seqno) 158 break; 159 } 160 161 list_cut_before(&to_be_signaled, &fence_drv->fences, 162 &userq_fence->link); 163 spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags); 164 165 if (list_empty(&to_be_signaled)) 166 return -ENOENT; 167 168 list_for_each_entry_safe(userq_fence, tmp, &to_be_signaled, link) { 169 fence = &userq_fence->base; 170 list_del_init(&userq_fence->link); 171 dma_fence_signal(fence); 172 /* Drop fence_drv_array outside fence_list_lock 173 * to avoid the recursion lock. 174 */ 175 amdgpu_userq_fence_put_fence_drv_array(userq_fence); 176 dma_fence_put(fence); 177 } 178 179 /* That doesn't need to be accurate so no locking */ 180 return list_empty(&fence_drv->fences) ? 0 : 1; 181 } 182 183 void amdgpu_userq_fence_driver_destroy(struct kref *ref) 184 { 185 struct amdgpu_userq_fence_driver *fence_drv = container_of(ref, 186 struct amdgpu_userq_fence_driver, 187 refcount); 188 struct amdgpu_device *adev = fence_drv->adev; 189 struct amdgpu_userq_fence *fence, *tmp; 190 unsigned long flags; 191 struct dma_fence *f; 192 193 spin_lock_irqsave(&fence_drv->fence_list_lock, flags); 194 lockdep_assert_held(&fence_drv->fence_list_lock); 195 list_for_each_entry_safe(fence, tmp, &fence_drv->fences, link) { 196 f = &fence->base; 197 spin_lock(dma_fence_spinlock(f)); 198 if (!dma_fence_is_signaled_locked(f)) { 199 dma_fence_set_error(f, -ECANCELED); 200 dma_fence_signal_locked(f); 201 } 202 spin_unlock(dma_fence_spinlock(f)); 203 list_del(&fence->link); 204 dma_fence_put(f); 205 } 206 spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags); 207 208 /* Free seq64 memory */ 209 amdgpu_seq64_free(adev, fence_drv->va); 210 kfree(fence_drv); 211 } 212 213 void amdgpu_userq_fence_driver_get(struct amdgpu_userq_fence_driver *fence_drv) 214 { 215 kref_get(&fence_drv->refcount); 216 } 217 218 void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv) 219 { 220 kref_put(&fence_drv->refcount, amdgpu_userq_fence_driver_destroy); 221 } 222 223 static int amdgpu_userq_fence_alloc(struct amdgpu_usermode_queue *userq, 224 struct amdgpu_userq_fence **pfence) 225 { 226 struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv; 227 struct amdgpu_userq_fence *userq_fence; 228 void *entry; 229 230 userq_fence = kmalloc_obj(*userq_fence); 231 if (!userq_fence) 232 return -ENOMEM; 233 234 /* 235 * Get the next unused entry, since we fill from the start this can be 236 * used as size to allocate the array. 237 */ 238 mutex_lock(&userq->fence_drv_lock); 239 XA_STATE(xas, &userq->fence_drv_xa, 0); 240 241 rcu_read_lock(); 242 do { 243 entry = xas_find_marked(&xas, ULONG_MAX, XA_FREE_MARK); 244 } while (xas_retry(&xas, entry)); 245 rcu_read_unlock(); 246 247 userq_fence->fence_drv_array = kvmalloc_objs(fence_drv, xas.xa_index); 248 if (!userq_fence->fence_drv_array) { 249 mutex_unlock(&userq->fence_drv_lock); 250 kfree(userq_fence); 251 return -ENOMEM; 252 } 253 254 userq_fence->fence_drv_array_count = xas.xa_index; 255 xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array, 256 0, ULONG_MAX, xas.xa_index, XA_PRESENT); 257 xa_destroy(&userq->fence_drv_xa); 258 259 mutex_unlock(&userq->fence_drv_lock); 260 261 amdgpu_userq_fence_driver_get(fence_drv); 262 userq_fence->fence_drv = fence_drv; 263 264 *pfence = userq_fence; 265 return 0; 266 } 267 268 static void amdgpu_userq_fence_init(struct amdgpu_usermode_queue *userq, 269 struct amdgpu_userq_fence *fence, 270 u64 seq) 271 { 272 struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv; 273 unsigned long flags; 274 bool signaled = false; 275 276 spin_lock_init(&fence->lock); 277 dma_fence_init64(&fence->base, &amdgpu_userq_fence_ops, &fence->lock, 278 fence_drv->context, seq); 279 280 /* Make sure the fence is visible to the hang detect worker */ 281 dma_fence_put(userq->last_fence); 282 userq->last_fence = dma_fence_get(&fence->base); 283 284 /* Check if hardware has already processed the fence */ 285 spin_lock_irqsave(&fence_drv->fence_list_lock, flags); 286 if (!dma_fence_is_signaled(&fence->base)) { 287 dma_fence_get(&fence->base); 288 list_add_tail(&fence->link, &fence_drv->fences); 289 } else { 290 INIT_LIST_HEAD(&fence->link); 291 signaled = true; 292 } 293 spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags); 294 295 if (signaled) 296 amdgpu_userq_fence_put_fence_drv_array(fence); 297 else 298 amdgpu_userq_start_hang_detect_work(userq); 299 } 300 301 static const char *amdgpu_userq_fence_get_driver_name(struct dma_fence *f) 302 { 303 return "amdgpu_userq_fence"; 304 } 305 306 static const char *amdgpu_userq_fence_get_timeline_name(struct dma_fence *f) 307 { 308 struct amdgpu_userq_fence *fence = to_amdgpu_userq_fence(f); 309 310 return fence->fence_drv->timeline_name; 311 } 312 313 static bool amdgpu_userq_fence_signaled(struct dma_fence *f) 314 { 315 struct amdgpu_userq_fence *fence = to_amdgpu_userq_fence(f); 316 struct amdgpu_userq_fence_driver *fence_drv = fence->fence_drv; 317 u64 fence_val, wptr; 318 319 fence_val = amdgpu_userq_fence_read(fence_drv); 320 wptr = fence->base.seqno; 321 322 if (fence_val >= wptr) 323 return true; 324 325 return false; 326 } 327 328 static void amdgpu_userq_fence_free(struct rcu_head *rcu) 329 { 330 struct dma_fence *fence = container_of(rcu, struct dma_fence, rcu); 331 struct amdgpu_userq_fence *userq_fence = to_amdgpu_userq_fence(fence); 332 struct amdgpu_userq_fence_driver *fence_drv = userq_fence->fence_drv; 333 334 /* Release the fence driver reference */ 335 amdgpu_userq_fence_driver_put(fence_drv); 336 337 kvfree(userq_fence->fence_drv_array); 338 kfree(userq_fence); 339 } 340 341 static void amdgpu_userq_fence_release(struct dma_fence *f) 342 { 343 call_rcu(&f->rcu, amdgpu_userq_fence_free); 344 } 345 346 static const struct dma_fence_ops amdgpu_userq_fence_ops = { 347 .get_driver_name = amdgpu_userq_fence_get_driver_name, 348 .get_timeline_name = amdgpu_userq_fence_get_timeline_name, 349 .signaled = amdgpu_userq_fence_signaled, 350 .release = amdgpu_userq_fence_release, 351 }; 352 353 /** 354 * amdgpu_userq_fence_read_wptr - Read the userq wptr value 355 * 356 * @adev: amdgpu_device pointer 357 * @queue: user mode queue structure pointer 358 * @wptr: write pointer value 359 * 360 * Read the wptr value from userq's MQD. The userq signal IOCTL 361 * creates a dma_fence for the shared buffers that expects the 362 * fence_val written to seq64 fence address >= WPTR. 363 * 364 * Returns wptr value on success, error on failure. 365 */ 366 static int amdgpu_userq_fence_read_wptr(struct amdgpu_device *adev, 367 struct amdgpu_usermode_queue *queue, 368 u64 *wptr) 369 { 370 struct amdgpu_bo_va_mapping *mapping; 371 struct amdgpu_bo *bo; 372 struct drm_exec exec; 373 u64 addr, offset, *ptr; 374 int ret; 375 376 addr = queue->userq_prop->wptr_gpu_addr; 377 addr &= AMDGPU_GMC_HOLE_MASK; 378 379 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 2); 380 drm_exec_until_all_locked(&exec) { 381 ret = amdgpu_vm_lock_pd(queue->vm, &exec, 1); 382 drm_exec_retry_on_contention(&exec); 383 if (unlikely(ret)) 384 goto lock_error; 385 386 mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, addr); 387 if (!mapping || !mapping->bo_va->base.bo) { 388 ret = -EINVAL; 389 goto lock_error; 390 } 391 392 ret = drm_exec_lock_obj(&exec, &mapping->bo_va->base.bo->tbo.base); 393 drm_exec_retry_on_contention(&exec); 394 if (unlikely(ret)) 395 goto lock_error; 396 } 397 398 bo = mapping->bo_va->base.bo; 399 ret = amdgpu_bo_kmap(bo, (void **)&ptr); 400 if (ret) { 401 DRM_ERROR("Failed mapping the userqueue wptr bo"); 402 goto lock_error; 403 } 404 405 /* The WPTR can be anywhere within the BO's VA mapping. */ 406 offset = addr - (mapping->start << AMDGPU_GPU_PAGE_SHIFT); 407 offset += mapping->offset; 408 *wptr = le64_to_cpu(ptr[offset / sizeof(*ptr)]); 409 410 amdgpu_bo_kunmap(bo); 411 drm_exec_fini(&exec); 412 return 0; 413 414 lock_error: 415 drm_exec_fini(&exec); 416 return ret; 417 } 418 419 static void 420 amdgpu_userq_fence_driver_set_error(struct amdgpu_userq_fence *fence, 421 int error) 422 { 423 struct amdgpu_userq_fence_driver *fence_drv = fence->fence_drv; 424 unsigned long flags; 425 struct dma_fence *f; 426 427 spin_lock_irqsave(&fence_drv->fence_list_lock, flags); 428 lockdep_assert_held(&fence_drv->fence_list_lock); 429 f = rcu_dereference_protected(&fence->base, 430 lockdep_is_held(&fence_drv->fence_list_lock)); 431 if (f) { 432 /* nest f->lock inside fence_list_lock */ 433 spin_lock(dma_fence_spinlock(f)); 434 if (!dma_fence_is_signaled_locked(f)) 435 dma_fence_set_error(f, error); 436 spin_unlock(dma_fence_spinlock(f)); 437 } 438 spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags); 439 } 440 441 void 442 amdgpu_userq_fence_driver_force_completion(struct amdgpu_usermode_queue *userq) 443 { 444 struct dma_fence *f = userq->last_fence; 445 446 if (f) { 447 struct amdgpu_userq_fence *fence = to_amdgpu_userq_fence(f); 448 struct amdgpu_userq_fence_driver *fence_drv = fence->fence_drv; 449 u64 wptr = fence->base.seqno; 450 451 amdgpu_userq_fence_driver_set_error(fence, -ECANCELED); 452 amdgpu_userq_fence_write(fence_drv, wptr); 453 amdgpu_userq_fence_driver_process(fence_drv); 454 455 } 456 } 457 458 int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data, 459 struct drm_file *filp) 460 { 461 struct amdgpu_device *adev = drm_to_adev(dev); 462 struct drm_amdgpu_userq_signal *args = data; 463 const unsigned int num_write_bo_handles = args->num_bo_write_handles; 464 const unsigned int num_read_bo_handles = args->num_bo_read_handles; 465 struct amdgpu_fpriv *fpriv = filp->driver_priv; 466 struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr; 467 468 struct drm_gem_object **gobj_write, **gobj_read; 469 u32 *syncobj_handles, num_syncobj_handles; 470 u64 *syncobj_points; 471 struct amdgpu_usermode_queue *queue; 472 struct amdgpu_userq_fence *fence; 473 struct { 474 struct drm_syncobj *syncobj; 475 struct dma_fence_chain *chain; 476 } *syncobj; 477 struct drm_exec exec; 478 void __user *ptr; 479 int r, i, entry; 480 u64 wptr; 481 482 if (!amdgpu_userq_enabled(dev)) 483 return -ENOTSUPP; 484 485 if (args->num_bo_write_handles > AMDGPU_USERQ_MAX_HANDLES || 486 args->num_bo_read_handles > AMDGPU_USERQ_MAX_HANDLES) 487 return -EINVAL; 488 489 num_syncobj_handles = args->num_syncobj_handles; 490 ptr = u64_to_user_ptr(args->syncobj_handles); 491 syncobj_handles = memdup_array_user(ptr, num_syncobj_handles, 492 sizeof(u32)); 493 if (IS_ERR(syncobj_handles)) 494 return PTR_ERR(syncobj_handles); 495 496 if (args->syncobj_points) { 497 ptr = u64_to_user_ptr(args->syncobj_points); 498 syncobj_points = memdup_array_user(ptr, num_syncobj_handles, 499 sizeof(u64)); 500 if (IS_ERR(syncobj_points)) { 501 r = PTR_ERR(syncobj_points); 502 goto free_syncobj_handles; 503 } 504 } else { 505 syncobj_points = NULL; 506 } 507 508 syncobj = kmalloc_objs(*syncobj, num_syncobj_handles); 509 if (!syncobj) { 510 r = -ENOMEM; 511 goto free_syncobj_points; 512 } 513 514 for (entry = 0; entry < num_syncobj_handles; entry++) { 515 syncobj[entry].chain = NULL; 516 syncobj[entry].syncobj = drm_syncobj_find(filp, syncobj_handles[entry]); 517 if (!syncobj[entry].syncobj) { 518 r = -ENOENT; 519 goto free_syncobj; 520 } 521 if (syncobj_points && syncobj_points[entry]) { 522 syncobj[entry].chain = dma_fence_chain_alloc(); 523 if (!syncobj[entry].chain) { 524 drm_syncobj_put(syncobj[entry].syncobj); 525 r = -ENOMEM; 526 goto free_syncobj; 527 } 528 } 529 } 530 531 ptr = u64_to_user_ptr(args->bo_read_handles); 532 r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read); 533 if (r) 534 goto free_syncobj; 535 536 ptr = u64_to_user_ptr(args->bo_write_handles); 537 r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles, 538 &gobj_write); 539 if (r) 540 goto put_gobj_read; 541 542 queue = amdgpu_userq_get(userq_mgr, args->queue_id); 543 if (!queue) { 544 r = -ENOENT; 545 goto put_gobj_write; 546 } 547 548 r = amdgpu_userq_fence_read_wptr(adev, queue, &wptr); 549 if (r) 550 goto put_queue; 551 552 r = amdgpu_userq_fence_alloc(queue, &fence); 553 if (r) 554 goto put_queue; 555 556 /* We are here means UQ is active, make sure the eviction fence is valid */ 557 amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr); 558 559 /* Create the new fence */ 560 amdgpu_userq_fence_init(queue, fence, wptr); 561 562 trace_amdgpu_userq_emit_fence(dev->dev, queue, fence); 563 564 mutex_unlock(&userq_mgr->userq_mutex); 565 566 /* 567 * This needs to come after the fence is created since 568 * amdgpu_userq_ensure_ev_fence() can't be called while holding the resv 569 * locks. 570 */ 571 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES, 572 (num_read_bo_handles + num_write_bo_handles)); 573 574 drm_exec_until_all_locked(&exec) { 575 r = drm_exec_prepare_array(&exec, gobj_read, 576 num_read_bo_handles, 1); 577 drm_exec_retry_on_contention(&exec); 578 if (r) 579 goto exec_fini; 580 581 r = drm_exec_prepare_array(&exec, gobj_write, 582 num_write_bo_handles, 1); 583 drm_exec_retry_on_contention(&exec); 584 if (r) 585 goto exec_fini; 586 } 587 588 /* And publish the new fence in the BOs and syncobj */ 589 for (i = 0; i < num_read_bo_handles; i++) 590 dma_resv_add_fence(gobj_read[i]->resv, &fence->base, 591 DMA_RESV_USAGE_READ); 592 593 for (i = 0; i < num_write_bo_handles; i++) 594 dma_resv_add_fence(gobj_write[i]->resv, &fence->base, 595 DMA_RESV_USAGE_WRITE); 596 597 for (i = 0; i < num_syncobj_handles; i++) { 598 if (syncobj[i].chain) { 599 drm_syncobj_add_point(syncobj[i].syncobj, syncobj[i].chain, 600 &fence->base, syncobj_points[i]); 601 syncobj[i].chain = NULL; 602 } else { 603 drm_syncobj_replace_fence(syncobj[i].syncobj, &fence->base); 604 } 605 } 606 607 exec_fini: 608 /* drop the reference acquired in fence creation function */ 609 dma_fence_put(&fence->base); 610 611 drm_exec_fini(&exec); 612 put_queue: 613 amdgpu_userq_put(queue); 614 put_gobj_write: 615 for (i = 0; i < num_write_bo_handles; i++) 616 drm_gem_object_put(gobj_write[i]); 617 kvfree(gobj_write); 618 put_gobj_read: 619 for (i = 0; i < num_read_bo_handles; i++) 620 drm_gem_object_put(gobj_read[i]); 621 kvfree(gobj_read); 622 free_syncobj: 623 while (entry-- > 0) { 624 drm_syncobj_put(syncobj[entry].syncobj); 625 dma_fence_chain_free(syncobj[entry].chain); 626 } 627 kfree(syncobj); 628 free_syncobj_points: 629 kfree(syncobj_points); 630 free_syncobj_handles: 631 kfree(syncobj_handles); 632 633 return r; 634 } 635 636 /* Count the number of expected fences so userspace can alloc a buffer */ 637 static int 638 amdgpu_userq_wait_count_fences(struct drm_file *filp, 639 struct drm_amdgpu_userq_wait *wait_info, 640 u32 *syncobj_handles, u64 *timeline_points, 641 u32 *timeline_handles, 642 struct drm_gem_object **gobj_write, 643 struct drm_gem_object **gobj_read) 644 { 645 int num_read_bo_handles, num_write_bo_handles; 646 struct dma_fence_unwrap iter; 647 struct dma_fence *fence, *f; 648 unsigned int num_fences = 0; 649 struct drm_exec exec; 650 int i, r; 651 652 /* 653 * This needs to be outside of the lock provided by drm_exec for 654 * DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT to work correctly. 655 */ 656 657 /* Count timeline fences */ 658 for (i = 0; i < wait_info->num_syncobj_timeline_handles; i++) { 659 r = drm_syncobj_find_fence(filp, timeline_handles[i], 660 timeline_points[i], 661 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT, 662 &fence); 663 if (r) 664 return r; 665 666 dma_fence_unwrap_for_each(f, &iter, fence) 667 num_fences++; 668 669 dma_fence_put(fence); 670 } 671 672 /* Count boolean fences */ 673 for (i = 0; i < wait_info->num_syncobj_handles; i++) { 674 r = drm_syncobj_find_fence(filp, syncobj_handles[i], 0, 675 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT, 676 &fence); 677 if (r) 678 return r; 679 680 num_fences++; 681 dma_fence_put(fence); 682 } 683 684 /* Lock all the GEM objects */ 685 /* TODO: It is actually not necessary to lock them */ 686 num_read_bo_handles = wait_info->num_bo_read_handles; 687 num_write_bo_handles = wait_info->num_bo_write_handles; 688 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES, 689 num_read_bo_handles + num_write_bo_handles); 690 691 drm_exec_until_all_locked(&exec) { 692 r = drm_exec_prepare_array(&exec, gobj_read, 693 num_read_bo_handles, 1); 694 drm_exec_retry_on_contention(&exec); 695 if (r) 696 goto error_unlock; 697 698 r = drm_exec_prepare_array(&exec, gobj_write, 699 num_write_bo_handles, 1); 700 drm_exec_retry_on_contention(&exec); 701 if (r) 702 goto error_unlock; 703 } 704 705 /* Count read fences */ 706 for (i = 0; i < num_read_bo_handles; i++) { 707 struct dma_resv_iter resv_cursor; 708 struct dma_fence *fence; 709 710 dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv, 711 DMA_RESV_USAGE_READ, fence) 712 num_fences++; 713 } 714 715 /* Count write fences */ 716 for (i = 0; i < num_write_bo_handles; i++) { 717 struct dma_resv_iter resv_cursor; 718 struct dma_fence *fence; 719 720 dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv, 721 DMA_RESV_USAGE_WRITE, fence) 722 num_fences++; 723 } 724 725 wait_info->num_fences = min(num_fences, USHRT_MAX); 726 r = 0; 727 728 error_unlock: 729 /* Unlock all the GEM objects */ 730 drm_exec_fini(&exec); 731 return r; 732 } 733 734 static int 735 amdgpu_userq_wait_add_fence(struct drm_amdgpu_userq_wait *wait_info, 736 struct dma_fence **fences, unsigned int *num_fences, 737 struct dma_fence *fence) 738 { 739 /* As fallback shouldn't userspace allocate enough space */ 740 if (*num_fences >= wait_info->num_fences) 741 return dma_fence_wait(fence, true); 742 743 fences[(*num_fences)++] = dma_fence_get(fence); 744 return 0; 745 } 746 747 static int 748 amdgpu_userq_wait_return_fence_info(struct drm_device *dev, struct drm_file *filp, 749 struct drm_amdgpu_userq_wait *wait_info, 750 u32 *syncobj_handles, u64 *timeline_points, 751 u32 *timeline_handles, 752 struct drm_gem_object **gobj_write, 753 struct drm_gem_object **gobj_read) 754 { 755 struct amdgpu_fpriv *fpriv = filp->driver_priv; 756 struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr; 757 struct drm_amdgpu_userq_fence_info *fence_info; 758 int num_read_bo_handles, num_write_bo_handles; 759 struct amdgpu_usermode_queue *waitq; 760 struct dma_fence **fences, *fence, *f; 761 struct dma_fence_unwrap iter; 762 int num_points, num_syncobj; 763 unsigned int num_fences = 0; 764 struct drm_exec exec; 765 int i, cnt, r; 766 767 fence_info = kmalloc_objs(*fence_info, wait_info->num_fences); 768 if (!fence_info) 769 return -ENOMEM; 770 771 fences = kmalloc_objs(*fences, wait_info->num_fences); 772 if (!fences) { 773 r = -ENOMEM; 774 goto free_fence_info; 775 } 776 777 /* Retrieve timeline fences */ 778 num_points = wait_info->num_syncobj_timeline_handles; 779 for (i = 0; i < num_points; i++) { 780 r = drm_syncobj_find_fence(filp, timeline_handles[i], 781 timeline_points[i], 782 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT, 783 &fence); 784 if (r) 785 goto free_fences; 786 787 dma_fence_unwrap_for_each(f, &iter, fence) { 788 r = amdgpu_userq_wait_add_fence(wait_info, fences, 789 &num_fences, f); 790 if (r) { 791 dma_fence_put(fence); 792 goto free_fences; 793 } 794 } 795 796 dma_fence_put(fence); 797 } 798 799 /* Retrieve boolean fences */ 800 num_syncobj = wait_info->num_syncobj_handles; 801 for (i = 0; i < num_syncobj; i++) { 802 struct dma_fence *fence; 803 804 r = drm_syncobj_find_fence(filp, syncobj_handles[i], 0, 805 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT, 806 &fence); 807 if (r) 808 goto free_fences; 809 810 r = amdgpu_userq_wait_add_fence(wait_info, fences, 811 &num_fences, fence); 812 dma_fence_put(fence); 813 if (r) 814 goto free_fences; 815 816 } 817 818 /* Lock all the GEM objects */ 819 num_read_bo_handles = wait_info->num_bo_read_handles; 820 num_write_bo_handles = wait_info->num_bo_write_handles; 821 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES, 822 num_read_bo_handles + num_write_bo_handles); 823 824 drm_exec_until_all_locked(&exec) { 825 r = drm_exec_prepare_array(&exec, gobj_read, 826 num_read_bo_handles, 1); 827 drm_exec_retry_on_contention(&exec); 828 if (r) 829 goto error_unlock; 830 831 r = drm_exec_prepare_array(&exec, gobj_write, 832 num_write_bo_handles, 1); 833 drm_exec_retry_on_contention(&exec); 834 if (r) 835 goto error_unlock; 836 } 837 838 /* Retrieve GEM read objects fence */ 839 for (i = 0; i < num_read_bo_handles; i++) { 840 struct dma_resv_iter resv_cursor; 841 struct dma_fence *fence; 842 843 dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv, 844 DMA_RESV_USAGE_READ, fence) { 845 r = amdgpu_userq_wait_add_fence(wait_info, fences, 846 &num_fences, fence); 847 if (r) 848 goto error_unlock; 849 } 850 } 851 852 /* Retrieve GEM write objects fence */ 853 for (i = 0; i < num_write_bo_handles; i++) { 854 struct dma_resv_iter resv_cursor; 855 struct dma_fence *fence; 856 857 dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv, 858 DMA_RESV_USAGE_WRITE, fence) { 859 r = amdgpu_userq_wait_add_fence(wait_info, fences, 860 &num_fences, fence); 861 if (r) 862 goto error_unlock; 863 } 864 } 865 866 drm_exec_fini(&exec); 867 868 /* 869 * Keep only the latest fences to reduce the number of values 870 * given back to userspace. 871 */ 872 num_fences = dma_fence_dedup_array(fences, num_fences); 873 874 waitq = amdgpu_userq_get(userq_mgr, wait_info->waitq_id); 875 if (!waitq) { 876 r = -EINVAL; 877 goto free_fences; 878 } 879 880 for (i = 0, cnt = 0; i < num_fences; i++) { 881 struct amdgpu_userq_fence_driver *fence_drv; 882 struct amdgpu_userq_fence *userq_fence; 883 u32 index; 884 885 userq_fence = to_amdgpu_userq_fence(fences[i]); 886 if (!userq_fence) { 887 /* 888 * Just waiting on other driver fences should 889 * be good for now 890 */ 891 r = dma_fence_wait(fences[i], true); 892 if (r) 893 goto put_waitq; 894 895 continue; 896 } 897 898 fence_drv = userq_fence->fence_drv; 899 /* 900 * We need to make sure the user queue release their reference 901 * to the fence drivers at some point before queue destruction. 902 * Otherwise, we would gather those references until we don't 903 * have any more space left and crash. 904 */ 905 mutex_lock(&waitq->fence_drv_lock); 906 r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv, 907 xa_limit_32b, GFP_KERNEL); 908 mutex_unlock(&waitq->fence_drv_lock); 909 if (r) 910 goto put_waitq; 911 912 amdgpu_userq_fence_driver_get(fence_drv); 913 914 trace_amdgpu_userq_wait_deps(dev->dev, waitq, userq_fence); 915 916 /* Store drm syncobj's gpu va address and value */ 917 fence_info[cnt].va = fence_drv->va; 918 fence_info[cnt].value = fences[i]->seqno; 919 920 /* Increment the actual userq fence count */ 921 cnt++; 922 } 923 wait_info->num_fences = cnt; 924 925 /* Copy userq fence info to user space */ 926 if (copy_to_user(u64_to_user_ptr(wait_info->out_fences), 927 fence_info, cnt * sizeof(*fence_info))) 928 r = -EFAULT; 929 else 930 r = 0; 931 932 put_waitq: 933 amdgpu_userq_put(waitq); 934 935 free_fences: 936 while (num_fences--) 937 dma_fence_put(fences[num_fences]); 938 kfree(fences); 939 940 free_fence_info: 941 kfree(fence_info); 942 return r; 943 944 error_unlock: 945 drm_exec_fini(&exec); 946 goto free_fences; 947 } 948 949 int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, 950 struct drm_file *filp) 951 { 952 int num_points, num_syncobj, num_read_bo_handles, num_write_bo_handles; 953 u32 *syncobj_handles, *timeline_handles; 954 u64 *timeline_points; 955 struct drm_amdgpu_userq_wait *wait_info = data; 956 struct drm_gem_object **gobj_write; 957 struct drm_gem_object **gobj_read; 958 void __user *ptr; 959 int r; 960 961 if (!amdgpu_userq_enabled(dev)) 962 return -ENOTSUPP; 963 964 if (wait_info->num_bo_write_handles > AMDGPU_USERQ_MAX_HANDLES || 965 wait_info->num_bo_read_handles > AMDGPU_USERQ_MAX_HANDLES) 966 return -EINVAL; 967 968 num_syncobj = wait_info->num_syncobj_handles; 969 ptr = u64_to_user_ptr(wait_info->syncobj_handles); 970 syncobj_handles = memdup_array_user(ptr, num_syncobj, sizeof(u32)); 971 if (IS_ERR(syncobj_handles)) 972 return PTR_ERR(syncobj_handles); 973 974 num_points = wait_info->num_syncobj_timeline_handles; 975 ptr = u64_to_user_ptr(wait_info->syncobj_timeline_handles); 976 timeline_handles = memdup_array_user(ptr, num_points, sizeof(u32)); 977 if (IS_ERR(timeline_handles)) { 978 r = PTR_ERR(timeline_handles); 979 goto free_syncobj_handles; 980 } 981 982 ptr = u64_to_user_ptr(wait_info->syncobj_timeline_points); 983 timeline_points = memdup_array_user(ptr, num_points, sizeof(u64)); 984 if (IS_ERR(timeline_points)) { 985 r = PTR_ERR(timeline_points); 986 goto free_timeline_handles; 987 } 988 989 num_read_bo_handles = wait_info->num_bo_read_handles; 990 ptr = u64_to_user_ptr(wait_info->bo_read_handles); 991 r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read); 992 if (r) 993 goto free_timeline_points; 994 995 num_write_bo_handles = wait_info->num_bo_write_handles; 996 ptr = u64_to_user_ptr(wait_info->bo_write_handles); 997 r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles, 998 &gobj_write); 999 if (r) 1000 goto put_gobj_read; 1001 1002 /* 1003 * Passing num_fences = 0 means that userspace doesn't want to 1004 * retrieve userq_fence_info. If num_fences = 0 we skip filling 1005 * userq_fence_info and return the actual number of fences on 1006 * args->num_fences. 1007 */ 1008 if (!wait_info->num_fences) { 1009 r = amdgpu_userq_wait_count_fences(filp, wait_info, 1010 syncobj_handles, 1011 timeline_points, 1012 timeline_handles, 1013 gobj_write, 1014 gobj_read); 1015 } else { 1016 r = amdgpu_userq_wait_return_fence_info(dev, filp, wait_info, 1017 syncobj_handles, 1018 timeline_points, 1019 timeline_handles, 1020 gobj_write, 1021 gobj_read); 1022 } 1023 1024 while (num_write_bo_handles--) 1025 drm_gem_object_put(gobj_write[num_write_bo_handles]); 1026 kvfree(gobj_write); 1027 1028 put_gobj_read: 1029 while (num_read_bo_handles--) 1030 drm_gem_object_put(gobj_read[num_read_bo_handles]); 1031 kvfree(gobj_read); 1032 1033 free_timeline_points: 1034 kfree(timeline_points); 1035 free_timeline_handles: 1036 kfree(timeline_handles); 1037 free_syncobj_handles: 1038 kfree(syncobj_handles); 1039 return r; 1040 } 1041