1 /* 2 * Copyright 2017 Red Hat 3 * Parts ported from amdgpu (fence wait code). 4 * Copyright 2016 Advanced Micro Devices, Inc. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 * IN THE SOFTWARE. 24 * 25 * Authors: 26 * 27 */ 28 29 /** 30 * DOC: Overview 31 * 32 * DRM synchronisation objects (syncobj, see struct &drm_syncobj) provide a 33 * container for a synchronization primitive which can be used by userspace 34 * to explicitly synchronize GPU commands, can be shared between userspace 35 * processes, and can be shared between different DRM drivers. 36 * Their primary use-case is to implement Vulkan fences and semaphores. 37 * The syncobj userspace API provides ioctls for several operations: 38 * 39 * - Creation and destruction of syncobjs 40 * - Import and export of syncobjs to/from a syncobj file descriptor 41 * - Import and export a syncobj's underlying fence to/from a sync file 42 * - Reset a syncobj (set its fence to NULL) 43 * - Signal a syncobj (set a trivially signaled fence) 44 * - Wait for a syncobj's fence to appear and be signaled 45 * 46 * The syncobj userspace API also provides operations to manipulate a syncobj 47 * in terms of a timeline of struct &dma_fence_chain rather than a single 48 * struct &dma_fence, through the following operations: 49 * 50 * - Signal a given point on the timeline 51 * - Wait for a given point to appear and/or be signaled 52 * - Import and export from/to a given point of a timeline 53 * 54 * At it's core, a syncobj is simply a wrapper around a pointer to a struct 55 * &dma_fence which may be NULL. 56 * When a syncobj is first created, its pointer is either NULL or a pointer 57 * to an already signaled fence depending on whether the 58 * &DRM_SYNCOBJ_CREATE_SIGNALED flag is passed to 59 * &DRM_IOCTL_SYNCOBJ_CREATE. 60 * 61 * If the syncobj is considered as a binary (its state is either signaled or 62 * unsignaled) primitive, when GPU work is enqueued in a DRM driver to signal 63 * the syncobj, the syncobj's fence is replaced with a fence which will be 64 * signaled by the completion of that work. 65 * If the syncobj is considered as a timeline primitive, when GPU work is 66 * enqueued in a DRM driver to signal the a given point of the syncobj, a new 67 * struct &dma_fence_chain pointing to the DRM driver's fence and also 68 * pointing to the previous fence that was in the syncobj. The new struct 69 * &dma_fence_chain fence replace the syncobj's fence and will be signaled by 70 * completion of the DRM driver's work and also any work associated with the 71 * fence previously in the syncobj. 72 * 73 * When GPU work which waits on a syncobj is enqueued in a DRM driver, at the 74 * time the work is enqueued, it waits on the syncobj's fence before 75 * submitting the work to hardware. That fence is either : 76 * 77 * - The syncobj's current fence if the syncobj is considered as a binary 78 * primitive. 79 * - The struct &dma_fence associated with a given point if the syncobj is 80 * considered as a timeline primitive. 81 * 82 * If the syncobj's fence is NULL or not present in the syncobj's timeline, 83 * the enqueue operation is expected to fail. 84 * 85 * With binary syncobj, all manipulation of the syncobjs's fence happens in 86 * terms of the current fence at the time the ioctl is called by userspace 87 * regardless of whether that operation is an immediate host-side operation 88 * (signal or reset) or or an operation which is enqueued in some driver 89 * queue. &DRM_IOCTL_SYNCOBJ_RESET and &DRM_IOCTL_SYNCOBJ_SIGNAL can be used 90 * to manipulate a syncobj from the host by resetting its pointer to NULL or 91 * setting its pointer to a fence which is already signaled. 92 * 93 * With a timeline syncobj, all manipulation of the synobj's fence happens in 94 * terms of a u64 value referring to point in the timeline. See 95 * dma_fence_chain_find_seqno() to see how a given point is found in the 96 * timeline. 97 * 98 * Note that applications should be careful to always use timeline set of 99 * ioctl() when dealing with syncobj considered as timeline. Using a binary 100 * set of ioctl() with a syncobj considered as timeline could result incorrect 101 * synchronization. The use of binary syncobj is supported through the 102 * timeline set of ioctl() by using a point value of 0, this will reproduce 103 * the behavior of the binary set of ioctl() (for example replace the 104 * syncobj's fence when signaling). 105 * 106 * 107 * Host-side wait on syncobjs 108 * -------------------------- 109 * 110 * &DRM_IOCTL_SYNCOBJ_WAIT takes an array of syncobj handles and does a 111 * host-side wait on all of the syncobj fences simultaneously. 112 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL is set, the wait ioctl will wait on 113 * all of the syncobj fences to be signaled before it returns. 114 * Otherwise, it returns once at least one syncobj fence has been signaled 115 * and the index of a signaled fence is written back to the client. 116 * 117 * Unlike the enqueued GPU work dependencies which fail if they see a NULL 118 * fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set, 119 * the host-side wait will first wait for the syncobj to receive a non-NULL 120 * fence and then wait on that fence. 121 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is not set and any one of the 122 * syncobjs in the array has a NULL fence, -EINVAL will be returned. 123 * Assuming the syncobj starts off with a NULL fence, this allows a client 124 * to do a host wait in one thread (or process) which waits on GPU work 125 * submitted in another thread (or process) without having to manually 126 * synchronize between the two. 127 * This requirement is inherited from the Vulkan fence API. 128 * 129 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE is set, the ioctl will also set 130 * a fence deadline hint on the backing fences before waiting, to provide the 131 * fence signaler with an appropriate sense of urgency. The deadline is 132 * specified as an absolute &CLOCK_MONOTONIC value in units of ns. 133 * 134 * Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj 135 * handles as well as an array of u64 points and does a host-side wait on all 136 * of syncobj fences at the given points simultaneously. 137 * 138 * &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT also adds the ability to wait for a given 139 * fence to materialize on the timeline without waiting for the fence to be 140 * signaled by using the &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE flag. This 141 * requirement is inherited from the wait-before-signal behavior required by 142 * the Vulkan timeline semaphore API. 143 * 144 * Alternatively, &DRM_IOCTL_SYNCOBJ_EVENTFD can be used to wait without 145 * blocking: an eventfd will be signaled when the syncobj is. This is useful to 146 * integrate the wait in an event loop. 147 * 148 * 149 * Import/export of syncobjs 150 * ------------------------- 151 * 152 * &DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE and &DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD 153 * provide two mechanisms for import/export of syncobjs. 154 * 155 * The first lets the client import or export an entire syncobj to a file 156 * descriptor. 157 * These fd's are opaque and have no other use case, except passing the 158 * syncobj between processes. 159 * All exported file descriptors and any syncobj handles created as a 160 * result of importing those file descriptors own a reference to the 161 * same underlying struct &drm_syncobj and the syncobj can be used 162 * persistently across all the processes with which it is shared. 163 * The syncobj is freed only once the last reference is dropped. 164 * Unlike dma-buf, importing a syncobj creates a new handle (with its own 165 * reference) for every import instead of de-duplicating. 166 * The primary use-case of this persistent import/export is for shared 167 * Vulkan fences and semaphores. 168 * 169 * The second import/export mechanism, which is indicated by 170 * &DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE or 171 * &DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE lets the client 172 * import/export the syncobj's current fence from/to a &sync_file. 173 * When a syncobj is exported to a sync file, that sync file wraps the 174 * sycnobj's fence at the time of export and any later signal or reset 175 * operations on the syncobj will not affect the exported sync file. 176 * When a sync file is imported into a syncobj, the syncobj's fence is set 177 * to the fence wrapped by that sync file. 178 * Because sync files are immutable, resetting or signaling the syncobj 179 * will not affect any sync files whose fences have been imported into the 180 * syncobj. 181 * 182 * 183 * Import/export of timeline points in timeline syncobjs 184 * ----------------------------------------------------- 185 * 186 * &DRM_IOCTL_SYNCOBJ_TRANSFER provides a mechanism to transfer a struct 187 * &dma_fence_chain of a syncobj at a given u64 point to another u64 point 188 * into another syncobj. 189 * 190 * Note that if you want to transfer a struct &dma_fence_chain from a given 191 * point on a timeline syncobj from/into a binary syncobj, you can use the 192 * point 0 to mean take/replace the fence in the syncobj. 193 */ 194 195 #include <linux/anon_inodes.h> 196 #include <linux/dma-fence-unwrap.h> 197 #include <linux/eventfd.h> 198 #include <linux/file.h> 199 #include <linux/fs.h> 200 #include <linux/sched/signal.h> 201 #include <linux/sync_file.h> 202 #include <linux/uaccess.h> 203 204 #include <drm/drm.h> 205 #include <drm/drm_drv.h> 206 #include <drm/drm_file.h> 207 #include <drm/drm_gem.h> 208 #include <drm/drm_print.h> 209 #include <drm/drm_syncobj.h> 210 #include <drm/drm_utils.h> 211 212 #include "drm_internal.h" 213 214 struct syncobj_wait_entry { 215 struct list_head node; 216 struct task_struct *task; 217 struct dma_fence *fence; 218 struct dma_fence_cb fence_cb; 219 u64 point; 220 }; 221 222 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj, 223 struct syncobj_wait_entry *wait); 224 225 struct syncobj_eventfd_entry { 226 struct list_head node; 227 struct dma_fence *fence; 228 struct dma_fence_cb fence_cb; 229 struct drm_syncobj *syncobj; 230 struct eventfd_ctx *ev_fd_ctx; 231 u64 point; 232 u32 flags; 233 }; 234 235 static void 236 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj, 237 struct syncobj_eventfd_entry *entry); 238 239 /** 240 * drm_syncobj_find - lookup and reference a sync object. 241 * @file_private: drm file private pointer 242 * @handle: sync object handle to lookup. 243 * 244 * Returns a reference to the syncobj pointed to by handle or NULL. The 245 * reference must be released by calling drm_syncobj_put(). 246 */ 247 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private, 248 u32 handle) 249 { 250 struct drm_syncobj *syncobj; 251 252 spin_lock(&file_private->syncobj_table_lock); 253 254 /* Check if we currently have a reference on the object */ 255 syncobj = idr_find(&file_private->syncobj_idr, handle); 256 if (syncobj) 257 drm_syncobj_get(syncobj); 258 259 spin_unlock(&file_private->syncobj_table_lock); 260 261 return syncobj; 262 } 263 EXPORT_SYMBOL(drm_syncobj_find); 264 265 static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj, 266 struct syncobj_wait_entry *wait) 267 { 268 struct dma_fence *fence; 269 270 if (wait->fence) 271 return; 272 273 spin_lock(&syncobj->lock); 274 /* We've already tried once to get a fence and failed. Now that we 275 * have the lock, try one more time just to be sure we don't add a 276 * callback when a fence has already been set. 277 */ 278 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1)); 279 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) { 280 dma_fence_put(fence); 281 list_add_tail(&wait->node, &syncobj->cb_list); 282 } else if (!fence) { 283 wait->fence = dma_fence_get_stub(); 284 } else { 285 wait->fence = fence; 286 } 287 spin_unlock(&syncobj->lock); 288 } 289 290 static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj, 291 struct syncobj_wait_entry *wait) 292 { 293 if (!wait->node.next) 294 return; 295 296 spin_lock(&syncobj->lock); 297 list_del_init(&wait->node); 298 spin_unlock(&syncobj->lock); 299 } 300 301 static void 302 syncobj_eventfd_entry_free(struct syncobj_eventfd_entry *entry) 303 { 304 eventfd_ctx_put(entry->ev_fd_ctx); 305 dma_fence_put(entry->fence); 306 /* This happens either inside the syncobj lock, or after the node has 307 * already been removed from the list. 308 */ 309 list_del(&entry->node); 310 kfree(entry); 311 } 312 313 static void 314 drm_syncobj_add_eventfd(struct drm_syncobj *syncobj, 315 struct syncobj_eventfd_entry *entry) 316 { 317 spin_lock(&syncobj->lock); 318 list_add_tail(&entry->node, &syncobj->ev_fd_list); 319 syncobj_eventfd_entry_func(syncobj, entry); 320 spin_unlock(&syncobj->lock); 321 } 322 323 /** 324 * drm_syncobj_add_point - add new timeline point to the syncobj 325 * @syncobj: sync object to add timeline point do 326 * @chain: chain node to use to add the point 327 * @fence: fence to encapsulate in the chain node 328 * @point: sequence number to use for the point 329 * 330 * Add the chain node as new timeline point to the syncobj. 331 */ 332 void drm_syncobj_add_point(struct drm_syncobj *syncobj, 333 struct dma_fence_chain *chain, 334 struct dma_fence *fence, 335 uint64_t point) 336 { 337 struct syncobj_wait_entry *wait_cur, *wait_tmp; 338 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 339 struct dma_fence *prev; 340 341 dma_fence_get(fence); 342 343 spin_lock(&syncobj->lock); 344 345 prev = drm_syncobj_fence_get(syncobj); 346 /* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */ 347 if (prev && prev->seqno >= point) 348 DRM_DEBUG("You are adding an unorder point to timeline!\n"); 349 dma_fence_chain_init(chain, prev, fence, point); 350 rcu_assign_pointer(syncobj->fence, &chain->base); 351 352 list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node) 353 syncobj_wait_syncobj_func(syncobj, wait_cur); 354 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 355 syncobj_eventfd_entry_func(syncobj, ev_fd_cur); 356 spin_unlock(&syncobj->lock); 357 358 /* Walk the chain once to trigger garbage collection */ 359 dma_fence_chain_for_each(fence, prev); 360 dma_fence_put(prev); 361 } 362 EXPORT_SYMBOL(drm_syncobj_add_point); 363 364 /** 365 * drm_syncobj_replace_fence - replace fence in a sync object. 366 * @syncobj: Sync object to replace fence in 367 * @fence: fence to install in sync file. 368 * 369 * This replaces the fence on a sync object. 370 */ 371 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj, 372 struct dma_fence *fence) 373 { 374 struct dma_fence *old_fence; 375 struct syncobj_wait_entry *wait_cur, *wait_tmp; 376 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 377 378 if (fence) 379 dma_fence_get(fence); 380 381 spin_lock(&syncobj->lock); 382 383 old_fence = rcu_dereference_protected(syncobj->fence, 384 lockdep_is_held(&syncobj->lock)); 385 rcu_assign_pointer(syncobj->fence, fence); 386 387 if (fence != old_fence) { 388 list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node) 389 syncobj_wait_syncobj_func(syncobj, wait_cur); 390 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 391 syncobj_eventfd_entry_func(syncobj, ev_fd_cur); 392 } 393 394 spin_unlock(&syncobj->lock); 395 396 dma_fence_put(old_fence); 397 } 398 EXPORT_SYMBOL(drm_syncobj_replace_fence); 399 400 /** 401 * drm_syncobj_assign_null_handle - assign a stub fence to the sync object 402 * @syncobj: sync object to assign the fence on 403 * 404 * Assign a already signaled stub fence to the sync object. 405 */ 406 static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj) 407 { 408 struct dma_fence *fence = dma_fence_allocate_private_stub(ktime_get()); 409 410 if (!fence) 411 return -ENOMEM; 412 413 drm_syncobj_replace_fence(syncobj, fence); 414 dma_fence_put(fence); 415 return 0; 416 } 417 418 /* 5s default for wait submission */ 419 #define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL 420 /** 421 * drm_syncobj_find_fence - lookup and reference the fence in a sync object 422 * @file_private: drm file private pointer 423 * @handle: sync object handle to lookup. 424 * @point: timeline point 425 * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not 426 * @fence: out parameter for the fence 427 * 428 * This is just a convenience function that combines drm_syncobj_find() and 429 * drm_syncobj_fence_get(). 430 * 431 * Returns 0 on success or a negative error value on failure. On success @fence 432 * contains a reference to the fence, which must be released by calling 433 * dma_fence_put(). 434 */ 435 int drm_syncobj_find_fence(struct drm_file *file_private, 436 u32 handle, u64 point, u64 flags, 437 struct dma_fence **fence) 438 { 439 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle); 440 struct syncobj_wait_entry wait; 441 u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT); 442 int ret; 443 444 if (flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) 445 return -EINVAL; 446 447 if (!syncobj) 448 return -ENOENT; 449 450 /* Waiting for userspace with locks help is illegal cause that can 451 * trivial deadlock with page faults for example. Make lockdep complain 452 * about it early on. 453 */ 454 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) { 455 might_sleep(); 456 lockdep_assert_none_held_once(); 457 } 458 459 *fence = drm_syncobj_fence_get(syncobj); 460 461 if (*fence) { 462 ret = dma_fence_chain_find_seqno(fence, point); 463 if (!ret) { 464 /* If the requested seqno is already signaled 465 * drm_syncobj_find_fence may return a NULL 466 * fence. To make sure the recipient gets 467 * signalled, use a new fence instead. 468 */ 469 if (!*fence) 470 *fence = dma_fence_get_stub(); 471 472 goto out; 473 } 474 dma_fence_put(*fence); 475 } else { 476 ret = -EINVAL; 477 } 478 479 if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)) 480 goto out; 481 482 memset(&wait, 0, sizeof(wait)); 483 wait.task = current; 484 wait.point = point; 485 drm_syncobj_fence_add_wait(syncobj, &wait); 486 487 do { 488 set_current_state(TASK_INTERRUPTIBLE); 489 if (wait.fence) { 490 ret = 0; 491 break; 492 } 493 if (timeout == 0) { 494 ret = -ETIME; 495 break; 496 } 497 498 if (signal_pending(current)) { 499 ret = -ERESTARTSYS; 500 break; 501 } 502 503 timeout = schedule_timeout(timeout); 504 } while (1); 505 506 __set_current_state(TASK_RUNNING); 507 *fence = wait.fence; 508 509 if (wait.node.next) 510 drm_syncobj_remove_wait(syncobj, &wait); 511 512 out: 513 drm_syncobj_put(syncobj); 514 515 return ret; 516 } 517 EXPORT_SYMBOL(drm_syncobj_find_fence); 518 519 /** 520 * drm_syncobj_free - free a sync object. 521 * @kref: kref to free. 522 * 523 * Only to be called from kref_put in drm_syncobj_put. 524 */ 525 void drm_syncobj_free(struct kref *kref) 526 { 527 struct drm_syncobj *syncobj = container_of(kref, 528 struct drm_syncobj, 529 refcount); 530 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 531 532 drm_syncobj_replace_fence(syncobj, NULL); 533 534 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 535 syncobj_eventfd_entry_free(ev_fd_cur); 536 537 kfree(syncobj); 538 } 539 EXPORT_SYMBOL(drm_syncobj_free); 540 541 /** 542 * drm_syncobj_create - create a new syncobj 543 * @out_syncobj: returned syncobj 544 * @flags: DRM_SYNCOBJ_* flags 545 * @fence: if non-NULL, the syncobj will represent this fence 546 * 547 * This is the first function to create a sync object. After creating, drivers 548 * probably want to make it available to userspace, either through 549 * drm_syncobj_get_handle() or drm_syncobj_get_fd(). 550 * 551 * Returns 0 on success or a negative error value on failure. 552 */ 553 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags, 554 struct dma_fence *fence) 555 { 556 int ret; 557 struct drm_syncobj *syncobj; 558 559 syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL); 560 if (!syncobj) 561 return -ENOMEM; 562 563 kref_init(&syncobj->refcount); 564 INIT_LIST_HEAD(&syncobj->cb_list); 565 INIT_LIST_HEAD(&syncobj->ev_fd_list); 566 spin_lock_init(&syncobj->lock); 567 568 if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) { 569 ret = drm_syncobj_assign_null_handle(syncobj); 570 if (ret < 0) { 571 drm_syncobj_put(syncobj); 572 return ret; 573 } 574 } 575 576 if (fence) 577 drm_syncobj_replace_fence(syncobj, fence); 578 579 *out_syncobj = syncobj; 580 return 0; 581 } 582 EXPORT_SYMBOL(drm_syncobj_create); 583 584 /** 585 * drm_syncobj_get_handle - get a handle from a syncobj 586 * @file_private: drm file private pointer 587 * @syncobj: Sync object to export 588 * @handle: out parameter with the new handle 589 * 590 * Exports a sync object created with drm_syncobj_create() as a handle on 591 * @file_private to userspace. 592 * 593 * Returns 0 on success or a negative error value on failure. 594 */ 595 int drm_syncobj_get_handle(struct drm_file *file_private, 596 struct drm_syncobj *syncobj, u32 *handle) 597 { 598 int ret; 599 600 /* take a reference to put in the idr */ 601 drm_syncobj_get(syncobj); 602 603 idr_preload(GFP_KERNEL); 604 spin_lock(&file_private->syncobj_table_lock); 605 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT); 606 spin_unlock(&file_private->syncobj_table_lock); 607 608 idr_preload_end(); 609 610 if (ret < 0) { 611 drm_syncobj_put(syncobj); 612 return ret; 613 } 614 615 *handle = ret; 616 return 0; 617 } 618 EXPORT_SYMBOL(drm_syncobj_get_handle); 619 620 static int drm_syncobj_create_as_handle(struct drm_file *file_private, 621 u32 *handle, uint32_t flags) 622 { 623 int ret; 624 struct drm_syncobj *syncobj; 625 626 ret = drm_syncobj_create(&syncobj, flags, NULL); 627 if (ret) 628 return ret; 629 630 ret = drm_syncobj_get_handle(file_private, syncobj, handle); 631 drm_syncobj_put(syncobj); 632 return ret; 633 } 634 635 static int drm_syncobj_destroy(struct drm_file *file_private, 636 u32 handle) 637 { 638 struct drm_syncobj *syncobj; 639 640 spin_lock(&file_private->syncobj_table_lock); 641 syncobj = idr_remove(&file_private->syncobj_idr, handle); 642 spin_unlock(&file_private->syncobj_table_lock); 643 644 if (!syncobj) 645 return -EINVAL; 646 647 drm_syncobj_put(syncobj); 648 return 0; 649 } 650 651 static int drm_syncobj_file_release(struct inode *inode, struct file *file) 652 { 653 struct drm_syncobj *syncobj = file->private_data; 654 655 drm_syncobj_put(syncobj); 656 return 0; 657 } 658 659 static const struct file_operations drm_syncobj_file_fops = { 660 .release = drm_syncobj_file_release, 661 }; 662 663 /** 664 * drm_syncobj_get_fd - get a file descriptor from a syncobj 665 * @syncobj: Sync object to export 666 * @p_fd: out parameter with the new file descriptor 667 * 668 * Exports a sync object created with drm_syncobj_create() as a file descriptor. 669 * 670 * Returns 0 on success or a negative error value on failure. 671 */ 672 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd) 673 { 674 struct file *file; 675 int fd; 676 677 fd = get_unused_fd_flags(O_CLOEXEC); 678 if (fd < 0) 679 return fd; 680 681 file = anon_inode_getfile("syncobj_file", 682 &drm_syncobj_file_fops, 683 syncobj, 0); 684 if (IS_ERR(file)) { 685 put_unused_fd(fd); 686 return PTR_ERR(file); 687 } 688 689 drm_syncobj_get(syncobj); 690 fd_install(fd, file); 691 692 *p_fd = fd; 693 return 0; 694 } 695 EXPORT_SYMBOL(drm_syncobj_get_fd); 696 697 static int drm_syncobj_handle_to_fd(struct drm_file *file_private, 698 u32 handle, int *p_fd) 699 { 700 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle); 701 int ret; 702 703 if (!syncobj) 704 return -EINVAL; 705 706 ret = drm_syncobj_get_fd(syncobj, p_fd); 707 drm_syncobj_put(syncobj); 708 return ret; 709 } 710 711 static int drm_syncobj_fd_to_handle(struct drm_file *file_private, 712 int fd, u32 *handle) 713 { 714 struct drm_syncobj *syncobj; 715 CLASS(fd, f)(fd); 716 int ret; 717 718 if (fd_empty(f)) 719 return -EINVAL; 720 721 if (fd_file(f)->f_op != &drm_syncobj_file_fops) 722 return -EINVAL; 723 724 /* take a reference to put in the idr */ 725 syncobj = fd_file(f)->private_data; 726 drm_syncobj_get(syncobj); 727 728 idr_preload(GFP_KERNEL); 729 spin_lock(&file_private->syncobj_table_lock); 730 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT); 731 spin_unlock(&file_private->syncobj_table_lock); 732 idr_preload_end(); 733 734 if (ret > 0) { 735 *handle = ret; 736 ret = 0; 737 } else 738 drm_syncobj_put(syncobj); 739 740 return ret; 741 } 742 743 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private, 744 int fd, int handle, u64 point) 745 { 746 struct dma_fence *fence = sync_file_get_fence(fd); 747 struct drm_syncobj *syncobj; 748 749 if (!fence) 750 return -EINVAL; 751 752 syncobj = drm_syncobj_find(file_private, handle); 753 if (!syncobj) { 754 dma_fence_put(fence); 755 return -ENOENT; 756 } 757 758 if (point) { 759 struct dma_fence_chain *chain = dma_fence_chain_alloc(); 760 761 if (!chain) 762 return -ENOMEM; 763 764 drm_syncobj_add_point(syncobj, chain, fence, point); 765 } else { 766 drm_syncobj_replace_fence(syncobj, fence); 767 } 768 769 dma_fence_put(fence); 770 drm_syncobj_put(syncobj); 771 return 0; 772 } 773 774 static int drm_syncobj_export_sync_file(struct drm_file *file_private, 775 int handle, u64 point, int *p_fd) 776 { 777 int ret; 778 struct dma_fence *fence; 779 struct sync_file *sync_file; 780 int fd = get_unused_fd_flags(O_CLOEXEC); 781 782 if (fd < 0) 783 return fd; 784 785 ret = drm_syncobj_find_fence(file_private, handle, point, 0, &fence); 786 if (ret) 787 goto err_put_fd; 788 789 sync_file = sync_file_create(fence); 790 791 dma_fence_put(fence); 792 793 if (!sync_file) { 794 ret = -EINVAL; 795 goto err_put_fd; 796 } 797 798 fd_install(fd, sync_file->file); 799 800 *p_fd = fd; 801 return 0; 802 err_put_fd: 803 put_unused_fd(fd); 804 return ret; 805 } 806 /** 807 * drm_syncobj_open - initializes syncobj file-private structures at devnode open time 808 * @file_private: drm file-private structure to set up 809 * 810 * Called at device open time, sets up the structure for handling refcounting 811 * of sync objects. 812 */ 813 void 814 drm_syncobj_open(struct drm_file *file_private) 815 { 816 idr_init_base(&file_private->syncobj_idr, 1); 817 spin_lock_init(&file_private->syncobj_table_lock); 818 } 819 820 static int 821 drm_syncobj_release_handle(int id, void *ptr, void *data) 822 { 823 struct drm_syncobj *syncobj = ptr; 824 825 drm_syncobj_put(syncobj); 826 return 0; 827 } 828 829 /** 830 * drm_syncobj_release - release file-private sync object resources 831 * @file_private: drm file-private structure to clean up 832 * 833 * Called at close time when the filp is going away. 834 * 835 * Releases any remaining references on objects by this filp. 836 */ 837 void 838 drm_syncobj_release(struct drm_file *file_private) 839 { 840 idr_for_each(&file_private->syncobj_idr, 841 &drm_syncobj_release_handle, file_private); 842 idr_destroy(&file_private->syncobj_idr); 843 } 844 845 int 846 drm_syncobj_create_ioctl(struct drm_device *dev, void *data, 847 struct drm_file *file_private) 848 { 849 struct drm_syncobj_create *args = data; 850 851 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 852 return -EOPNOTSUPP; 853 854 /* no valid flags yet */ 855 if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED) 856 return -EINVAL; 857 858 return drm_syncobj_create_as_handle(file_private, 859 &args->handle, args->flags); 860 } 861 862 int 863 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data, 864 struct drm_file *file_private) 865 { 866 struct drm_syncobj_destroy *args = data; 867 868 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 869 return -EOPNOTSUPP; 870 871 /* make sure padding is empty */ 872 if (args->pad) 873 return -EINVAL; 874 return drm_syncobj_destroy(file_private, args->handle); 875 } 876 877 int 878 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data, 879 struct drm_file *file_private) 880 { 881 struct drm_syncobj_handle *args = data; 882 unsigned int valid_flags = DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_TIMELINE | 883 DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE; 884 u64 point = 0; 885 886 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 887 return -EOPNOTSUPP; 888 889 if (args->pad) 890 return -EINVAL; 891 892 if (args->flags & ~valid_flags) 893 return -EINVAL; 894 895 if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_TIMELINE) 896 point = args->point; 897 898 if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE) 899 return drm_syncobj_export_sync_file(file_private, args->handle, 900 point, &args->fd); 901 902 if (args->point) 903 return -EINVAL; 904 905 return drm_syncobj_handle_to_fd(file_private, args->handle, 906 &args->fd); 907 } 908 909 int 910 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data, 911 struct drm_file *file_private) 912 { 913 struct drm_syncobj_handle *args = data; 914 unsigned int valid_flags = DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_TIMELINE | 915 DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE; 916 u64 point = 0; 917 918 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 919 return -EOPNOTSUPP; 920 921 if (args->pad) 922 return -EINVAL; 923 924 if (args->flags & ~valid_flags) 925 return -EINVAL; 926 927 if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_TIMELINE) 928 point = args->point; 929 930 if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE) 931 return drm_syncobj_import_sync_file_fence(file_private, 932 args->fd, 933 args->handle, 934 point); 935 936 if (args->point) 937 return -EINVAL; 938 939 return drm_syncobj_fd_to_handle(file_private, args->fd, 940 &args->handle); 941 } 942 943 static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private, 944 struct drm_syncobj_transfer *args) 945 { 946 struct drm_syncobj *timeline_syncobj = NULL; 947 struct dma_fence *fence, *tmp; 948 struct dma_fence_chain *chain; 949 int ret; 950 951 timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle); 952 if (!timeline_syncobj) { 953 return -ENOENT; 954 } 955 ret = drm_syncobj_find_fence(file_private, args->src_handle, 956 args->src_point, args->flags, 957 &tmp); 958 if (ret) 959 goto err_put_timeline; 960 961 fence = dma_fence_unwrap_merge(tmp); 962 dma_fence_put(tmp); 963 if (!fence) { 964 ret = -ENOMEM; 965 goto err_put_timeline; 966 } 967 968 chain = dma_fence_chain_alloc(); 969 if (!chain) { 970 ret = -ENOMEM; 971 goto err_free_fence; 972 } 973 974 drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point); 975 err_free_fence: 976 dma_fence_put(fence); 977 err_put_timeline: 978 drm_syncobj_put(timeline_syncobj); 979 980 return ret; 981 } 982 983 static int 984 drm_syncobj_transfer_to_binary(struct drm_file *file_private, 985 struct drm_syncobj_transfer *args) 986 { 987 struct drm_syncobj *binary_syncobj = NULL; 988 struct dma_fence *fence; 989 int ret; 990 991 binary_syncobj = drm_syncobj_find(file_private, args->dst_handle); 992 if (!binary_syncobj) 993 return -ENOENT; 994 ret = drm_syncobj_find_fence(file_private, args->src_handle, 995 args->src_point, args->flags, &fence); 996 if (ret) 997 goto err; 998 drm_syncobj_replace_fence(binary_syncobj, fence); 999 dma_fence_put(fence); 1000 err: 1001 drm_syncobj_put(binary_syncobj); 1002 1003 return ret; 1004 } 1005 int 1006 drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data, 1007 struct drm_file *file_private) 1008 { 1009 struct drm_syncobj_transfer *args = data; 1010 int ret; 1011 1012 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1013 return -EOPNOTSUPP; 1014 1015 if (args->pad) 1016 return -EINVAL; 1017 1018 if (args->dst_point) 1019 ret = drm_syncobj_transfer_to_timeline(file_private, args); 1020 else 1021 ret = drm_syncobj_transfer_to_binary(file_private, args); 1022 1023 return ret; 1024 } 1025 1026 static void syncobj_wait_fence_func(struct dma_fence *fence, 1027 struct dma_fence_cb *cb) 1028 { 1029 struct syncobj_wait_entry *wait = 1030 container_of(cb, struct syncobj_wait_entry, fence_cb); 1031 1032 wake_up_process(wait->task); 1033 } 1034 1035 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj, 1036 struct syncobj_wait_entry *wait) 1037 { 1038 struct dma_fence *fence; 1039 1040 /* This happens inside the syncobj lock */ 1041 fence = rcu_dereference_protected(syncobj->fence, 1042 lockdep_is_held(&syncobj->lock)); 1043 dma_fence_get(fence); 1044 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) { 1045 dma_fence_put(fence); 1046 return; 1047 } else if (!fence) { 1048 wait->fence = dma_fence_get_stub(); 1049 } else { 1050 wait->fence = fence; 1051 } 1052 1053 wake_up_process(wait->task); 1054 list_del_init(&wait->node); 1055 } 1056 1057 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs, 1058 void __user *user_points, 1059 uint32_t count, 1060 uint32_t flags, 1061 signed long timeout, 1062 uint32_t *idx, 1063 ktime_t *deadline) 1064 { 1065 struct syncobj_wait_entry *entries; 1066 struct dma_fence *fence; 1067 uint64_t *points; 1068 uint32_t signaled_count, i; 1069 1070 if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1071 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) { 1072 might_sleep(); 1073 lockdep_assert_none_held_once(); 1074 } 1075 1076 points = kmalloc_array(count, sizeof(*points), GFP_KERNEL); 1077 if (points == NULL) 1078 return -ENOMEM; 1079 1080 if (!user_points) { 1081 memset(points, 0, count * sizeof(uint64_t)); 1082 1083 } else if (copy_from_user(points, user_points, 1084 sizeof(uint64_t) * count)) { 1085 timeout = -EFAULT; 1086 goto err_free_points; 1087 } 1088 1089 entries = kcalloc(count, sizeof(*entries), GFP_KERNEL); 1090 if (!entries) { 1091 timeout = -ENOMEM; 1092 goto err_free_points; 1093 } 1094 /* Walk the list of sync objects and initialize entries. We do 1095 * this up-front so that we can properly return -EINVAL if there is 1096 * a syncobj with a missing fence and then never have the chance of 1097 * returning -EINVAL again. 1098 */ 1099 signaled_count = 0; 1100 for (i = 0; i < count; ++i) { 1101 struct dma_fence *fence; 1102 1103 entries[i].task = current; 1104 entries[i].point = points[i]; 1105 fence = drm_syncobj_fence_get(syncobjs[i]); 1106 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) { 1107 dma_fence_put(fence); 1108 if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1109 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) { 1110 continue; 1111 } else { 1112 timeout = -EINVAL; 1113 goto cleanup_entries; 1114 } 1115 } 1116 1117 if (fence) 1118 entries[i].fence = fence; 1119 else 1120 entries[i].fence = dma_fence_get_stub(); 1121 1122 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) || 1123 dma_fence_is_signaled(entries[i].fence)) { 1124 if (signaled_count == 0 && idx) 1125 *idx = i; 1126 signaled_count++; 1127 } 1128 } 1129 1130 if (signaled_count == count || 1131 (signaled_count > 0 && 1132 !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL))) 1133 goto cleanup_entries; 1134 1135 /* There's a very annoying laxness in the dma_fence API here, in 1136 * that backends are not required to automatically report when a 1137 * fence is signaled prior to fence->ops->enable_signaling() being 1138 * called. So here if we fail to match signaled_count, we need to 1139 * fallthough and try a 0 timeout wait! 1140 */ 1141 1142 if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1143 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) { 1144 for (i = 0; i < count; ++i) 1145 drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]); 1146 } 1147 1148 if (deadline) { 1149 for (i = 0; i < count; ++i) { 1150 fence = entries[i].fence; 1151 if (!fence) 1152 continue; 1153 dma_fence_set_deadline(fence, *deadline); 1154 } 1155 } 1156 1157 do { 1158 set_current_state(TASK_INTERRUPTIBLE); 1159 1160 signaled_count = 0; 1161 for (i = 0; i < count; ++i) { 1162 fence = entries[i].fence; 1163 if (!fence) 1164 continue; 1165 1166 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) || 1167 dma_fence_is_signaled(fence) || 1168 (!entries[i].fence_cb.func && 1169 dma_fence_add_callback(fence, 1170 &entries[i].fence_cb, 1171 syncobj_wait_fence_func))) { 1172 /* The fence has been signaled */ 1173 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) { 1174 signaled_count++; 1175 } else { 1176 if (idx) 1177 *idx = i; 1178 goto done_waiting; 1179 } 1180 } 1181 } 1182 1183 if (signaled_count == count) 1184 goto done_waiting; 1185 1186 if (timeout == 0) { 1187 timeout = -ETIME; 1188 goto done_waiting; 1189 } 1190 1191 if (signal_pending(current)) { 1192 timeout = -ERESTARTSYS; 1193 goto done_waiting; 1194 } 1195 1196 timeout = schedule_timeout(timeout); 1197 } while (1); 1198 1199 done_waiting: 1200 __set_current_state(TASK_RUNNING); 1201 1202 cleanup_entries: 1203 for (i = 0; i < count; ++i) { 1204 drm_syncobj_remove_wait(syncobjs[i], &entries[i]); 1205 if (entries[i].fence_cb.func) 1206 dma_fence_remove_callback(entries[i].fence, 1207 &entries[i].fence_cb); 1208 dma_fence_put(entries[i].fence); 1209 } 1210 kfree(entries); 1211 1212 err_free_points: 1213 kfree(points); 1214 1215 return timeout; 1216 } 1217 1218 /** 1219 * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value 1220 * 1221 * @timeout_nsec: timeout nsec component in ns, 0 for poll 1222 * 1223 * Calculate the timeout in jiffies from an absolute time in sec/nsec. 1224 */ 1225 signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec) 1226 { 1227 ktime_t abs_timeout, now; 1228 u64 timeout_ns, timeout_jiffies64; 1229 1230 /* make 0 timeout means poll - absolute 0 doesn't seem valid */ 1231 if (timeout_nsec == 0) 1232 return 0; 1233 1234 abs_timeout = ns_to_ktime(timeout_nsec); 1235 now = ktime_get(); 1236 1237 if (!ktime_after(abs_timeout, now)) 1238 return 0; 1239 1240 timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now)); 1241 1242 timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns); 1243 /* clamp timeout to avoid infinite timeout */ 1244 if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1) 1245 return MAX_SCHEDULE_TIMEOUT - 1; 1246 1247 return timeout_jiffies64 + 1; 1248 } 1249 EXPORT_SYMBOL(drm_timeout_abs_to_jiffies); 1250 1251 static int drm_syncobj_array_wait(struct drm_device *dev, 1252 struct drm_file *file_private, 1253 struct drm_syncobj_wait *wait, 1254 struct drm_syncobj_timeline_wait *timeline_wait, 1255 struct drm_syncobj **syncobjs, bool timeline, 1256 ktime_t *deadline) 1257 { 1258 signed long timeout = 0; 1259 uint32_t first = ~0; 1260 1261 if (!timeline) { 1262 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec); 1263 timeout = drm_syncobj_array_wait_timeout(syncobjs, 1264 NULL, 1265 wait->count_handles, 1266 wait->flags, 1267 timeout, &first, 1268 deadline); 1269 if (timeout < 0) 1270 return timeout; 1271 wait->first_signaled = first; 1272 } else { 1273 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec); 1274 timeout = drm_syncobj_array_wait_timeout(syncobjs, 1275 u64_to_user_ptr(timeline_wait->points), 1276 timeline_wait->count_handles, 1277 timeline_wait->flags, 1278 timeout, &first, 1279 deadline); 1280 if (timeout < 0) 1281 return timeout; 1282 timeline_wait->first_signaled = first; 1283 } 1284 return 0; 1285 } 1286 1287 static int drm_syncobj_array_find(struct drm_file *file_private, 1288 void __user *user_handles, 1289 uint32_t count_handles, 1290 struct drm_syncobj ***syncobjs_out) 1291 { 1292 uint32_t i, *handles; 1293 struct drm_syncobj **syncobjs; 1294 int ret; 1295 1296 handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL); 1297 if (handles == NULL) 1298 return -ENOMEM; 1299 1300 if (copy_from_user(handles, user_handles, 1301 sizeof(uint32_t) * count_handles)) { 1302 ret = -EFAULT; 1303 goto err_free_handles; 1304 } 1305 1306 syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL); 1307 if (syncobjs == NULL) { 1308 ret = -ENOMEM; 1309 goto err_free_handles; 1310 } 1311 1312 for (i = 0; i < count_handles; i++) { 1313 syncobjs[i] = drm_syncobj_find(file_private, handles[i]); 1314 if (!syncobjs[i]) { 1315 ret = -ENOENT; 1316 goto err_put_syncobjs; 1317 } 1318 } 1319 1320 kfree(handles); 1321 *syncobjs_out = syncobjs; 1322 return 0; 1323 1324 err_put_syncobjs: 1325 while (i-- > 0) 1326 drm_syncobj_put(syncobjs[i]); 1327 kfree(syncobjs); 1328 err_free_handles: 1329 kfree(handles); 1330 1331 return ret; 1332 } 1333 1334 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs, 1335 uint32_t count) 1336 { 1337 uint32_t i; 1338 1339 for (i = 0; i < count; i++) 1340 drm_syncobj_put(syncobjs[i]); 1341 kfree(syncobjs); 1342 } 1343 1344 int 1345 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data, 1346 struct drm_file *file_private) 1347 { 1348 struct drm_syncobj_wait *args = data; 1349 struct drm_syncobj **syncobjs; 1350 unsigned int possible_flags; 1351 ktime_t t, *tp = NULL; 1352 int ret = 0; 1353 1354 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1355 return -EOPNOTSUPP; 1356 1357 possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL | 1358 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1359 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE; 1360 1361 if (args->flags & ~possible_flags) 1362 return -EINVAL; 1363 1364 if (args->count_handles == 0) 1365 return 0; 1366 1367 ret = drm_syncobj_array_find(file_private, 1368 u64_to_user_ptr(args->handles), 1369 args->count_handles, 1370 &syncobjs); 1371 if (ret < 0) 1372 return ret; 1373 1374 if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) { 1375 t = ns_to_ktime(args->deadline_nsec); 1376 tp = &t; 1377 } 1378 1379 ret = drm_syncobj_array_wait(dev, file_private, 1380 args, NULL, syncobjs, false, tp); 1381 1382 drm_syncobj_array_free(syncobjs, args->count_handles); 1383 1384 return ret; 1385 } 1386 1387 int 1388 drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data, 1389 struct drm_file *file_private) 1390 { 1391 struct drm_syncobj_timeline_wait *args = data; 1392 struct drm_syncobj **syncobjs; 1393 unsigned int possible_flags; 1394 ktime_t t, *tp = NULL; 1395 int ret = 0; 1396 1397 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1398 return -EOPNOTSUPP; 1399 1400 possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL | 1401 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1402 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE | 1403 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE; 1404 1405 if (args->flags & ~possible_flags) 1406 return -EINVAL; 1407 1408 if (args->count_handles == 0) 1409 return 0; 1410 1411 ret = drm_syncobj_array_find(file_private, 1412 u64_to_user_ptr(args->handles), 1413 args->count_handles, 1414 &syncobjs); 1415 if (ret < 0) 1416 return ret; 1417 1418 if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) { 1419 t = ns_to_ktime(args->deadline_nsec); 1420 tp = &t; 1421 } 1422 1423 ret = drm_syncobj_array_wait(dev, file_private, 1424 NULL, args, syncobjs, true, tp); 1425 1426 drm_syncobj_array_free(syncobjs, args->count_handles); 1427 1428 return ret; 1429 } 1430 1431 static void syncobj_eventfd_entry_fence_func(struct dma_fence *fence, 1432 struct dma_fence_cb *cb) 1433 { 1434 struct syncobj_eventfd_entry *entry = 1435 container_of(cb, struct syncobj_eventfd_entry, fence_cb); 1436 1437 eventfd_signal(entry->ev_fd_ctx); 1438 syncobj_eventfd_entry_free(entry); 1439 } 1440 1441 static void 1442 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj, 1443 struct syncobj_eventfd_entry *entry) 1444 { 1445 int ret; 1446 struct dma_fence *fence; 1447 1448 /* This happens inside the syncobj lock */ 1449 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1)); 1450 if (!fence) 1451 return; 1452 1453 ret = dma_fence_chain_find_seqno(&fence, entry->point); 1454 if (ret != 0) { 1455 /* The given seqno has not been submitted yet. */ 1456 dma_fence_put(fence); 1457 return; 1458 } else if (!fence) { 1459 /* If dma_fence_chain_find_seqno returns 0 but sets the fence 1460 * to NULL, it implies that the given seqno is signaled and a 1461 * later seqno has already been submitted. Assign a stub fence 1462 * so that the eventfd still gets signaled below. 1463 */ 1464 fence = dma_fence_get_stub(); 1465 } 1466 1467 list_del_init(&entry->node); 1468 entry->fence = fence; 1469 1470 if (entry->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) { 1471 eventfd_signal(entry->ev_fd_ctx); 1472 syncobj_eventfd_entry_free(entry); 1473 } else { 1474 ret = dma_fence_add_callback(fence, &entry->fence_cb, 1475 syncobj_eventfd_entry_fence_func); 1476 if (ret == -ENOENT) { 1477 eventfd_signal(entry->ev_fd_ctx); 1478 syncobj_eventfd_entry_free(entry); 1479 } 1480 } 1481 } 1482 1483 int 1484 drm_syncobj_eventfd_ioctl(struct drm_device *dev, void *data, 1485 struct drm_file *file_private) 1486 { 1487 struct drm_syncobj_eventfd *args = data; 1488 struct drm_syncobj *syncobj; 1489 struct eventfd_ctx *ev_fd_ctx; 1490 struct syncobj_eventfd_entry *entry; 1491 int ret; 1492 1493 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1494 return -EOPNOTSUPP; 1495 1496 if (args->flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) 1497 return -EINVAL; 1498 1499 if (args->pad) 1500 return -EINVAL; 1501 1502 syncobj = drm_syncobj_find(file_private, args->handle); 1503 if (!syncobj) 1504 return -ENOENT; 1505 1506 ev_fd_ctx = eventfd_ctx_fdget(args->fd); 1507 if (IS_ERR(ev_fd_ctx)) { 1508 ret = PTR_ERR(ev_fd_ctx); 1509 goto err_fdget; 1510 } 1511 1512 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 1513 if (!entry) { 1514 ret = -ENOMEM; 1515 goto err_kzalloc; 1516 } 1517 entry->syncobj = syncobj; 1518 entry->ev_fd_ctx = ev_fd_ctx; 1519 entry->point = args->point; 1520 entry->flags = args->flags; 1521 1522 drm_syncobj_add_eventfd(syncobj, entry); 1523 drm_syncobj_put(syncobj); 1524 1525 return 0; 1526 1527 err_kzalloc: 1528 eventfd_ctx_put(ev_fd_ctx); 1529 err_fdget: 1530 drm_syncobj_put(syncobj); 1531 return ret; 1532 } 1533 1534 int 1535 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data, 1536 struct drm_file *file_private) 1537 { 1538 struct drm_syncobj_array *args = data; 1539 struct drm_syncobj **syncobjs; 1540 uint32_t i; 1541 int ret; 1542 1543 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1544 return -EOPNOTSUPP; 1545 1546 if (args->pad != 0) 1547 return -EINVAL; 1548 1549 if (args->count_handles == 0) 1550 return -EINVAL; 1551 1552 ret = drm_syncobj_array_find(file_private, 1553 u64_to_user_ptr(args->handles), 1554 args->count_handles, 1555 &syncobjs); 1556 if (ret < 0) 1557 return ret; 1558 1559 for (i = 0; i < args->count_handles; i++) 1560 drm_syncobj_replace_fence(syncobjs[i], NULL); 1561 1562 drm_syncobj_array_free(syncobjs, args->count_handles); 1563 1564 return 0; 1565 } 1566 1567 int 1568 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data, 1569 struct drm_file *file_private) 1570 { 1571 struct drm_syncobj_array *args = data; 1572 struct drm_syncobj **syncobjs; 1573 uint32_t i; 1574 int ret; 1575 1576 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1577 return -EOPNOTSUPP; 1578 1579 if (args->pad != 0) 1580 return -EINVAL; 1581 1582 if (args->count_handles == 0) 1583 return -EINVAL; 1584 1585 ret = drm_syncobj_array_find(file_private, 1586 u64_to_user_ptr(args->handles), 1587 args->count_handles, 1588 &syncobjs); 1589 if (ret < 0) 1590 return ret; 1591 1592 for (i = 0; i < args->count_handles; i++) { 1593 ret = drm_syncobj_assign_null_handle(syncobjs[i]); 1594 if (ret < 0) 1595 break; 1596 } 1597 1598 drm_syncobj_array_free(syncobjs, args->count_handles); 1599 1600 return ret; 1601 } 1602 1603 int 1604 drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data, 1605 struct drm_file *file_private) 1606 { 1607 struct drm_syncobj_timeline_array *args = data; 1608 struct drm_syncobj **syncobjs; 1609 struct dma_fence_chain **chains; 1610 uint64_t *points; 1611 uint32_t i, j; 1612 int ret; 1613 1614 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1615 return -EOPNOTSUPP; 1616 1617 if (args->flags != 0) 1618 return -EINVAL; 1619 1620 if (args->count_handles == 0) 1621 return -EINVAL; 1622 1623 ret = drm_syncobj_array_find(file_private, 1624 u64_to_user_ptr(args->handles), 1625 args->count_handles, 1626 &syncobjs); 1627 if (ret < 0) 1628 return ret; 1629 1630 points = kmalloc_array(args->count_handles, sizeof(*points), 1631 GFP_KERNEL); 1632 if (!points) { 1633 ret = -ENOMEM; 1634 goto out; 1635 } 1636 if (!u64_to_user_ptr(args->points)) { 1637 memset(points, 0, args->count_handles * sizeof(uint64_t)); 1638 } else if (copy_from_user(points, u64_to_user_ptr(args->points), 1639 sizeof(uint64_t) * args->count_handles)) { 1640 ret = -EFAULT; 1641 goto err_points; 1642 } 1643 1644 chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL); 1645 if (!chains) { 1646 ret = -ENOMEM; 1647 goto err_points; 1648 } 1649 for (i = 0; i < args->count_handles; i++) { 1650 chains[i] = dma_fence_chain_alloc(); 1651 if (!chains[i]) { 1652 for (j = 0; j < i; j++) 1653 dma_fence_chain_free(chains[j]); 1654 ret = -ENOMEM; 1655 goto err_chains; 1656 } 1657 } 1658 1659 for (i = 0; i < args->count_handles; i++) { 1660 struct dma_fence *fence = dma_fence_get_stub(); 1661 1662 drm_syncobj_add_point(syncobjs[i], chains[i], 1663 fence, points[i]); 1664 dma_fence_put(fence); 1665 } 1666 err_chains: 1667 kfree(chains); 1668 err_points: 1669 kfree(points); 1670 out: 1671 drm_syncobj_array_free(syncobjs, args->count_handles); 1672 1673 return ret; 1674 } 1675 1676 int drm_syncobj_query_ioctl(struct drm_device *dev, void *data, 1677 struct drm_file *file_private) 1678 { 1679 struct drm_syncobj_timeline_array *args = data; 1680 struct drm_syncobj **syncobjs; 1681 uint64_t __user *points = u64_to_user_ptr(args->points); 1682 uint32_t i; 1683 int ret; 1684 1685 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1686 return -EOPNOTSUPP; 1687 1688 if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) 1689 return -EINVAL; 1690 1691 if (args->count_handles == 0) 1692 return -EINVAL; 1693 1694 ret = drm_syncobj_array_find(file_private, 1695 u64_to_user_ptr(args->handles), 1696 args->count_handles, 1697 &syncobjs); 1698 if (ret < 0) 1699 return ret; 1700 1701 for (i = 0; i < args->count_handles; i++) { 1702 struct dma_fence_chain *chain; 1703 struct dma_fence *fence; 1704 uint64_t point; 1705 1706 fence = drm_syncobj_fence_get(syncobjs[i]); 1707 chain = to_dma_fence_chain(fence); 1708 if (chain) { 1709 struct dma_fence *iter, *last_signaled = 1710 dma_fence_get(fence); 1711 1712 if (args->flags & 1713 DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) { 1714 point = fence->seqno; 1715 } else { 1716 dma_fence_chain_for_each(iter, fence) { 1717 if (iter->context != fence->context) { 1718 dma_fence_put(iter); 1719 /* It is most likely that timeline has 1720 * unorder points. */ 1721 break; 1722 } 1723 dma_fence_put(last_signaled); 1724 last_signaled = dma_fence_get(iter); 1725 } 1726 point = dma_fence_is_signaled(last_signaled) ? 1727 last_signaled->seqno : 1728 to_dma_fence_chain(last_signaled)->prev_seqno; 1729 } 1730 dma_fence_put(last_signaled); 1731 } else { 1732 point = 0; 1733 } 1734 dma_fence_put(fence); 1735 ret = copy_to_user(&points[i], &point, sizeof(uint64_t)); 1736 ret = ret ? -EFAULT : 0; 1737 if (ret) 1738 break; 1739 } 1740 drm_syncobj_array_free(syncobjs, args->count_handles); 1741 1742 return ret; 1743 } 1744