1 /* 2 * Copyright 2017 Red Hat 3 * Parts ported from amdgpu (fence wait code). 4 * Copyright 2016 Advanced Micro Devices, Inc. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 * IN THE SOFTWARE. 24 * 25 * Authors: 26 * 27 */ 28 29 /** 30 * DOC: Overview 31 * 32 * DRM synchronisation objects (syncobj, see struct &drm_syncobj) provide a 33 * container for a synchronization primitive which can be used by userspace 34 * to explicitly synchronize GPU commands, can be shared between userspace 35 * processes, and can be shared between different DRM drivers. 36 * Their primary use-case is to implement Vulkan fences and semaphores. 37 * The syncobj userspace API provides ioctls for several operations: 38 * 39 * - Creation and destruction of syncobjs 40 * - Import and export of syncobjs to/from a syncobj file descriptor 41 * - Import and export a syncobj's underlying fence to/from a sync file 42 * - Reset a syncobj (set its fence to NULL) 43 * - Signal a syncobj (set a trivially signaled fence) 44 * - Wait for a syncobj's fence to appear and be signaled 45 * 46 * The syncobj userspace API also provides operations to manipulate a syncobj 47 * in terms of a timeline of struct &dma_fence_chain rather than a single 48 * struct &dma_fence, through the following operations: 49 * 50 * - Signal a given point on the timeline 51 * - Wait for a given point to appear and/or be signaled 52 * - Import and export from/to a given point of a timeline 53 * 54 * At it's core, a syncobj is simply a wrapper around a pointer to a struct 55 * &dma_fence which may be NULL. 56 * When a syncobj is first created, its pointer is either NULL or a pointer 57 * to an already signaled fence depending on whether the 58 * &DRM_SYNCOBJ_CREATE_SIGNALED flag is passed to 59 * &DRM_IOCTL_SYNCOBJ_CREATE. 60 * 61 * If the syncobj is considered as a binary (its state is either signaled or 62 * unsignaled) primitive, when GPU work is enqueued in a DRM driver to signal 63 * the syncobj, the syncobj's fence is replaced with a fence which will be 64 * signaled by the completion of that work. 65 * If the syncobj is considered as a timeline primitive, when GPU work is 66 * enqueued in a DRM driver to signal the a given point of the syncobj, a new 67 * struct &dma_fence_chain pointing to the DRM driver's fence and also 68 * pointing to the previous fence that was in the syncobj. The new struct 69 * &dma_fence_chain fence replace the syncobj's fence and will be signaled by 70 * completion of the DRM driver's work and also any work associated with the 71 * fence previously in the syncobj. 72 * 73 * When GPU work which waits on a syncobj is enqueued in a DRM driver, at the 74 * time the work is enqueued, it waits on the syncobj's fence before 75 * submitting the work to hardware. That fence is either : 76 * 77 * - The syncobj's current fence if the syncobj is considered as a binary 78 * primitive. 79 * - The struct &dma_fence associated with a given point if the syncobj is 80 * considered as a timeline primitive. 81 * 82 * If the syncobj's fence is NULL or not present in the syncobj's timeline, 83 * the enqueue operation is expected to fail. 84 * 85 * With binary syncobj, all manipulation of the syncobjs's fence happens in 86 * terms of the current fence at the time the ioctl is called by userspace 87 * regardless of whether that operation is an immediate host-side operation 88 * (signal or reset) or or an operation which is enqueued in some driver 89 * queue. &DRM_IOCTL_SYNCOBJ_RESET and &DRM_IOCTL_SYNCOBJ_SIGNAL can be used 90 * to manipulate a syncobj from the host by resetting its pointer to NULL or 91 * setting its pointer to a fence which is already signaled. 92 * 93 * With a timeline syncobj, all manipulation of the synobj's fence happens in 94 * terms of a u64 value referring to point in the timeline. See 95 * dma_fence_chain_find_seqno() to see how a given point is found in the 96 * timeline. 97 * 98 * Note that applications should be careful to always use timeline set of 99 * ioctl() when dealing with syncobj considered as timeline. Using a binary 100 * set of ioctl() with a syncobj considered as timeline could result incorrect 101 * synchronization. The use of binary syncobj is supported through the 102 * timeline set of ioctl() by using a point value of 0, this will reproduce 103 * the behavior of the binary set of ioctl() (for example replace the 104 * syncobj's fence when signaling). 105 * 106 * 107 * Host-side wait on syncobjs 108 * -------------------------- 109 * 110 * &DRM_IOCTL_SYNCOBJ_WAIT takes an array of syncobj handles and does a 111 * host-side wait on all of the syncobj fences simultaneously. 112 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL is set, the wait ioctl will wait on 113 * all of the syncobj fences to be signaled before it returns. 114 * Otherwise, it returns once at least one syncobj fence has been signaled 115 * and the index of a signaled fence is written back to the client. 116 * 117 * Unlike the enqueued GPU work dependencies which fail if they see a NULL 118 * fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set, 119 * the host-side wait will first wait for the syncobj to receive a non-NULL 120 * fence and then wait on that fence. 121 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is not set and any one of the 122 * syncobjs in the array has a NULL fence, -EINVAL will be returned. 123 * Assuming the syncobj starts off with a NULL fence, this allows a client 124 * to do a host wait in one thread (or process) which waits on GPU work 125 * submitted in another thread (or process) without having to manually 126 * synchronize between the two. 127 * This requirement is inherited from the Vulkan fence API. 128 * 129 * Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj 130 * handles as well as an array of u64 points and does a host-side wait on all 131 * of syncobj fences at the given points simultaneously. 132 * 133 * &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT also adds the ability to wait for a given 134 * fence to materialize on the timeline without waiting for the fence to be 135 * signaled by using the &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE flag. This 136 * requirement is inherited from the wait-before-signal behavior required by 137 * the Vulkan timeline semaphore API. 138 * 139 * Alternatively, &DRM_IOCTL_SYNCOBJ_EVENTFD can be used to wait without 140 * blocking: an eventfd will be signaled when the syncobj is. This is useful to 141 * integrate the wait in an event loop. 142 * 143 * 144 * Import/export of syncobjs 145 * ------------------------- 146 * 147 * &DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE and &DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD 148 * provide two mechanisms for import/export of syncobjs. 149 * 150 * The first lets the client import or export an entire syncobj to a file 151 * descriptor. 152 * These fd's are opaque and have no other use case, except passing the 153 * syncobj between processes. 154 * All exported file descriptors and any syncobj handles created as a 155 * result of importing those file descriptors own a reference to the 156 * same underlying struct &drm_syncobj and the syncobj can be used 157 * persistently across all the processes with which it is shared. 158 * The syncobj is freed only once the last reference is dropped. 159 * Unlike dma-buf, importing a syncobj creates a new handle (with its own 160 * reference) for every import instead of de-duplicating. 161 * The primary use-case of this persistent import/export is for shared 162 * Vulkan fences and semaphores. 163 * 164 * The second import/export mechanism, which is indicated by 165 * &DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE or 166 * &DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE lets the client 167 * import/export the syncobj's current fence from/to a &sync_file. 168 * When a syncobj is exported to a sync file, that sync file wraps the 169 * sycnobj's fence at the time of export and any later signal or reset 170 * operations on the syncobj will not affect the exported sync file. 171 * When a sync file is imported into a syncobj, the syncobj's fence is set 172 * to the fence wrapped by that sync file. 173 * Because sync files are immutable, resetting or signaling the syncobj 174 * will not affect any sync files whose fences have been imported into the 175 * syncobj. 176 * 177 * 178 * Import/export of timeline points in timeline syncobjs 179 * ----------------------------------------------------- 180 * 181 * &DRM_IOCTL_SYNCOBJ_TRANSFER provides a mechanism to transfer a struct 182 * &dma_fence_chain of a syncobj at a given u64 point to another u64 point 183 * into another syncobj. 184 * 185 * Note that if you want to transfer a struct &dma_fence_chain from a given 186 * point on a timeline syncobj from/into a binary syncobj, you can use the 187 * point 0 to mean take/replace the fence in the syncobj. 188 */ 189 190 #include <linux/anon_inodes.h> 191 #include <linux/dma-fence-unwrap.h> 192 #include <linux/eventfd.h> 193 #include <linux/file.h> 194 #include <linux/fs.h> 195 #include <linux/sched/signal.h> 196 #include <linux/sync_file.h> 197 #include <linux/uaccess.h> 198 199 #include <drm/drm.h> 200 #include <drm/drm_drv.h> 201 #include <drm/drm_file.h> 202 #include <drm/drm_gem.h> 203 #include <drm/drm_print.h> 204 #include <drm/drm_syncobj.h> 205 #include <drm/drm_utils.h> 206 207 #include "drm_internal.h" 208 209 struct syncobj_wait_entry { 210 struct list_head node; 211 struct task_struct *task; 212 struct dma_fence *fence; 213 struct dma_fence_cb fence_cb; 214 u64 point; 215 }; 216 217 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj, 218 struct syncobj_wait_entry *wait); 219 220 struct syncobj_eventfd_entry { 221 struct list_head node; 222 struct dma_fence *fence; 223 struct dma_fence_cb fence_cb; 224 struct drm_syncobj *syncobj; 225 struct eventfd_ctx *ev_fd_ctx; 226 u64 point; 227 u32 flags; 228 }; 229 230 static void 231 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj, 232 struct syncobj_eventfd_entry *entry); 233 234 /** 235 * drm_syncobj_find - lookup and reference a sync object. 236 * @file_private: drm file private pointer 237 * @handle: sync object handle to lookup. 238 * 239 * Returns a reference to the syncobj pointed to by handle or NULL. The 240 * reference must be released by calling drm_syncobj_put(). 241 */ 242 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private, 243 u32 handle) 244 { 245 struct drm_syncobj *syncobj; 246 247 spin_lock(&file_private->syncobj_table_lock); 248 249 /* Check if we currently have a reference on the object */ 250 syncobj = idr_find(&file_private->syncobj_idr, handle); 251 if (syncobj) 252 drm_syncobj_get(syncobj); 253 254 spin_unlock(&file_private->syncobj_table_lock); 255 256 return syncobj; 257 } 258 EXPORT_SYMBOL(drm_syncobj_find); 259 260 static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj, 261 struct syncobj_wait_entry *wait) 262 { 263 struct dma_fence *fence; 264 265 if (wait->fence) 266 return; 267 268 spin_lock(&syncobj->lock); 269 /* We've already tried once to get a fence and failed. Now that we 270 * have the lock, try one more time just to be sure we don't add a 271 * callback when a fence has already been set. 272 */ 273 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1)); 274 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) { 275 dma_fence_put(fence); 276 list_add_tail(&wait->node, &syncobj->cb_list); 277 } else if (!fence) { 278 wait->fence = dma_fence_get_stub(); 279 } else { 280 wait->fence = fence; 281 } 282 spin_unlock(&syncobj->lock); 283 } 284 285 static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj, 286 struct syncobj_wait_entry *wait) 287 { 288 if (!wait->node.next) 289 return; 290 291 spin_lock(&syncobj->lock); 292 list_del_init(&wait->node); 293 spin_unlock(&syncobj->lock); 294 } 295 296 static void 297 syncobj_eventfd_entry_free(struct syncobj_eventfd_entry *entry) 298 { 299 eventfd_ctx_put(entry->ev_fd_ctx); 300 dma_fence_put(entry->fence); 301 /* This happens either inside the syncobj lock, or after the node has 302 * already been removed from the list. 303 */ 304 list_del(&entry->node); 305 kfree(entry); 306 } 307 308 static void 309 drm_syncobj_add_eventfd(struct drm_syncobj *syncobj, 310 struct syncobj_eventfd_entry *entry) 311 { 312 spin_lock(&syncobj->lock); 313 list_add_tail(&entry->node, &syncobj->ev_fd_list); 314 syncobj_eventfd_entry_func(syncobj, entry); 315 spin_unlock(&syncobj->lock); 316 } 317 318 /** 319 * drm_syncobj_add_point - add new timeline point to the syncobj 320 * @syncobj: sync object to add timeline point do 321 * @chain: chain node to use to add the point 322 * @fence: fence to encapsulate in the chain node 323 * @point: sequence number to use for the point 324 * 325 * Add the chain node as new timeline point to the syncobj. 326 */ 327 void drm_syncobj_add_point(struct drm_syncobj *syncobj, 328 struct dma_fence_chain *chain, 329 struct dma_fence *fence, 330 uint64_t point) 331 { 332 struct syncobj_wait_entry *wait_cur, *wait_tmp; 333 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 334 struct dma_fence *prev; 335 336 dma_fence_get(fence); 337 338 spin_lock(&syncobj->lock); 339 340 prev = drm_syncobj_fence_get(syncobj); 341 /* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */ 342 if (prev && prev->seqno >= point) 343 DRM_DEBUG("You are adding an unorder point to timeline!\n"); 344 dma_fence_chain_init(chain, prev, fence, point); 345 rcu_assign_pointer(syncobj->fence, &chain->base); 346 347 list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node) 348 syncobj_wait_syncobj_func(syncobj, wait_cur); 349 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 350 syncobj_eventfd_entry_func(syncobj, ev_fd_cur); 351 spin_unlock(&syncobj->lock); 352 353 /* Walk the chain once to trigger garbage collection */ 354 dma_fence_chain_for_each(fence, prev); 355 dma_fence_put(prev); 356 } 357 EXPORT_SYMBOL(drm_syncobj_add_point); 358 359 /** 360 * drm_syncobj_replace_fence - replace fence in a sync object. 361 * @syncobj: Sync object to replace fence in 362 * @fence: fence to install in sync file. 363 * 364 * This replaces the fence on a sync object. 365 */ 366 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj, 367 struct dma_fence *fence) 368 { 369 struct dma_fence *old_fence; 370 struct syncobj_wait_entry *wait_cur, *wait_tmp; 371 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 372 373 if (fence) 374 dma_fence_get(fence); 375 376 spin_lock(&syncobj->lock); 377 378 old_fence = rcu_dereference_protected(syncobj->fence, 379 lockdep_is_held(&syncobj->lock)); 380 rcu_assign_pointer(syncobj->fence, fence); 381 382 if (fence != old_fence) { 383 list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node) 384 syncobj_wait_syncobj_func(syncobj, wait_cur); 385 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 386 syncobj_eventfd_entry_func(syncobj, ev_fd_cur); 387 } 388 389 spin_unlock(&syncobj->lock); 390 391 dma_fence_put(old_fence); 392 } 393 EXPORT_SYMBOL(drm_syncobj_replace_fence); 394 395 /** 396 * drm_syncobj_assign_null_handle - assign a stub fence to the sync object 397 * @syncobj: sync object to assign the fence on 398 * 399 * Assign a already signaled stub fence to the sync object. 400 */ 401 static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj) 402 { 403 struct dma_fence *fence = dma_fence_allocate_private_stub(ktime_get()); 404 405 if (!fence) 406 return -ENOMEM; 407 408 drm_syncobj_replace_fence(syncobj, fence); 409 dma_fence_put(fence); 410 return 0; 411 } 412 413 /* 5s default for wait submission */ 414 #define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL 415 /** 416 * drm_syncobj_find_fence - lookup and reference the fence in a sync object 417 * @file_private: drm file private pointer 418 * @handle: sync object handle to lookup. 419 * @point: timeline point 420 * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not 421 * @fence: out parameter for the fence 422 * 423 * This is just a convenience function that combines drm_syncobj_find() and 424 * drm_syncobj_fence_get(). 425 * 426 * Returns 0 on success or a negative error value on failure. On success @fence 427 * contains a reference to the fence, which must be released by calling 428 * dma_fence_put(). 429 */ 430 int drm_syncobj_find_fence(struct drm_file *file_private, 431 u32 handle, u64 point, u64 flags, 432 struct dma_fence **fence) 433 { 434 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle); 435 struct syncobj_wait_entry wait; 436 u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT); 437 int ret; 438 439 if (!syncobj) 440 return -ENOENT; 441 442 /* Waiting for userspace with locks help is illegal cause that can 443 * trivial deadlock with page faults for example. Make lockdep complain 444 * about it early on. 445 */ 446 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) { 447 might_sleep(); 448 lockdep_assert_none_held_once(); 449 } 450 451 *fence = drm_syncobj_fence_get(syncobj); 452 453 if (*fence) { 454 ret = dma_fence_chain_find_seqno(fence, point); 455 if (!ret) { 456 /* If the requested seqno is already signaled 457 * drm_syncobj_find_fence may return a NULL 458 * fence. To make sure the recipient gets 459 * signalled, use a new fence instead. 460 */ 461 if (!*fence) 462 *fence = dma_fence_get_stub(); 463 464 goto out; 465 } 466 dma_fence_put(*fence); 467 } else { 468 ret = -EINVAL; 469 } 470 471 if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)) 472 goto out; 473 474 memset(&wait, 0, sizeof(wait)); 475 wait.task = current; 476 wait.point = point; 477 drm_syncobj_fence_add_wait(syncobj, &wait); 478 479 do { 480 set_current_state(TASK_INTERRUPTIBLE); 481 if (wait.fence) { 482 ret = 0; 483 break; 484 } 485 if (timeout == 0) { 486 ret = -ETIME; 487 break; 488 } 489 490 if (signal_pending(current)) { 491 ret = -ERESTARTSYS; 492 break; 493 } 494 495 timeout = schedule_timeout(timeout); 496 } while (1); 497 498 __set_current_state(TASK_RUNNING); 499 *fence = wait.fence; 500 501 if (wait.node.next) 502 drm_syncobj_remove_wait(syncobj, &wait); 503 504 out: 505 drm_syncobj_put(syncobj); 506 507 return ret; 508 } 509 EXPORT_SYMBOL(drm_syncobj_find_fence); 510 511 /** 512 * drm_syncobj_free - free a sync object. 513 * @kref: kref to free. 514 * 515 * Only to be called from kref_put in drm_syncobj_put. 516 */ 517 void drm_syncobj_free(struct kref *kref) 518 { 519 struct drm_syncobj *syncobj = container_of(kref, 520 struct drm_syncobj, 521 refcount); 522 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 523 524 drm_syncobj_replace_fence(syncobj, NULL); 525 526 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 527 syncobj_eventfd_entry_free(ev_fd_cur); 528 529 kfree(syncobj); 530 } 531 EXPORT_SYMBOL(drm_syncobj_free); 532 533 /** 534 * drm_syncobj_create - create a new syncobj 535 * @out_syncobj: returned syncobj 536 * @flags: DRM_SYNCOBJ_* flags 537 * @fence: if non-NULL, the syncobj will represent this fence 538 * 539 * This is the first function to create a sync object. After creating, drivers 540 * probably want to make it available to userspace, either through 541 * drm_syncobj_get_handle() or drm_syncobj_get_fd(). 542 * 543 * Returns 0 on success or a negative error value on failure. 544 */ 545 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags, 546 struct dma_fence *fence) 547 { 548 int ret; 549 struct drm_syncobj *syncobj; 550 551 syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL); 552 if (!syncobj) 553 return -ENOMEM; 554 555 kref_init(&syncobj->refcount); 556 INIT_LIST_HEAD(&syncobj->cb_list); 557 INIT_LIST_HEAD(&syncobj->ev_fd_list); 558 spin_lock_init(&syncobj->lock); 559 560 if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) { 561 ret = drm_syncobj_assign_null_handle(syncobj); 562 if (ret < 0) { 563 drm_syncobj_put(syncobj); 564 return ret; 565 } 566 } 567 568 if (fence) 569 drm_syncobj_replace_fence(syncobj, fence); 570 571 *out_syncobj = syncobj; 572 return 0; 573 } 574 EXPORT_SYMBOL(drm_syncobj_create); 575 576 /** 577 * drm_syncobj_get_handle - get a handle from a syncobj 578 * @file_private: drm file private pointer 579 * @syncobj: Sync object to export 580 * @handle: out parameter with the new handle 581 * 582 * Exports a sync object created with drm_syncobj_create() as a handle on 583 * @file_private to userspace. 584 * 585 * Returns 0 on success or a negative error value on failure. 586 */ 587 int drm_syncobj_get_handle(struct drm_file *file_private, 588 struct drm_syncobj *syncobj, u32 *handle) 589 { 590 int ret; 591 592 /* take a reference to put in the idr */ 593 drm_syncobj_get(syncobj); 594 595 idr_preload(GFP_KERNEL); 596 spin_lock(&file_private->syncobj_table_lock); 597 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT); 598 spin_unlock(&file_private->syncobj_table_lock); 599 600 idr_preload_end(); 601 602 if (ret < 0) { 603 drm_syncobj_put(syncobj); 604 return ret; 605 } 606 607 *handle = ret; 608 return 0; 609 } 610 EXPORT_SYMBOL(drm_syncobj_get_handle); 611 612 static int drm_syncobj_create_as_handle(struct drm_file *file_private, 613 u32 *handle, uint32_t flags) 614 { 615 int ret; 616 struct drm_syncobj *syncobj; 617 618 ret = drm_syncobj_create(&syncobj, flags, NULL); 619 if (ret) 620 return ret; 621 622 ret = drm_syncobj_get_handle(file_private, syncobj, handle); 623 drm_syncobj_put(syncobj); 624 return ret; 625 } 626 627 static int drm_syncobj_destroy(struct drm_file *file_private, 628 u32 handle) 629 { 630 struct drm_syncobj *syncobj; 631 632 spin_lock(&file_private->syncobj_table_lock); 633 syncobj = idr_remove(&file_private->syncobj_idr, handle); 634 spin_unlock(&file_private->syncobj_table_lock); 635 636 if (!syncobj) 637 return -EINVAL; 638 639 drm_syncobj_put(syncobj); 640 return 0; 641 } 642 643 static int drm_syncobj_file_release(struct inode *inode, struct file *file) 644 { 645 struct drm_syncobj *syncobj = file->private_data; 646 647 drm_syncobj_put(syncobj); 648 return 0; 649 } 650 651 static const struct file_operations drm_syncobj_file_fops = { 652 .release = drm_syncobj_file_release, 653 }; 654 655 /** 656 * drm_syncobj_get_fd - get a file descriptor from a syncobj 657 * @syncobj: Sync object to export 658 * @p_fd: out parameter with the new file descriptor 659 * 660 * Exports a sync object created with drm_syncobj_create() as a file descriptor. 661 * 662 * Returns 0 on success or a negative error value on failure. 663 */ 664 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd) 665 { 666 struct file *file; 667 int fd; 668 669 fd = get_unused_fd_flags(O_CLOEXEC); 670 if (fd < 0) 671 return fd; 672 673 file = anon_inode_getfile("syncobj_file", 674 &drm_syncobj_file_fops, 675 syncobj, 0); 676 if (IS_ERR(file)) { 677 put_unused_fd(fd); 678 return PTR_ERR(file); 679 } 680 681 drm_syncobj_get(syncobj); 682 fd_install(fd, file); 683 684 *p_fd = fd; 685 return 0; 686 } 687 EXPORT_SYMBOL(drm_syncobj_get_fd); 688 689 static int drm_syncobj_handle_to_fd(struct drm_file *file_private, 690 u32 handle, int *p_fd) 691 { 692 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle); 693 int ret; 694 695 if (!syncobj) 696 return -EINVAL; 697 698 ret = drm_syncobj_get_fd(syncobj, p_fd); 699 drm_syncobj_put(syncobj); 700 return ret; 701 } 702 703 static int drm_syncobj_fd_to_handle(struct drm_file *file_private, 704 int fd, u32 *handle) 705 { 706 struct drm_syncobj *syncobj; 707 struct fd f = fdget(fd); 708 int ret; 709 710 if (!f.file) 711 return -EINVAL; 712 713 if (f.file->f_op != &drm_syncobj_file_fops) { 714 fdput(f); 715 return -EINVAL; 716 } 717 718 /* take a reference to put in the idr */ 719 syncobj = f.file->private_data; 720 drm_syncobj_get(syncobj); 721 722 idr_preload(GFP_KERNEL); 723 spin_lock(&file_private->syncobj_table_lock); 724 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT); 725 spin_unlock(&file_private->syncobj_table_lock); 726 idr_preload_end(); 727 728 if (ret > 0) { 729 *handle = ret; 730 ret = 0; 731 } else 732 drm_syncobj_put(syncobj); 733 734 fdput(f); 735 return ret; 736 } 737 738 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private, 739 int fd, int handle) 740 { 741 struct dma_fence *fence = sync_file_get_fence(fd); 742 struct drm_syncobj *syncobj; 743 744 if (!fence) 745 return -EINVAL; 746 747 syncobj = drm_syncobj_find(file_private, handle); 748 if (!syncobj) { 749 dma_fence_put(fence); 750 return -ENOENT; 751 } 752 753 drm_syncobj_replace_fence(syncobj, fence); 754 dma_fence_put(fence); 755 drm_syncobj_put(syncobj); 756 return 0; 757 } 758 759 static int drm_syncobj_export_sync_file(struct drm_file *file_private, 760 int handle, int *p_fd) 761 { 762 int ret; 763 struct dma_fence *fence; 764 struct sync_file *sync_file; 765 int fd = get_unused_fd_flags(O_CLOEXEC); 766 767 if (fd < 0) 768 return fd; 769 770 ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence); 771 if (ret) 772 goto err_put_fd; 773 774 sync_file = sync_file_create(fence); 775 776 dma_fence_put(fence); 777 778 if (!sync_file) { 779 ret = -EINVAL; 780 goto err_put_fd; 781 } 782 783 fd_install(fd, sync_file->file); 784 785 *p_fd = fd; 786 return 0; 787 err_put_fd: 788 put_unused_fd(fd); 789 return ret; 790 } 791 /** 792 * drm_syncobj_open - initializes syncobj file-private structures at devnode open time 793 * @file_private: drm file-private structure to set up 794 * 795 * Called at device open time, sets up the structure for handling refcounting 796 * of sync objects. 797 */ 798 void 799 drm_syncobj_open(struct drm_file *file_private) 800 { 801 idr_init_base(&file_private->syncobj_idr, 1); 802 spin_lock_init(&file_private->syncobj_table_lock); 803 } 804 805 static int 806 drm_syncobj_release_handle(int id, void *ptr, void *data) 807 { 808 struct drm_syncobj *syncobj = ptr; 809 810 drm_syncobj_put(syncobj); 811 return 0; 812 } 813 814 /** 815 * drm_syncobj_release - release file-private sync object resources 816 * @file_private: drm file-private structure to clean up 817 * 818 * Called at close time when the filp is going away. 819 * 820 * Releases any remaining references on objects by this filp. 821 */ 822 void 823 drm_syncobj_release(struct drm_file *file_private) 824 { 825 idr_for_each(&file_private->syncobj_idr, 826 &drm_syncobj_release_handle, file_private); 827 idr_destroy(&file_private->syncobj_idr); 828 } 829 830 int 831 drm_syncobj_create_ioctl(struct drm_device *dev, void *data, 832 struct drm_file *file_private) 833 { 834 struct drm_syncobj_create *args = data; 835 836 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 837 return -EOPNOTSUPP; 838 839 /* no valid flags yet */ 840 if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED) 841 return -EINVAL; 842 843 return drm_syncobj_create_as_handle(file_private, 844 &args->handle, args->flags); 845 } 846 847 int 848 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data, 849 struct drm_file *file_private) 850 { 851 struct drm_syncobj_destroy *args = data; 852 853 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 854 return -EOPNOTSUPP; 855 856 /* make sure padding is empty */ 857 if (args->pad) 858 return -EINVAL; 859 return drm_syncobj_destroy(file_private, args->handle); 860 } 861 862 int 863 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data, 864 struct drm_file *file_private) 865 { 866 struct drm_syncobj_handle *args = data; 867 868 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 869 return -EOPNOTSUPP; 870 871 if (args->pad) 872 return -EINVAL; 873 874 if (args->flags != 0 && 875 args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE) 876 return -EINVAL; 877 878 if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE) 879 return drm_syncobj_export_sync_file(file_private, args->handle, 880 &args->fd); 881 882 return drm_syncobj_handle_to_fd(file_private, args->handle, 883 &args->fd); 884 } 885 886 int 887 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data, 888 struct drm_file *file_private) 889 { 890 struct drm_syncobj_handle *args = data; 891 892 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 893 return -EOPNOTSUPP; 894 895 if (args->pad) 896 return -EINVAL; 897 898 if (args->flags != 0 && 899 args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE) 900 return -EINVAL; 901 902 if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE) 903 return drm_syncobj_import_sync_file_fence(file_private, 904 args->fd, 905 args->handle); 906 907 return drm_syncobj_fd_to_handle(file_private, args->fd, 908 &args->handle); 909 } 910 911 static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private, 912 struct drm_syncobj_transfer *args) 913 { 914 struct drm_syncobj *timeline_syncobj = NULL; 915 struct dma_fence *fence, *tmp; 916 struct dma_fence_chain *chain; 917 int ret; 918 919 timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle); 920 if (!timeline_syncobj) { 921 return -ENOENT; 922 } 923 ret = drm_syncobj_find_fence(file_private, args->src_handle, 924 args->src_point, args->flags, 925 &tmp); 926 if (ret) 927 goto err_put_timeline; 928 929 fence = dma_fence_unwrap_merge(tmp); 930 dma_fence_put(tmp); 931 if (!fence) { 932 ret = -ENOMEM; 933 goto err_put_timeline; 934 } 935 936 chain = dma_fence_chain_alloc(); 937 if (!chain) { 938 ret = -ENOMEM; 939 goto err_free_fence; 940 } 941 942 drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point); 943 err_free_fence: 944 dma_fence_put(fence); 945 err_put_timeline: 946 drm_syncobj_put(timeline_syncobj); 947 948 return ret; 949 } 950 951 static int 952 drm_syncobj_transfer_to_binary(struct drm_file *file_private, 953 struct drm_syncobj_transfer *args) 954 { 955 struct drm_syncobj *binary_syncobj = NULL; 956 struct dma_fence *fence; 957 int ret; 958 959 binary_syncobj = drm_syncobj_find(file_private, args->dst_handle); 960 if (!binary_syncobj) 961 return -ENOENT; 962 ret = drm_syncobj_find_fence(file_private, args->src_handle, 963 args->src_point, args->flags, &fence); 964 if (ret) 965 goto err; 966 drm_syncobj_replace_fence(binary_syncobj, fence); 967 dma_fence_put(fence); 968 err: 969 drm_syncobj_put(binary_syncobj); 970 971 return ret; 972 } 973 int 974 drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data, 975 struct drm_file *file_private) 976 { 977 struct drm_syncobj_transfer *args = data; 978 int ret; 979 980 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 981 return -EOPNOTSUPP; 982 983 if (args->pad) 984 return -EINVAL; 985 986 if (args->dst_point) 987 ret = drm_syncobj_transfer_to_timeline(file_private, args); 988 else 989 ret = drm_syncobj_transfer_to_binary(file_private, args); 990 991 return ret; 992 } 993 994 static void syncobj_wait_fence_func(struct dma_fence *fence, 995 struct dma_fence_cb *cb) 996 { 997 struct syncobj_wait_entry *wait = 998 container_of(cb, struct syncobj_wait_entry, fence_cb); 999 1000 wake_up_process(wait->task); 1001 } 1002 1003 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj, 1004 struct syncobj_wait_entry *wait) 1005 { 1006 struct dma_fence *fence; 1007 1008 /* This happens inside the syncobj lock */ 1009 fence = rcu_dereference_protected(syncobj->fence, 1010 lockdep_is_held(&syncobj->lock)); 1011 dma_fence_get(fence); 1012 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) { 1013 dma_fence_put(fence); 1014 return; 1015 } else if (!fence) { 1016 wait->fence = dma_fence_get_stub(); 1017 } else { 1018 wait->fence = fence; 1019 } 1020 1021 wake_up_process(wait->task); 1022 list_del_init(&wait->node); 1023 } 1024 1025 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs, 1026 void __user *user_points, 1027 uint32_t count, 1028 uint32_t flags, 1029 signed long timeout, 1030 uint32_t *idx) 1031 { 1032 struct syncobj_wait_entry *entries; 1033 struct dma_fence *fence; 1034 uint64_t *points; 1035 uint32_t signaled_count, i; 1036 1037 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) 1038 lockdep_assert_none_held_once(); 1039 1040 points = kmalloc_array(count, sizeof(*points), GFP_KERNEL); 1041 if (points == NULL) 1042 return -ENOMEM; 1043 1044 if (!user_points) { 1045 memset(points, 0, count * sizeof(uint64_t)); 1046 1047 } else if (copy_from_user(points, user_points, 1048 sizeof(uint64_t) * count)) { 1049 timeout = -EFAULT; 1050 goto err_free_points; 1051 } 1052 1053 entries = kcalloc(count, sizeof(*entries), GFP_KERNEL); 1054 if (!entries) { 1055 timeout = -ENOMEM; 1056 goto err_free_points; 1057 } 1058 /* Walk the list of sync objects and initialize entries. We do 1059 * this up-front so that we can properly return -EINVAL if there is 1060 * a syncobj with a missing fence and then never have the chance of 1061 * returning -EINVAL again. 1062 */ 1063 signaled_count = 0; 1064 for (i = 0; i < count; ++i) { 1065 struct dma_fence *fence; 1066 1067 entries[i].task = current; 1068 entries[i].point = points[i]; 1069 fence = drm_syncobj_fence_get(syncobjs[i]); 1070 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) { 1071 dma_fence_put(fence); 1072 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) { 1073 continue; 1074 } else { 1075 timeout = -EINVAL; 1076 goto cleanup_entries; 1077 } 1078 } 1079 1080 if (fence) 1081 entries[i].fence = fence; 1082 else 1083 entries[i].fence = dma_fence_get_stub(); 1084 1085 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) || 1086 dma_fence_is_signaled(entries[i].fence)) { 1087 if (signaled_count == 0 && idx) 1088 *idx = i; 1089 signaled_count++; 1090 } 1091 } 1092 1093 if (signaled_count == count || 1094 (signaled_count > 0 && 1095 !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL))) 1096 goto cleanup_entries; 1097 1098 /* There's a very annoying laxness in the dma_fence API here, in 1099 * that backends are not required to automatically report when a 1100 * fence is signaled prior to fence->ops->enable_signaling() being 1101 * called. So here if we fail to match signaled_count, we need to 1102 * fallthough and try a 0 timeout wait! 1103 */ 1104 1105 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) { 1106 for (i = 0; i < count; ++i) 1107 drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]); 1108 } 1109 1110 do { 1111 set_current_state(TASK_INTERRUPTIBLE); 1112 1113 signaled_count = 0; 1114 for (i = 0; i < count; ++i) { 1115 fence = entries[i].fence; 1116 if (!fence) 1117 continue; 1118 1119 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) || 1120 dma_fence_is_signaled(fence) || 1121 (!entries[i].fence_cb.func && 1122 dma_fence_add_callback(fence, 1123 &entries[i].fence_cb, 1124 syncobj_wait_fence_func))) { 1125 /* The fence has been signaled */ 1126 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) { 1127 signaled_count++; 1128 } else { 1129 if (idx) 1130 *idx = i; 1131 goto done_waiting; 1132 } 1133 } 1134 } 1135 1136 if (signaled_count == count) 1137 goto done_waiting; 1138 1139 if (timeout == 0) { 1140 timeout = -ETIME; 1141 goto done_waiting; 1142 } 1143 1144 if (signal_pending(current)) { 1145 timeout = -ERESTARTSYS; 1146 goto done_waiting; 1147 } 1148 1149 timeout = schedule_timeout(timeout); 1150 } while (1); 1151 1152 done_waiting: 1153 __set_current_state(TASK_RUNNING); 1154 1155 cleanup_entries: 1156 for (i = 0; i < count; ++i) { 1157 drm_syncobj_remove_wait(syncobjs[i], &entries[i]); 1158 if (entries[i].fence_cb.func) 1159 dma_fence_remove_callback(entries[i].fence, 1160 &entries[i].fence_cb); 1161 dma_fence_put(entries[i].fence); 1162 } 1163 kfree(entries); 1164 1165 err_free_points: 1166 kfree(points); 1167 1168 return timeout; 1169 } 1170 1171 /** 1172 * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value 1173 * 1174 * @timeout_nsec: timeout nsec component in ns, 0 for poll 1175 * 1176 * Calculate the timeout in jiffies from an absolute time in sec/nsec. 1177 */ 1178 signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec) 1179 { 1180 ktime_t abs_timeout, now; 1181 u64 timeout_ns, timeout_jiffies64; 1182 1183 /* make 0 timeout means poll - absolute 0 doesn't seem valid */ 1184 if (timeout_nsec == 0) 1185 return 0; 1186 1187 abs_timeout = ns_to_ktime(timeout_nsec); 1188 now = ktime_get(); 1189 1190 if (!ktime_after(abs_timeout, now)) 1191 return 0; 1192 1193 timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now)); 1194 1195 timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns); 1196 /* clamp timeout to avoid infinite timeout */ 1197 if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1) 1198 return MAX_SCHEDULE_TIMEOUT - 1; 1199 1200 return timeout_jiffies64 + 1; 1201 } 1202 EXPORT_SYMBOL(drm_timeout_abs_to_jiffies); 1203 1204 static int drm_syncobj_array_wait(struct drm_device *dev, 1205 struct drm_file *file_private, 1206 struct drm_syncobj_wait *wait, 1207 struct drm_syncobj_timeline_wait *timeline_wait, 1208 struct drm_syncobj **syncobjs, bool timeline) 1209 { 1210 signed long timeout = 0; 1211 uint32_t first = ~0; 1212 1213 if (!timeline) { 1214 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec); 1215 timeout = drm_syncobj_array_wait_timeout(syncobjs, 1216 NULL, 1217 wait->count_handles, 1218 wait->flags, 1219 timeout, &first); 1220 if (timeout < 0) 1221 return timeout; 1222 wait->first_signaled = first; 1223 } else { 1224 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec); 1225 timeout = drm_syncobj_array_wait_timeout(syncobjs, 1226 u64_to_user_ptr(timeline_wait->points), 1227 timeline_wait->count_handles, 1228 timeline_wait->flags, 1229 timeout, &first); 1230 if (timeout < 0) 1231 return timeout; 1232 timeline_wait->first_signaled = first; 1233 } 1234 return 0; 1235 } 1236 1237 static int drm_syncobj_array_find(struct drm_file *file_private, 1238 void __user *user_handles, 1239 uint32_t count_handles, 1240 struct drm_syncobj ***syncobjs_out) 1241 { 1242 uint32_t i, *handles; 1243 struct drm_syncobj **syncobjs; 1244 int ret; 1245 1246 handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL); 1247 if (handles == NULL) 1248 return -ENOMEM; 1249 1250 if (copy_from_user(handles, user_handles, 1251 sizeof(uint32_t) * count_handles)) { 1252 ret = -EFAULT; 1253 goto err_free_handles; 1254 } 1255 1256 syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL); 1257 if (syncobjs == NULL) { 1258 ret = -ENOMEM; 1259 goto err_free_handles; 1260 } 1261 1262 for (i = 0; i < count_handles; i++) { 1263 syncobjs[i] = drm_syncobj_find(file_private, handles[i]); 1264 if (!syncobjs[i]) { 1265 ret = -ENOENT; 1266 goto err_put_syncobjs; 1267 } 1268 } 1269 1270 kfree(handles); 1271 *syncobjs_out = syncobjs; 1272 return 0; 1273 1274 err_put_syncobjs: 1275 while (i-- > 0) 1276 drm_syncobj_put(syncobjs[i]); 1277 kfree(syncobjs); 1278 err_free_handles: 1279 kfree(handles); 1280 1281 return ret; 1282 } 1283 1284 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs, 1285 uint32_t count) 1286 { 1287 uint32_t i; 1288 1289 for (i = 0; i < count; i++) 1290 drm_syncobj_put(syncobjs[i]); 1291 kfree(syncobjs); 1292 } 1293 1294 int 1295 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data, 1296 struct drm_file *file_private) 1297 { 1298 struct drm_syncobj_wait *args = data; 1299 struct drm_syncobj **syncobjs; 1300 int ret = 0; 1301 1302 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1303 return -EOPNOTSUPP; 1304 1305 if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL | 1306 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)) 1307 return -EINVAL; 1308 1309 if (args->count_handles == 0) 1310 return -EINVAL; 1311 1312 ret = drm_syncobj_array_find(file_private, 1313 u64_to_user_ptr(args->handles), 1314 args->count_handles, 1315 &syncobjs); 1316 if (ret < 0) 1317 return ret; 1318 1319 ret = drm_syncobj_array_wait(dev, file_private, 1320 args, NULL, syncobjs, false); 1321 1322 drm_syncobj_array_free(syncobjs, args->count_handles); 1323 1324 return ret; 1325 } 1326 1327 int 1328 drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data, 1329 struct drm_file *file_private) 1330 { 1331 struct drm_syncobj_timeline_wait *args = data; 1332 struct drm_syncobj **syncobjs; 1333 int ret = 0; 1334 1335 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1336 return -EOPNOTSUPP; 1337 1338 if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL | 1339 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1340 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) 1341 return -EINVAL; 1342 1343 if (args->count_handles == 0) 1344 return -EINVAL; 1345 1346 ret = drm_syncobj_array_find(file_private, 1347 u64_to_user_ptr(args->handles), 1348 args->count_handles, 1349 &syncobjs); 1350 if (ret < 0) 1351 return ret; 1352 1353 ret = drm_syncobj_array_wait(dev, file_private, 1354 NULL, args, syncobjs, true); 1355 1356 drm_syncobj_array_free(syncobjs, args->count_handles); 1357 1358 return ret; 1359 } 1360 1361 static void syncobj_eventfd_entry_fence_func(struct dma_fence *fence, 1362 struct dma_fence_cb *cb) 1363 { 1364 struct syncobj_eventfd_entry *entry = 1365 container_of(cb, struct syncobj_eventfd_entry, fence_cb); 1366 1367 eventfd_signal(entry->ev_fd_ctx, 1); 1368 syncobj_eventfd_entry_free(entry); 1369 } 1370 1371 static void 1372 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj, 1373 struct syncobj_eventfd_entry *entry) 1374 { 1375 int ret; 1376 struct dma_fence *fence; 1377 1378 /* This happens inside the syncobj lock */ 1379 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1)); 1380 ret = dma_fence_chain_find_seqno(&fence, entry->point); 1381 if (ret != 0 || !fence) { 1382 dma_fence_put(fence); 1383 return; 1384 } 1385 1386 list_del_init(&entry->node); 1387 entry->fence = fence; 1388 1389 if (entry->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) { 1390 eventfd_signal(entry->ev_fd_ctx, 1); 1391 syncobj_eventfd_entry_free(entry); 1392 } else { 1393 ret = dma_fence_add_callback(fence, &entry->fence_cb, 1394 syncobj_eventfd_entry_fence_func); 1395 if (ret == -ENOENT) { 1396 eventfd_signal(entry->ev_fd_ctx, 1); 1397 syncobj_eventfd_entry_free(entry); 1398 } 1399 } 1400 } 1401 1402 int 1403 drm_syncobj_eventfd_ioctl(struct drm_device *dev, void *data, 1404 struct drm_file *file_private) 1405 { 1406 struct drm_syncobj_eventfd *args = data; 1407 struct drm_syncobj *syncobj; 1408 struct eventfd_ctx *ev_fd_ctx; 1409 struct syncobj_eventfd_entry *entry; 1410 1411 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1412 return -EOPNOTSUPP; 1413 1414 if (args->flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) 1415 return -EINVAL; 1416 1417 if (args->pad) 1418 return -EINVAL; 1419 1420 syncobj = drm_syncobj_find(file_private, args->handle); 1421 if (!syncobj) 1422 return -ENOENT; 1423 1424 ev_fd_ctx = eventfd_ctx_fdget(args->fd); 1425 if (IS_ERR(ev_fd_ctx)) 1426 return PTR_ERR(ev_fd_ctx); 1427 1428 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 1429 if (!entry) { 1430 eventfd_ctx_put(ev_fd_ctx); 1431 return -ENOMEM; 1432 } 1433 entry->syncobj = syncobj; 1434 entry->ev_fd_ctx = ev_fd_ctx; 1435 entry->point = args->point; 1436 entry->flags = args->flags; 1437 1438 drm_syncobj_add_eventfd(syncobj, entry); 1439 drm_syncobj_put(syncobj); 1440 1441 return 0; 1442 } 1443 1444 int 1445 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data, 1446 struct drm_file *file_private) 1447 { 1448 struct drm_syncobj_array *args = data; 1449 struct drm_syncobj **syncobjs; 1450 uint32_t i; 1451 int ret; 1452 1453 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1454 return -EOPNOTSUPP; 1455 1456 if (args->pad != 0) 1457 return -EINVAL; 1458 1459 if (args->count_handles == 0) 1460 return -EINVAL; 1461 1462 ret = drm_syncobj_array_find(file_private, 1463 u64_to_user_ptr(args->handles), 1464 args->count_handles, 1465 &syncobjs); 1466 if (ret < 0) 1467 return ret; 1468 1469 for (i = 0; i < args->count_handles; i++) 1470 drm_syncobj_replace_fence(syncobjs[i], NULL); 1471 1472 drm_syncobj_array_free(syncobjs, args->count_handles); 1473 1474 return 0; 1475 } 1476 1477 int 1478 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data, 1479 struct drm_file *file_private) 1480 { 1481 struct drm_syncobj_array *args = data; 1482 struct drm_syncobj **syncobjs; 1483 uint32_t i; 1484 int ret; 1485 1486 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1487 return -EOPNOTSUPP; 1488 1489 if (args->pad != 0) 1490 return -EINVAL; 1491 1492 if (args->count_handles == 0) 1493 return -EINVAL; 1494 1495 ret = drm_syncobj_array_find(file_private, 1496 u64_to_user_ptr(args->handles), 1497 args->count_handles, 1498 &syncobjs); 1499 if (ret < 0) 1500 return ret; 1501 1502 for (i = 0; i < args->count_handles; i++) { 1503 ret = drm_syncobj_assign_null_handle(syncobjs[i]); 1504 if (ret < 0) 1505 break; 1506 } 1507 1508 drm_syncobj_array_free(syncobjs, args->count_handles); 1509 1510 return ret; 1511 } 1512 1513 int 1514 drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data, 1515 struct drm_file *file_private) 1516 { 1517 struct drm_syncobj_timeline_array *args = data; 1518 struct drm_syncobj **syncobjs; 1519 struct dma_fence_chain **chains; 1520 uint64_t *points; 1521 uint32_t i, j; 1522 int ret; 1523 1524 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1525 return -EOPNOTSUPP; 1526 1527 if (args->flags != 0) 1528 return -EINVAL; 1529 1530 if (args->count_handles == 0) 1531 return -EINVAL; 1532 1533 ret = drm_syncobj_array_find(file_private, 1534 u64_to_user_ptr(args->handles), 1535 args->count_handles, 1536 &syncobjs); 1537 if (ret < 0) 1538 return ret; 1539 1540 points = kmalloc_array(args->count_handles, sizeof(*points), 1541 GFP_KERNEL); 1542 if (!points) { 1543 ret = -ENOMEM; 1544 goto out; 1545 } 1546 if (!u64_to_user_ptr(args->points)) { 1547 memset(points, 0, args->count_handles * sizeof(uint64_t)); 1548 } else if (copy_from_user(points, u64_to_user_ptr(args->points), 1549 sizeof(uint64_t) * args->count_handles)) { 1550 ret = -EFAULT; 1551 goto err_points; 1552 } 1553 1554 chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL); 1555 if (!chains) { 1556 ret = -ENOMEM; 1557 goto err_points; 1558 } 1559 for (i = 0; i < args->count_handles; i++) { 1560 chains[i] = dma_fence_chain_alloc(); 1561 if (!chains[i]) { 1562 for (j = 0; j < i; j++) 1563 dma_fence_chain_free(chains[j]); 1564 ret = -ENOMEM; 1565 goto err_chains; 1566 } 1567 } 1568 1569 for (i = 0; i < args->count_handles; i++) { 1570 struct dma_fence *fence = dma_fence_get_stub(); 1571 1572 drm_syncobj_add_point(syncobjs[i], chains[i], 1573 fence, points[i]); 1574 dma_fence_put(fence); 1575 } 1576 err_chains: 1577 kfree(chains); 1578 err_points: 1579 kfree(points); 1580 out: 1581 drm_syncobj_array_free(syncobjs, args->count_handles); 1582 1583 return ret; 1584 } 1585 1586 int drm_syncobj_query_ioctl(struct drm_device *dev, void *data, 1587 struct drm_file *file_private) 1588 { 1589 struct drm_syncobj_timeline_array *args = data; 1590 struct drm_syncobj **syncobjs; 1591 uint64_t __user *points = u64_to_user_ptr(args->points); 1592 uint32_t i; 1593 int ret; 1594 1595 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1596 return -EOPNOTSUPP; 1597 1598 if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) 1599 return -EINVAL; 1600 1601 if (args->count_handles == 0) 1602 return -EINVAL; 1603 1604 ret = drm_syncobj_array_find(file_private, 1605 u64_to_user_ptr(args->handles), 1606 args->count_handles, 1607 &syncobjs); 1608 if (ret < 0) 1609 return ret; 1610 1611 for (i = 0; i < args->count_handles; i++) { 1612 struct dma_fence_chain *chain; 1613 struct dma_fence *fence; 1614 uint64_t point; 1615 1616 fence = drm_syncobj_fence_get(syncobjs[i]); 1617 chain = to_dma_fence_chain(fence); 1618 if (chain) { 1619 struct dma_fence *iter, *last_signaled = 1620 dma_fence_get(fence); 1621 1622 if (args->flags & 1623 DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) { 1624 point = fence->seqno; 1625 } else { 1626 dma_fence_chain_for_each(iter, fence) { 1627 if (iter->context != fence->context) { 1628 dma_fence_put(iter); 1629 /* It is most likely that timeline has 1630 * unorder points. */ 1631 break; 1632 } 1633 dma_fence_put(last_signaled); 1634 last_signaled = dma_fence_get(iter); 1635 } 1636 point = dma_fence_is_signaled(last_signaled) ? 1637 last_signaled->seqno : 1638 to_dma_fence_chain(last_signaled)->prev_seqno; 1639 } 1640 dma_fence_put(last_signaled); 1641 } else { 1642 point = 0; 1643 } 1644 dma_fence_put(fence); 1645 ret = copy_to_user(&points[i], &point, sizeof(uint64_t)); 1646 ret = ret ? -EFAULT : 0; 1647 if (ret) 1648 break; 1649 } 1650 drm_syncobj_array_free(syncobjs, args->count_handles); 1651 1652 return ret; 1653 } 1654