1 /* 2 * Copyright 2017 Red Hat 3 * Parts ported from amdgpu (fence wait code). 4 * Copyright 2016 Advanced Micro Devices, Inc. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 * IN THE SOFTWARE. 24 * 25 * Authors: 26 * 27 */ 28 29 /** 30 * DOC: Overview 31 * 32 * DRM synchronisation objects (syncobj, see struct &drm_syncobj) provide a 33 * container for a synchronization primitive which can be used by userspace 34 * to explicitly synchronize GPU commands, can be shared between userspace 35 * processes, and can be shared between different DRM drivers. 36 * Their primary use-case is to implement Vulkan fences and semaphores. 37 * The syncobj userspace API provides ioctls for several operations: 38 * 39 * - Creation and destruction of syncobjs 40 * - Import and export of syncobjs to/from a syncobj file descriptor 41 * - Import and export a syncobj's underlying fence to/from a sync file 42 * - Reset a syncobj (set its fence to NULL) 43 * - Signal a syncobj (set a trivially signaled fence) 44 * - Wait for a syncobj's fence to appear and be signaled 45 * 46 * The syncobj userspace API also provides operations to manipulate a syncobj 47 * in terms of a timeline of struct &dma_fence_chain rather than a single 48 * struct &dma_fence, through the following operations: 49 * 50 * - Signal a given point on the timeline 51 * - Wait for a given point to appear and/or be signaled 52 * - Import and export from/to a given point of a timeline 53 * 54 * At it's core, a syncobj is simply a wrapper around a pointer to a struct 55 * &dma_fence which may be NULL. 56 * When a syncobj is first created, its pointer is either NULL or a pointer 57 * to an already signaled fence depending on whether the 58 * &DRM_SYNCOBJ_CREATE_SIGNALED flag is passed to 59 * &DRM_IOCTL_SYNCOBJ_CREATE. 60 * 61 * If the syncobj is considered as a binary (its state is either signaled or 62 * unsignaled) primitive, when GPU work is enqueued in a DRM driver to signal 63 * the syncobj, the syncobj's fence is replaced with a fence which will be 64 * signaled by the completion of that work. 65 * If the syncobj is considered as a timeline primitive, when GPU work is 66 * enqueued in a DRM driver to signal the a given point of the syncobj, a new 67 * struct &dma_fence_chain pointing to the DRM driver's fence and also 68 * pointing to the previous fence that was in the syncobj. The new struct 69 * &dma_fence_chain fence replace the syncobj's fence and will be signaled by 70 * completion of the DRM driver's work and also any work associated with the 71 * fence previously in the syncobj. 72 * 73 * When GPU work which waits on a syncobj is enqueued in a DRM driver, at the 74 * time the work is enqueued, it waits on the syncobj's fence before 75 * submitting the work to hardware. That fence is either : 76 * 77 * - The syncobj's current fence if the syncobj is considered as a binary 78 * primitive. 79 * - The struct &dma_fence associated with a given point if the syncobj is 80 * considered as a timeline primitive. 81 * 82 * If the syncobj's fence is NULL or not present in the syncobj's timeline, 83 * the enqueue operation is expected to fail. 84 * 85 * With binary syncobj, all manipulation of the syncobjs's fence happens in 86 * terms of the current fence at the time the ioctl is called by userspace 87 * regardless of whether that operation is an immediate host-side operation 88 * (signal or reset) or or an operation which is enqueued in some driver 89 * queue. &DRM_IOCTL_SYNCOBJ_RESET and &DRM_IOCTL_SYNCOBJ_SIGNAL can be used 90 * to manipulate a syncobj from the host by resetting its pointer to NULL or 91 * setting its pointer to a fence which is already signaled. 92 * 93 * With a timeline syncobj, all manipulation of the synobj's fence happens in 94 * terms of a u64 value referring to point in the timeline. See 95 * dma_fence_chain_find_seqno() to see how a given point is found in the 96 * timeline. 97 * 98 * Note that applications should be careful to always use timeline set of 99 * ioctl() when dealing with syncobj considered as timeline. Using a binary 100 * set of ioctl() with a syncobj considered as timeline could result incorrect 101 * synchronization. The use of binary syncobj is supported through the 102 * timeline set of ioctl() by using a point value of 0, this will reproduce 103 * the behavior of the binary set of ioctl() (for example replace the 104 * syncobj's fence when signaling). 105 * 106 * 107 * Host-side wait on syncobjs 108 * -------------------------- 109 * 110 * &DRM_IOCTL_SYNCOBJ_WAIT takes an array of syncobj handles and does a 111 * host-side wait on all of the syncobj fences simultaneously. 112 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL is set, the wait ioctl will wait on 113 * all of the syncobj fences to be signaled before it returns. 114 * Otherwise, it returns once at least one syncobj fence has been signaled 115 * and the index of a signaled fence is written back to the client. 116 * 117 * Unlike the enqueued GPU work dependencies which fail if they see a NULL 118 * fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set, 119 * the host-side wait will first wait for the syncobj to receive a non-NULL 120 * fence and then wait on that fence. 121 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is not set and any one of the 122 * syncobjs in the array has a NULL fence, -EINVAL will be returned. 123 * Assuming the syncobj starts off with a NULL fence, this allows a client 124 * to do a host wait in one thread (or process) which waits on GPU work 125 * submitted in another thread (or process) without having to manually 126 * synchronize between the two. 127 * This requirement is inherited from the Vulkan fence API. 128 * 129 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE is set, the ioctl will also set 130 * a fence deadline hint on the backing fences before waiting, to provide the 131 * fence signaler with an appropriate sense of urgency. The deadline is 132 * specified as an absolute &CLOCK_MONOTONIC value in units of ns. 133 * 134 * Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj 135 * handles as well as an array of u64 points and does a host-side wait on all 136 * of syncobj fences at the given points simultaneously. 137 * 138 * &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT also adds the ability to wait for a given 139 * fence to materialize on the timeline without waiting for the fence to be 140 * signaled by using the &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE flag. This 141 * requirement is inherited from the wait-before-signal behavior required by 142 * the Vulkan timeline semaphore API. 143 * 144 * Alternatively, &DRM_IOCTL_SYNCOBJ_EVENTFD can be used to wait without 145 * blocking: an eventfd will be signaled when the syncobj is. This is useful to 146 * integrate the wait in an event loop. 147 * 148 * 149 * Import/export of syncobjs 150 * ------------------------- 151 * 152 * &DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE and &DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD 153 * provide two mechanisms for import/export of syncobjs. 154 * 155 * The first lets the client import or export an entire syncobj to a file 156 * descriptor. 157 * These fd's are opaque and have no other use case, except passing the 158 * syncobj between processes. 159 * All exported file descriptors and any syncobj handles created as a 160 * result of importing those file descriptors own a reference to the 161 * same underlying struct &drm_syncobj and the syncobj can be used 162 * persistently across all the processes with which it is shared. 163 * The syncobj is freed only once the last reference is dropped. 164 * Unlike dma-buf, importing a syncobj creates a new handle (with its own 165 * reference) for every import instead of de-duplicating. 166 * The primary use-case of this persistent import/export is for shared 167 * Vulkan fences and semaphores. 168 * 169 * The second import/export mechanism, which is indicated by 170 * &DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE or 171 * &DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE lets the client 172 * import/export the syncobj's current fence from/to a &sync_file. 173 * When a syncobj is exported to a sync file, that sync file wraps the 174 * sycnobj's fence at the time of export and any later signal or reset 175 * operations on the syncobj will not affect the exported sync file. 176 * When a sync file is imported into a syncobj, the syncobj's fence is set 177 * to the fence wrapped by that sync file. 178 * Because sync files are immutable, resetting or signaling the syncobj 179 * will not affect any sync files whose fences have been imported into the 180 * syncobj. 181 * 182 * 183 * Import/export of timeline points in timeline syncobjs 184 * ----------------------------------------------------- 185 * 186 * &DRM_IOCTL_SYNCOBJ_TRANSFER provides a mechanism to transfer a struct 187 * &dma_fence_chain of a syncobj at a given u64 point to another u64 point 188 * into another syncobj. 189 * 190 * Note that if you want to transfer a struct &dma_fence_chain from a given 191 * point on a timeline syncobj from/into a binary syncobj, you can use the 192 * point 0 to mean take/replace the fence in the syncobj. 193 */ 194 195 #include <linux/anon_inodes.h> 196 #include <linux/dma-fence-unwrap.h> 197 #include <linux/eventfd.h> 198 #include <linux/file.h> 199 #include <linux/fs.h> 200 #include <linux/sched/signal.h> 201 #include <linux/sync_file.h> 202 #include <linux/uaccess.h> 203 204 #include <drm/drm.h> 205 #include <drm/drm_drv.h> 206 #include <drm/drm_file.h> 207 #include <drm/drm_gem.h> 208 #include <drm/drm_print.h> 209 #include <drm/drm_syncobj.h> 210 #include <drm/drm_utils.h> 211 212 #include "drm_internal.h" 213 214 struct syncobj_wait_entry { 215 struct list_head node; 216 struct task_struct *task; 217 struct dma_fence *fence; 218 struct dma_fence_cb fence_cb; 219 u64 point; 220 }; 221 222 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj, 223 struct syncobj_wait_entry *wait); 224 225 struct syncobj_eventfd_entry { 226 struct list_head node; 227 struct dma_fence *fence; 228 struct dma_fence_cb fence_cb; 229 struct drm_syncobj *syncobj; 230 struct eventfd_ctx *ev_fd_ctx; 231 u64 point; 232 u32 flags; 233 }; 234 235 static void 236 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj, 237 struct syncobj_eventfd_entry *entry); 238 239 /** 240 * drm_syncobj_find - lookup and reference a sync object. 241 * @file_private: drm file private pointer 242 * @handle: sync object handle to lookup. 243 * 244 * Returns a reference to the syncobj pointed to by handle or NULL. The 245 * reference must be released by calling drm_syncobj_put(). 246 */ 247 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private, 248 u32 handle) 249 { 250 struct drm_syncobj *syncobj; 251 252 spin_lock(&file_private->syncobj_table_lock); 253 254 /* Check if we currently have a reference on the object */ 255 syncobj = idr_find(&file_private->syncobj_idr, handle); 256 if (syncobj) 257 drm_syncobj_get(syncobj); 258 259 spin_unlock(&file_private->syncobj_table_lock); 260 261 return syncobj; 262 } 263 EXPORT_SYMBOL(drm_syncobj_find); 264 265 static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj, 266 struct syncobj_wait_entry *wait) 267 { 268 struct dma_fence *fence; 269 270 if (wait->fence) 271 return; 272 273 spin_lock(&syncobj->lock); 274 /* We've already tried once to get a fence and failed. Now that we 275 * have the lock, try one more time just to be sure we don't add a 276 * callback when a fence has already been set. 277 */ 278 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1)); 279 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) { 280 dma_fence_put(fence); 281 list_add_tail(&wait->node, &syncobj->cb_list); 282 } else if (!fence) { 283 wait->fence = dma_fence_get_stub(); 284 } else { 285 wait->fence = fence; 286 } 287 spin_unlock(&syncobj->lock); 288 } 289 290 static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj, 291 struct syncobj_wait_entry *wait) 292 { 293 if (!wait->node.next) 294 return; 295 296 spin_lock(&syncobj->lock); 297 list_del_init(&wait->node); 298 spin_unlock(&syncobj->lock); 299 } 300 301 static void 302 syncobj_eventfd_entry_free(struct syncobj_eventfd_entry *entry) 303 { 304 eventfd_ctx_put(entry->ev_fd_ctx); 305 dma_fence_put(entry->fence); 306 /* This happens either inside the syncobj lock, or after the node has 307 * already been removed from the list. 308 */ 309 list_del(&entry->node); 310 kfree(entry); 311 } 312 313 static void 314 drm_syncobj_add_eventfd(struct drm_syncobj *syncobj, 315 struct syncobj_eventfd_entry *entry) 316 { 317 spin_lock(&syncobj->lock); 318 list_add_tail(&entry->node, &syncobj->ev_fd_list); 319 syncobj_eventfd_entry_func(syncobj, entry); 320 spin_unlock(&syncobj->lock); 321 } 322 323 /** 324 * drm_syncobj_add_point - add new timeline point to the syncobj 325 * @syncobj: sync object to add timeline point do 326 * @chain: chain node to use to add the point 327 * @fence: fence to encapsulate in the chain node 328 * @point: sequence number to use for the point 329 * 330 * Add the chain node as new timeline point to the syncobj. 331 */ 332 void drm_syncobj_add_point(struct drm_syncobj *syncobj, 333 struct dma_fence_chain *chain, 334 struct dma_fence *fence, 335 uint64_t point) 336 { 337 struct syncobj_wait_entry *wait_cur, *wait_tmp; 338 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 339 struct dma_fence *prev; 340 341 dma_fence_get(fence); 342 343 spin_lock(&syncobj->lock); 344 345 prev = drm_syncobj_fence_get(syncobj); 346 /* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */ 347 if (prev && prev->seqno >= point) 348 DRM_DEBUG("You are adding an unorder point to timeline!\n"); 349 dma_fence_chain_init(chain, prev, fence, point); 350 rcu_assign_pointer(syncobj->fence, &chain->base); 351 352 list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node) 353 syncobj_wait_syncobj_func(syncobj, wait_cur); 354 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 355 syncobj_eventfd_entry_func(syncobj, ev_fd_cur); 356 spin_unlock(&syncobj->lock); 357 358 /* Walk the chain once to trigger garbage collection */ 359 dma_fence_chain_for_each(fence, prev); 360 dma_fence_put(prev); 361 } 362 EXPORT_SYMBOL(drm_syncobj_add_point); 363 364 /** 365 * drm_syncobj_replace_fence - replace fence in a sync object. 366 * @syncobj: Sync object to replace fence in 367 * @fence: fence to install in sync file. 368 * 369 * This replaces the fence on a sync object. 370 */ 371 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj, 372 struct dma_fence *fence) 373 { 374 struct dma_fence *old_fence; 375 struct syncobj_wait_entry *wait_cur, *wait_tmp; 376 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 377 378 if (fence) 379 dma_fence_get(fence); 380 381 spin_lock(&syncobj->lock); 382 383 old_fence = rcu_dereference_protected(syncobj->fence, 384 lockdep_is_held(&syncobj->lock)); 385 rcu_assign_pointer(syncobj->fence, fence); 386 387 if (fence != old_fence) { 388 list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node) 389 syncobj_wait_syncobj_func(syncobj, wait_cur); 390 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 391 syncobj_eventfd_entry_func(syncobj, ev_fd_cur); 392 } 393 394 spin_unlock(&syncobj->lock); 395 396 dma_fence_put(old_fence); 397 } 398 EXPORT_SYMBOL(drm_syncobj_replace_fence); 399 400 /** 401 * drm_syncobj_assign_null_handle - assign a stub fence to the sync object 402 * @syncobj: sync object to assign the fence on 403 * 404 * Assign a already signaled stub fence to the sync object. 405 */ 406 static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj) 407 { 408 struct dma_fence *fence = dma_fence_allocate_private_stub(ktime_get()); 409 410 if (!fence) 411 return -ENOMEM; 412 413 drm_syncobj_replace_fence(syncobj, fence); 414 dma_fence_put(fence); 415 return 0; 416 } 417 418 /* 5s default for wait submission */ 419 #define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL 420 /** 421 * drm_syncobj_find_fence - lookup and reference the fence in a sync object 422 * @file_private: drm file private pointer 423 * @handle: sync object handle to lookup. 424 * @point: timeline point 425 * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not 426 * @fence: out parameter for the fence 427 * 428 * This is just a convenience function that combines drm_syncobj_find() and 429 * drm_syncobj_fence_get(). 430 * 431 * Returns 0 on success or a negative error value on failure. On success @fence 432 * contains a reference to the fence, which must be released by calling 433 * dma_fence_put(). 434 */ 435 int drm_syncobj_find_fence(struct drm_file *file_private, 436 u32 handle, u64 point, u64 flags, 437 struct dma_fence **fence) 438 { 439 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle); 440 struct syncobj_wait_entry wait; 441 u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT); 442 int ret; 443 444 if (!syncobj) 445 return -ENOENT; 446 447 /* Waiting for userspace with locks help is illegal cause that can 448 * trivial deadlock with page faults for example. Make lockdep complain 449 * about it early on. 450 */ 451 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) { 452 might_sleep(); 453 lockdep_assert_none_held_once(); 454 } 455 456 *fence = drm_syncobj_fence_get(syncobj); 457 458 if (*fence) { 459 ret = dma_fence_chain_find_seqno(fence, point); 460 if (!ret) { 461 /* If the requested seqno is already signaled 462 * drm_syncobj_find_fence may return a NULL 463 * fence. To make sure the recipient gets 464 * signalled, use a new fence instead. 465 */ 466 if (!*fence) 467 *fence = dma_fence_get_stub(); 468 469 goto out; 470 } 471 dma_fence_put(*fence); 472 } else { 473 ret = -EINVAL; 474 } 475 476 if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)) 477 goto out; 478 479 memset(&wait, 0, sizeof(wait)); 480 wait.task = current; 481 wait.point = point; 482 drm_syncobj_fence_add_wait(syncobj, &wait); 483 484 do { 485 set_current_state(TASK_INTERRUPTIBLE); 486 if (wait.fence) { 487 ret = 0; 488 break; 489 } 490 if (timeout == 0) { 491 ret = -ETIME; 492 break; 493 } 494 495 if (signal_pending(current)) { 496 ret = -ERESTARTSYS; 497 break; 498 } 499 500 timeout = schedule_timeout(timeout); 501 } while (1); 502 503 __set_current_state(TASK_RUNNING); 504 *fence = wait.fence; 505 506 if (wait.node.next) 507 drm_syncobj_remove_wait(syncobj, &wait); 508 509 out: 510 drm_syncobj_put(syncobj); 511 512 return ret; 513 } 514 EXPORT_SYMBOL(drm_syncobj_find_fence); 515 516 /** 517 * drm_syncobj_free - free a sync object. 518 * @kref: kref to free. 519 * 520 * Only to be called from kref_put in drm_syncobj_put. 521 */ 522 void drm_syncobj_free(struct kref *kref) 523 { 524 struct drm_syncobj *syncobj = container_of(kref, 525 struct drm_syncobj, 526 refcount); 527 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 528 529 drm_syncobj_replace_fence(syncobj, NULL); 530 531 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 532 syncobj_eventfd_entry_free(ev_fd_cur); 533 534 kfree(syncobj); 535 } 536 EXPORT_SYMBOL(drm_syncobj_free); 537 538 /** 539 * drm_syncobj_create - create a new syncobj 540 * @out_syncobj: returned syncobj 541 * @flags: DRM_SYNCOBJ_* flags 542 * @fence: if non-NULL, the syncobj will represent this fence 543 * 544 * This is the first function to create a sync object. After creating, drivers 545 * probably want to make it available to userspace, either through 546 * drm_syncobj_get_handle() or drm_syncobj_get_fd(). 547 * 548 * Returns 0 on success or a negative error value on failure. 549 */ 550 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags, 551 struct dma_fence *fence) 552 { 553 int ret; 554 struct drm_syncobj *syncobj; 555 556 syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL); 557 if (!syncobj) 558 return -ENOMEM; 559 560 kref_init(&syncobj->refcount); 561 INIT_LIST_HEAD(&syncobj->cb_list); 562 INIT_LIST_HEAD(&syncobj->ev_fd_list); 563 spin_lock_init(&syncobj->lock); 564 565 if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) { 566 ret = drm_syncobj_assign_null_handle(syncobj); 567 if (ret < 0) { 568 drm_syncobj_put(syncobj); 569 return ret; 570 } 571 } 572 573 if (fence) 574 drm_syncobj_replace_fence(syncobj, fence); 575 576 *out_syncobj = syncobj; 577 return 0; 578 } 579 EXPORT_SYMBOL(drm_syncobj_create); 580 581 /** 582 * drm_syncobj_get_handle - get a handle from a syncobj 583 * @file_private: drm file private pointer 584 * @syncobj: Sync object to export 585 * @handle: out parameter with the new handle 586 * 587 * Exports a sync object created with drm_syncobj_create() as a handle on 588 * @file_private to userspace. 589 * 590 * Returns 0 on success or a negative error value on failure. 591 */ 592 int drm_syncobj_get_handle(struct drm_file *file_private, 593 struct drm_syncobj *syncobj, u32 *handle) 594 { 595 int ret; 596 597 /* take a reference to put in the idr */ 598 drm_syncobj_get(syncobj); 599 600 idr_preload(GFP_KERNEL); 601 spin_lock(&file_private->syncobj_table_lock); 602 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT); 603 spin_unlock(&file_private->syncobj_table_lock); 604 605 idr_preload_end(); 606 607 if (ret < 0) { 608 drm_syncobj_put(syncobj); 609 return ret; 610 } 611 612 *handle = ret; 613 return 0; 614 } 615 EXPORT_SYMBOL(drm_syncobj_get_handle); 616 617 static int drm_syncobj_create_as_handle(struct drm_file *file_private, 618 u32 *handle, uint32_t flags) 619 { 620 int ret; 621 struct drm_syncobj *syncobj; 622 623 ret = drm_syncobj_create(&syncobj, flags, NULL); 624 if (ret) 625 return ret; 626 627 ret = drm_syncobj_get_handle(file_private, syncobj, handle); 628 drm_syncobj_put(syncobj); 629 return ret; 630 } 631 632 static int drm_syncobj_destroy(struct drm_file *file_private, 633 u32 handle) 634 { 635 struct drm_syncobj *syncobj; 636 637 spin_lock(&file_private->syncobj_table_lock); 638 syncobj = idr_remove(&file_private->syncobj_idr, handle); 639 spin_unlock(&file_private->syncobj_table_lock); 640 641 if (!syncobj) 642 return -EINVAL; 643 644 drm_syncobj_put(syncobj); 645 return 0; 646 } 647 648 static int drm_syncobj_file_release(struct inode *inode, struct file *file) 649 { 650 struct drm_syncobj *syncobj = file->private_data; 651 652 drm_syncobj_put(syncobj); 653 return 0; 654 } 655 656 static const struct file_operations drm_syncobj_file_fops = { 657 .release = drm_syncobj_file_release, 658 }; 659 660 /** 661 * drm_syncobj_get_fd - get a file descriptor from a syncobj 662 * @syncobj: Sync object to export 663 * @p_fd: out parameter with the new file descriptor 664 * 665 * Exports a sync object created with drm_syncobj_create() as a file descriptor. 666 * 667 * Returns 0 on success or a negative error value on failure. 668 */ 669 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd) 670 { 671 struct file *file; 672 int fd; 673 674 fd = get_unused_fd_flags(O_CLOEXEC); 675 if (fd < 0) 676 return fd; 677 678 file = anon_inode_getfile("syncobj_file", 679 &drm_syncobj_file_fops, 680 syncobj, 0); 681 if (IS_ERR(file)) { 682 put_unused_fd(fd); 683 return PTR_ERR(file); 684 } 685 686 drm_syncobj_get(syncobj); 687 fd_install(fd, file); 688 689 *p_fd = fd; 690 return 0; 691 } 692 EXPORT_SYMBOL(drm_syncobj_get_fd); 693 694 static int drm_syncobj_handle_to_fd(struct drm_file *file_private, 695 u32 handle, int *p_fd) 696 { 697 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle); 698 int ret; 699 700 if (!syncobj) 701 return -EINVAL; 702 703 ret = drm_syncobj_get_fd(syncobj, p_fd); 704 drm_syncobj_put(syncobj); 705 return ret; 706 } 707 708 static int drm_syncobj_fd_to_handle(struct drm_file *file_private, 709 int fd, u32 *handle) 710 { 711 struct drm_syncobj *syncobj; 712 struct fd f = fdget(fd); 713 int ret; 714 715 if (!f.file) 716 return -EINVAL; 717 718 if (f.file->f_op != &drm_syncobj_file_fops) { 719 fdput(f); 720 return -EINVAL; 721 } 722 723 /* take a reference to put in the idr */ 724 syncobj = f.file->private_data; 725 drm_syncobj_get(syncobj); 726 727 idr_preload(GFP_KERNEL); 728 spin_lock(&file_private->syncobj_table_lock); 729 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT); 730 spin_unlock(&file_private->syncobj_table_lock); 731 idr_preload_end(); 732 733 if (ret > 0) { 734 *handle = ret; 735 ret = 0; 736 } else 737 drm_syncobj_put(syncobj); 738 739 fdput(f); 740 return ret; 741 } 742 743 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private, 744 int fd, int handle) 745 { 746 struct dma_fence *fence = sync_file_get_fence(fd); 747 struct drm_syncobj *syncobj; 748 749 if (!fence) 750 return -EINVAL; 751 752 syncobj = drm_syncobj_find(file_private, handle); 753 if (!syncobj) { 754 dma_fence_put(fence); 755 return -ENOENT; 756 } 757 758 drm_syncobj_replace_fence(syncobj, fence); 759 dma_fence_put(fence); 760 drm_syncobj_put(syncobj); 761 return 0; 762 } 763 764 static int drm_syncobj_export_sync_file(struct drm_file *file_private, 765 int handle, int *p_fd) 766 { 767 int ret; 768 struct dma_fence *fence; 769 struct sync_file *sync_file; 770 int fd = get_unused_fd_flags(O_CLOEXEC); 771 772 if (fd < 0) 773 return fd; 774 775 ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence); 776 if (ret) 777 goto err_put_fd; 778 779 sync_file = sync_file_create(fence); 780 781 dma_fence_put(fence); 782 783 if (!sync_file) { 784 ret = -EINVAL; 785 goto err_put_fd; 786 } 787 788 fd_install(fd, sync_file->file); 789 790 *p_fd = fd; 791 return 0; 792 err_put_fd: 793 put_unused_fd(fd); 794 return ret; 795 } 796 /** 797 * drm_syncobj_open - initializes syncobj file-private structures at devnode open time 798 * @file_private: drm file-private structure to set up 799 * 800 * Called at device open time, sets up the structure for handling refcounting 801 * of sync objects. 802 */ 803 void 804 drm_syncobj_open(struct drm_file *file_private) 805 { 806 idr_init_base(&file_private->syncobj_idr, 1); 807 spin_lock_init(&file_private->syncobj_table_lock); 808 } 809 810 static int 811 drm_syncobj_release_handle(int id, void *ptr, void *data) 812 { 813 struct drm_syncobj *syncobj = ptr; 814 815 drm_syncobj_put(syncobj); 816 return 0; 817 } 818 819 /** 820 * drm_syncobj_release - release file-private sync object resources 821 * @file_private: drm file-private structure to clean up 822 * 823 * Called at close time when the filp is going away. 824 * 825 * Releases any remaining references on objects by this filp. 826 */ 827 void 828 drm_syncobj_release(struct drm_file *file_private) 829 { 830 idr_for_each(&file_private->syncobj_idr, 831 &drm_syncobj_release_handle, file_private); 832 idr_destroy(&file_private->syncobj_idr); 833 } 834 835 int 836 drm_syncobj_create_ioctl(struct drm_device *dev, void *data, 837 struct drm_file *file_private) 838 { 839 struct drm_syncobj_create *args = data; 840 841 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 842 return -EOPNOTSUPP; 843 844 /* no valid flags yet */ 845 if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED) 846 return -EINVAL; 847 848 return drm_syncobj_create_as_handle(file_private, 849 &args->handle, args->flags); 850 } 851 852 int 853 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data, 854 struct drm_file *file_private) 855 { 856 struct drm_syncobj_destroy *args = data; 857 858 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 859 return -EOPNOTSUPP; 860 861 /* make sure padding is empty */ 862 if (args->pad) 863 return -EINVAL; 864 return drm_syncobj_destroy(file_private, args->handle); 865 } 866 867 int 868 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data, 869 struct drm_file *file_private) 870 { 871 struct drm_syncobj_handle *args = data; 872 873 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 874 return -EOPNOTSUPP; 875 876 if (args->pad) 877 return -EINVAL; 878 879 if (args->flags != 0 && 880 args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE) 881 return -EINVAL; 882 883 if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE) 884 return drm_syncobj_export_sync_file(file_private, args->handle, 885 &args->fd); 886 887 return drm_syncobj_handle_to_fd(file_private, args->handle, 888 &args->fd); 889 } 890 891 int 892 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data, 893 struct drm_file *file_private) 894 { 895 struct drm_syncobj_handle *args = data; 896 897 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 898 return -EOPNOTSUPP; 899 900 if (args->pad) 901 return -EINVAL; 902 903 if (args->flags != 0 && 904 args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE) 905 return -EINVAL; 906 907 if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE) 908 return drm_syncobj_import_sync_file_fence(file_private, 909 args->fd, 910 args->handle); 911 912 return drm_syncobj_fd_to_handle(file_private, args->fd, 913 &args->handle); 914 } 915 916 static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private, 917 struct drm_syncobj_transfer *args) 918 { 919 struct drm_syncobj *timeline_syncobj = NULL; 920 struct dma_fence *fence, *tmp; 921 struct dma_fence_chain *chain; 922 int ret; 923 924 timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle); 925 if (!timeline_syncobj) { 926 return -ENOENT; 927 } 928 ret = drm_syncobj_find_fence(file_private, args->src_handle, 929 args->src_point, args->flags, 930 &tmp); 931 if (ret) 932 goto err_put_timeline; 933 934 fence = dma_fence_unwrap_merge(tmp); 935 dma_fence_put(tmp); 936 if (!fence) { 937 ret = -ENOMEM; 938 goto err_put_timeline; 939 } 940 941 chain = dma_fence_chain_alloc(); 942 if (!chain) { 943 ret = -ENOMEM; 944 goto err_free_fence; 945 } 946 947 drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point); 948 err_free_fence: 949 dma_fence_put(fence); 950 err_put_timeline: 951 drm_syncobj_put(timeline_syncobj); 952 953 return ret; 954 } 955 956 static int 957 drm_syncobj_transfer_to_binary(struct drm_file *file_private, 958 struct drm_syncobj_transfer *args) 959 { 960 struct drm_syncobj *binary_syncobj = NULL; 961 struct dma_fence *fence; 962 int ret; 963 964 binary_syncobj = drm_syncobj_find(file_private, args->dst_handle); 965 if (!binary_syncobj) 966 return -ENOENT; 967 ret = drm_syncobj_find_fence(file_private, args->src_handle, 968 args->src_point, args->flags, &fence); 969 if (ret) 970 goto err; 971 drm_syncobj_replace_fence(binary_syncobj, fence); 972 dma_fence_put(fence); 973 err: 974 drm_syncobj_put(binary_syncobj); 975 976 return ret; 977 } 978 int 979 drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data, 980 struct drm_file *file_private) 981 { 982 struct drm_syncobj_transfer *args = data; 983 int ret; 984 985 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 986 return -EOPNOTSUPP; 987 988 if (args->pad) 989 return -EINVAL; 990 991 if (args->dst_point) 992 ret = drm_syncobj_transfer_to_timeline(file_private, args); 993 else 994 ret = drm_syncobj_transfer_to_binary(file_private, args); 995 996 return ret; 997 } 998 999 static void syncobj_wait_fence_func(struct dma_fence *fence, 1000 struct dma_fence_cb *cb) 1001 { 1002 struct syncobj_wait_entry *wait = 1003 container_of(cb, struct syncobj_wait_entry, fence_cb); 1004 1005 wake_up_process(wait->task); 1006 } 1007 1008 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj, 1009 struct syncobj_wait_entry *wait) 1010 { 1011 struct dma_fence *fence; 1012 1013 /* This happens inside the syncobj lock */ 1014 fence = rcu_dereference_protected(syncobj->fence, 1015 lockdep_is_held(&syncobj->lock)); 1016 dma_fence_get(fence); 1017 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) { 1018 dma_fence_put(fence); 1019 return; 1020 } else if (!fence) { 1021 wait->fence = dma_fence_get_stub(); 1022 } else { 1023 wait->fence = fence; 1024 } 1025 1026 wake_up_process(wait->task); 1027 list_del_init(&wait->node); 1028 } 1029 1030 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs, 1031 void __user *user_points, 1032 uint32_t count, 1033 uint32_t flags, 1034 signed long timeout, 1035 uint32_t *idx, 1036 ktime_t *deadline) 1037 { 1038 struct syncobj_wait_entry *entries; 1039 struct dma_fence *fence; 1040 uint64_t *points; 1041 uint32_t signaled_count, i; 1042 1043 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) 1044 lockdep_assert_none_held_once(); 1045 1046 points = kmalloc_array(count, sizeof(*points), GFP_KERNEL); 1047 if (points == NULL) 1048 return -ENOMEM; 1049 1050 if (!user_points) { 1051 memset(points, 0, count * sizeof(uint64_t)); 1052 1053 } else if (copy_from_user(points, user_points, 1054 sizeof(uint64_t) * count)) { 1055 timeout = -EFAULT; 1056 goto err_free_points; 1057 } 1058 1059 entries = kcalloc(count, sizeof(*entries), GFP_KERNEL); 1060 if (!entries) { 1061 timeout = -ENOMEM; 1062 goto err_free_points; 1063 } 1064 /* Walk the list of sync objects and initialize entries. We do 1065 * this up-front so that we can properly return -EINVAL if there is 1066 * a syncobj with a missing fence and then never have the chance of 1067 * returning -EINVAL again. 1068 */ 1069 signaled_count = 0; 1070 for (i = 0; i < count; ++i) { 1071 struct dma_fence *fence; 1072 1073 entries[i].task = current; 1074 entries[i].point = points[i]; 1075 fence = drm_syncobj_fence_get(syncobjs[i]); 1076 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) { 1077 dma_fence_put(fence); 1078 if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1079 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) { 1080 continue; 1081 } else { 1082 timeout = -EINVAL; 1083 goto cleanup_entries; 1084 } 1085 } 1086 1087 if (fence) 1088 entries[i].fence = fence; 1089 else 1090 entries[i].fence = dma_fence_get_stub(); 1091 1092 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) || 1093 dma_fence_is_signaled(entries[i].fence)) { 1094 if (signaled_count == 0 && idx) 1095 *idx = i; 1096 signaled_count++; 1097 } 1098 } 1099 1100 if (signaled_count == count || 1101 (signaled_count > 0 && 1102 !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL))) 1103 goto cleanup_entries; 1104 1105 /* There's a very annoying laxness in the dma_fence API here, in 1106 * that backends are not required to automatically report when a 1107 * fence is signaled prior to fence->ops->enable_signaling() being 1108 * called. So here if we fail to match signaled_count, we need to 1109 * fallthough and try a 0 timeout wait! 1110 */ 1111 1112 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) { 1113 for (i = 0; i < count; ++i) 1114 drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]); 1115 } 1116 1117 if (deadline) { 1118 for (i = 0; i < count; ++i) { 1119 fence = entries[i].fence; 1120 if (!fence) 1121 continue; 1122 dma_fence_set_deadline(fence, *deadline); 1123 } 1124 } 1125 1126 do { 1127 set_current_state(TASK_INTERRUPTIBLE); 1128 1129 signaled_count = 0; 1130 for (i = 0; i < count; ++i) { 1131 fence = entries[i].fence; 1132 if (!fence) 1133 continue; 1134 1135 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) || 1136 dma_fence_is_signaled(fence) || 1137 (!entries[i].fence_cb.func && 1138 dma_fence_add_callback(fence, 1139 &entries[i].fence_cb, 1140 syncobj_wait_fence_func))) { 1141 /* The fence has been signaled */ 1142 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) { 1143 signaled_count++; 1144 } else { 1145 if (idx) 1146 *idx = i; 1147 goto done_waiting; 1148 } 1149 } 1150 } 1151 1152 if (signaled_count == count) 1153 goto done_waiting; 1154 1155 if (timeout == 0) { 1156 timeout = -ETIME; 1157 goto done_waiting; 1158 } 1159 1160 if (signal_pending(current)) { 1161 timeout = -ERESTARTSYS; 1162 goto done_waiting; 1163 } 1164 1165 timeout = schedule_timeout(timeout); 1166 } while (1); 1167 1168 done_waiting: 1169 __set_current_state(TASK_RUNNING); 1170 1171 cleanup_entries: 1172 for (i = 0; i < count; ++i) { 1173 drm_syncobj_remove_wait(syncobjs[i], &entries[i]); 1174 if (entries[i].fence_cb.func) 1175 dma_fence_remove_callback(entries[i].fence, 1176 &entries[i].fence_cb); 1177 dma_fence_put(entries[i].fence); 1178 } 1179 kfree(entries); 1180 1181 err_free_points: 1182 kfree(points); 1183 1184 return timeout; 1185 } 1186 1187 /** 1188 * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value 1189 * 1190 * @timeout_nsec: timeout nsec component in ns, 0 for poll 1191 * 1192 * Calculate the timeout in jiffies from an absolute time in sec/nsec. 1193 */ 1194 signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec) 1195 { 1196 ktime_t abs_timeout, now; 1197 u64 timeout_ns, timeout_jiffies64; 1198 1199 /* make 0 timeout means poll - absolute 0 doesn't seem valid */ 1200 if (timeout_nsec == 0) 1201 return 0; 1202 1203 abs_timeout = ns_to_ktime(timeout_nsec); 1204 now = ktime_get(); 1205 1206 if (!ktime_after(abs_timeout, now)) 1207 return 0; 1208 1209 timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now)); 1210 1211 timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns); 1212 /* clamp timeout to avoid infinite timeout */ 1213 if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1) 1214 return MAX_SCHEDULE_TIMEOUT - 1; 1215 1216 return timeout_jiffies64 + 1; 1217 } 1218 EXPORT_SYMBOL(drm_timeout_abs_to_jiffies); 1219 1220 static int drm_syncobj_array_wait(struct drm_device *dev, 1221 struct drm_file *file_private, 1222 struct drm_syncobj_wait *wait, 1223 struct drm_syncobj_timeline_wait *timeline_wait, 1224 struct drm_syncobj **syncobjs, bool timeline, 1225 ktime_t *deadline) 1226 { 1227 signed long timeout = 0; 1228 uint32_t first = ~0; 1229 1230 if (!timeline) { 1231 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec); 1232 timeout = drm_syncobj_array_wait_timeout(syncobjs, 1233 NULL, 1234 wait->count_handles, 1235 wait->flags, 1236 timeout, &first, 1237 deadline); 1238 if (timeout < 0) 1239 return timeout; 1240 wait->first_signaled = first; 1241 } else { 1242 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec); 1243 timeout = drm_syncobj_array_wait_timeout(syncobjs, 1244 u64_to_user_ptr(timeline_wait->points), 1245 timeline_wait->count_handles, 1246 timeline_wait->flags, 1247 timeout, &first, 1248 deadline); 1249 if (timeout < 0) 1250 return timeout; 1251 timeline_wait->first_signaled = first; 1252 } 1253 return 0; 1254 } 1255 1256 static int drm_syncobj_array_find(struct drm_file *file_private, 1257 void __user *user_handles, 1258 uint32_t count_handles, 1259 struct drm_syncobj ***syncobjs_out) 1260 { 1261 uint32_t i, *handles; 1262 struct drm_syncobj **syncobjs; 1263 int ret; 1264 1265 handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL); 1266 if (handles == NULL) 1267 return -ENOMEM; 1268 1269 if (copy_from_user(handles, user_handles, 1270 sizeof(uint32_t) * count_handles)) { 1271 ret = -EFAULT; 1272 goto err_free_handles; 1273 } 1274 1275 syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL); 1276 if (syncobjs == NULL) { 1277 ret = -ENOMEM; 1278 goto err_free_handles; 1279 } 1280 1281 for (i = 0; i < count_handles; i++) { 1282 syncobjs[i] = drm_syncobj_find(file_private, handles[i]); 1283 if (!syncobjs[i]) { 1284 ret = -ENOENT; 1285 goto err_put_syncobjs; 1286 } 1287 } 1288 1289 kfree(handles); 1290 *syncobjs_out = syncobjs; 1291 return 0; 1292 1293 err_put_syncobjs: 1294 while (i-- > 0) 1295 drm_syncobj_put(syncobjs[i]); 1296 kfree(syncobjs); 1297 err_free_handles: 1298 kfree(handles); 1299 1300 return ret; 1301 } 1302 1303 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs, 1304 uint32_t count) 1305 { 1306 uint32_t i; 1307 1308 for (i = 0; i < count; i++) 1309 drm_syncobj_put(syncobjs[i]); 1310 kfree(syncobjs); 1311 } 1312 1313 int 1314 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data, 1315 struct drm_file *file_private) 1316 { 1317 struct drm_syncobj_wait *args = data; 1318 struct drm_syncobj **syncobjs; 1319 unsigned int possible_flags; 1320 ktime_t t, *tp = NULL; 1321 int ret = 0; 1322 1323 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1324 return -EOPNOTSUPP; 1325 1326 possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL | 1327 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1328 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE; 1329 1330 if (args->flags & ~possible_flags) 1331 return -EINVAL; 1332 1333 if (args->count_handles == 0) 1334 return 0; 1335 1336 ret = drm_syncobj_array_find(file_private, 1337 u64_to_user_ptr(args->handles), 1338 args->count_handles, 1339 &syncobjs); 1340 if (ret < 0) 1341 return ret; 1342 1343 if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) { 1344 t = ns_to_ktime(args->deadline_nsec); 1345 tp = &t; 1346 } 1347 1348 ret = drm_syncobj_array_wait(dev, file_private, 1349 args, NULL, syncobjs, false, tp); 1350 1351 drm_syncobj_array_free(syncobjs, args->count_handles); 1352 1353 return ret; 1354 } 1355 1356 int 1357 drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data, 1358 struct drm_file *file_private) 1359 { 1360 struct drm_syncobj_timeline_wait *args = data; 1361 struct drm_syncobj **syncobjs; 1362 unsigned int possible_flags; 1363 ktime_t t, *tp = NULL; 1364 int ret = 0; 1365 1366 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1367 return -EOPNOTSUPP; 1368 1369 possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL | 1370 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1371 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE | 1372 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE; 1373 1374 if (args->flags & ~possible_flags) 1375 return -EINVAL; 1376 1377 if (args->count_handles == 0) 1378 return 0; 1379 1380 ret = drm_syncobj_array_find(file_private, 1381 u64_to_user_ptr(args->handles), 1382 args->count_handles, 1383 &syncobjs); 1384 if (ret < 0) 1385 return ret; 1386 1387 if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) { 1388 t = ns_to_ktime(args->deadline_nsec); 1389 tp = &t; 1390 } 1391 1392 ret = drm_syncobj_array_wait(dev, file_private, 1393 NULL, args, syncobjs, true, tp); 1394 1395 drm_syncobj_array_free(syncobjs, args->count_handles); 1396 1397 return ret; 1398 } 1399 1400 static void syncobj_eventfd_entry_fence_func(struct dma_fence *fence, 1401 struct dma_fence_cb *cb) 1402 { 1403 struct syncobj_eventfd_entry *entry = 1404 container_of(cb, struct syncobj_eventfd_entry, fence_cb); 1405 1406 eventfd_signal(entry->ev_fd_ctx, 1); 1407 syncobj_eventfd_entry_free(entry); 1408 } 1409 1410 static void 1411 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj, 1412 struct syncobj_eventfd_entry *entry) 1413 { 1414 int ret; 1415 struct dma_fence *fence; 1416 1417 /* This happens inside the syncobj lock */ 1418 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1)); 1419 ret = dma_fence_chain_find_seqno(&fence, entry->point); 1420 if (ret != 0 || !fence) { 1421 dma_fence_put(fence); 1422 return; 1423 } 1424 1425 list_del_init(&entry->node); 1426 entry->fence = fence; 1427 1428 if (entry->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) { 1429 eventfd_signal(entry->ev_fd_ctx, 1); 1430 syncobj_eventfd_entry_free(entry); 1431 } else { 1432 ret = dma_fence_add_callback(fence, &entry->fence_cb, 1433 syncobj_eventfd_entry_fence_func); 1434 if (ret == -ENOENT) { 1435 eventfd_signal(entry->ev_fd_ctx, 1); 1436 syncobj_eventfd_entry_free(entry); 1437 } 1438 } 1439 } 1440 1441 int 1442 drm_syncobj_eventfd_ioctl(struct drm_device *dev, void *data, 1443 struct drm_file *file_private) 1444 { 1445 struct drm_syncobj_eventfd *args = data; 1446 struct drm_syncobj *syncobj; 1447 struct eventfd_ctx *ev_fd_ctx; 1448 struct syncobj_eventfd_entry *entry; 1449 1450 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1451 return -EOPNOTSUPP; 1452 1453 if (args->flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) 1454 return -EINVAL; 1455 1456 if (args->pad) 1457 return -EINVAL; 1458 1459 syncobj = drm_syncobj_find(file_private, args->handle); 1460 if (!syncobj) 1461 return -ENOENT; 1462 1463 ev_fd_ctx = eventfd_ctx_fdget(args->fd); 1464 if (IS_ERR(ev_fd_ctx)) 1465 return PTR_ERR(ev_fd_ctx); 1466 1467 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 1468 if (!entry) { 1469 eventfd_ctx_put(ev_fd_ctx); 1470 return -ENOMEM; 1471 } 1472 entry->syncobj = syncobj; 1473 entry->ev_fd_ctx = ev_fd_ctx; 1474 entry->point = args->point; 1475 entry->flags = args->flags; 1476 1477 drm_syncobj_add_eventfd(syncobj, entry); 1478 drm_syncobj_put(syncobj); 1479 1480 return 0; 1481 } 1482 1483 int 1484 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data, 1485 struct drm_file *file_private) 1486 { 1487 struct drm_syncobj_array *args = data; 1488 struct drm_syncobj **syncobjs; 1489 uint32_t i; 1490 int ret; 1491 1492 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1493 return -EOPNOTSUPP; 1494 1495 if (args->pad != 0) 1496 return -EINVAL; 1497 1498 if (args->count_handles == 0) 1499 return -EINVAL; 1500 1501 ret = drm_syncobj_array_find(file_private, 1502 u64_to_user_ptr(args->handles), 1503 args->count_handles, 1504 &syncobjs); 1505 if (ret < 0) 1506 return ret; 1507 1508 for (i = 0; i < args->count_handles; i++) 1509 drm_syncobj_replace_fence(syncobjs[i], NULL); 1510 1511 drm_syncobj_array_free(syncobjs, args->count_handles); 1512 1513 return 0; 1514 } 1515 1516 int 1517 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data, 1518 struct drm_file *file_private) 1519 { 1520 struct drm_syncobj_array *args = data; 1521 struct drm_syncobj **syncobjs; 1522 uint32_t i; 1523 int ret; 1524 1525 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1526 return -EOPNOTSUPP; 1527 1528 if (args->pad != 0) 1529 return -EINVAL; 1530 1531 if (args->count_handles == 0) 1532 return -EINVAL; 1533 1534 ret = drm_syncobj_array_find(file_private, 1535 u64_to_user_ptr(args->handles), 1536 args->count_handles, 1537 &syncobjs); 1538 if (ret < 0) 1539 return ret; 1540 1541 for (i = 0; i < args->count_handles; i++) { 1542 ret = drm_syncobj_assign_null_handle(syncobjs[i]); 1543 if (ret < 0) 1544 break; 1545 } 1546 1547 drm_syncobj_array_free(syncobjs, args->count_handles); 1548 1549 return ret; 1550 } 1551 1552 int 1553 drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data, 1554 struct drm_file *file_private) 1555 { 1556 struct drm_syncobj_timeline_array *args = data; 1557 struct drm_syncobj **syncobjs; 1558 struct dma_fence_chain **chains; 1559 uint64_t *points; 1560 uint32_t i, j; 1561 int ret; 1562 1563 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1564 return -EOPNOTSUPP; 1565 1566 if (args->flags != 0) 1567 return -EINVAL; 1568 1569 if (args->count_handles == 0) 1570 return -EINVAL; 1571 1572 ret = drm_syncobj_array_find(file_private, 1573 u64_to_user_ptr(args->handles), 1574 args->count_handles, 1575 &syncobjs); 1576 if (ret < 0) 1577 return ret; 1578 1579 points = kmalloc_array(args->count_handles, sizeof(*points), 1580 GFP_KERNEL); 1581 if (!points) { 1582 ret = -ENOMEM; 1583 goto out; 1584 } 1585 if (!u64_to_user_ptr(args->points)) { 1586 memset(points, 0, args->count_handles * sizeof(uint64_t)); 1587 } else if (copy_from_user(points, u64_to_user_ptr(args->points), 1588 sizeof(uint64_t) * args->count_handles)) { 1589 ret = -EFAULT; 1590 goto err_points; 1591 } 1592 1593 chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL); 1594 if (!chains) { 1595 ret = -ENOMEM; 1596 goto err_points; 1597 } 1598 for (i = 0; i < args->count_handles; i++) { 1599 chains[i] = dma_fence_chain_alloc(); 1600 if (!chains[i]) { 1601 for (j = 0; j < i; j++) 1602 dma_fence_chain_free(chains[j]); 1603 ret = -ENOMEM; 1604 goto err_chains; 1605 } 1606 } 1607 1608 for (i = 0; i < args->count_handles; i++) { 1609 struct dma_fence *fence = dma_fence_get_stub(); 1610 1611 drm_syncobj_add_point(syncobjs[i], chains[i], 1612 fence, points[i]); 1613 dma_fence_put(fence); 1614 } 1615 err_chains: 1616 kfree(chains); 1617 err_points: 1618 kfree(points); 1619 out: 1620 drm_syncobj_array_free(syncobjs, args->count_handles); 1621 1622 return ret; 1623 } 1624 1625 int drm_syncobj_query_ioctl(struct drm_device *dev, void *data, 1626 struct drm_file *file_private) 1627 { 1628 struct drm_syncobj_timeline_array *args = data; 1629 struct drm_syncobj **syncobjs; 1630 uint64_t __user *points = u64_to_user_ptr(args->points); 1631 uint32_t i; 1632 int ret; 1633 1634 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1635 return -EOPNOTSUPP; 1636 1637 if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) 1638 return -EINVAL; 1639 1640 if (args->count_handles == 0) 1641 return -EINVAL; 1642 1643 ret = drm_syncobj_array_find(file_private, 1644 u64_to_user_ptr(args->handles), 1645 args->count_handles, 1646 &syncobjs); 1647 if (ret < 0) 1648 return ret; 1649 1650 for (i = 0; i < args->count_handles; i++) { 1651 struct dma_fence_chain *chain; 1652 struct dma_fence *fence; 1653 uint64_t point; 1654 1655 fence = drm_syncobj_fence_get(syncobjs[i]); 1656 chain = to_dma_fence_chain(fence); 1657 if (chain) { 1658 struct dma_fence *iter, *last_signaled = 1659 dma_fence_get(fence); 1660 1661 if (args->flags & 1662 DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) { 1663 point = fence->seqno; 1664 } else { 1665 dma_fence_chain_for_each(iter, fence) { 1666 if (iter->context != fence->context) { 1667 dma_fence_put(iter); 1668 /* It is most likely that timeline has 1669 * unorder points. */ 1670 break; 1671 } 1672 dma_fence_put(last_signaled); 1673 last_signaled = dma_fence_get(iter); 1674 } 1675 point = dma_fence_is_signaled(last_signaled) ? 1676 last_signaled->seqno : 1677 to_dma_fence_chain(last_signaled)->prev_seqno; 1678 } 1679 dma_fence_put(last_signaled); 1680 } else { 1681 point = 0; 1682 } 1683 dma_fence_put(fence); 1684 ret = copy_to_user(&points[i], &point, sizeof(uint64_t)); 1685 ret = ret ? -EFAULT : 0; 1686 if (ret) 1687 break; 1688 } 1689 drm_syncobj_array_free(syncobjs, args->count_handles); 1690 1691 return ret; 1692 } 1693