1 /* 2 * Copyright 2017 Red Hat 3 * Parts ported from amdgpu (fence wait code). 4 * Copyright 2016 Advanced Micro Devices, Inc. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 * IN THE SOFTWARE. 24 * 25 * Authors: 26 * 27 */ 28 29 /** 30 * DOC: Overview 31 * 32 * DRM synchronisation objects (syncobj, see struct &drm_syncobj) provide a 33 * container for a synchronization primitive which can be used by userspace 34 * to explicitly synchronize GPU commands, can be shared between userspace 35 * processes, and can be shared between different DRM drivers. 36 * Their primary use-case is to implement Vulkan fences and semaphores. 37 * The syncobj userspace API provides ioctls for several operations: 38 * 39 * - Creation and destruction of syncobjs 40 * - Import and export of syncobjs to/from a syncobj file descriptor 41 * - Import and export a syncobj's underlying fence to/from a sync file 42 * - Reset a syncobj (set its fence to NULL) 43 * - Signal a syncobj (set a trivially signaled fence) 44 * - Wait for a syncobj's fence to appear and be signaled 45 * 46 * The syncobj userspace API also provides operations to manipulate a syncobj 47 * in terms of a timeline of struct &dma_fence_chain rather than a single 48 * struct &dma_fence, through the following operations: 49 * 50 * - Signal a given point on the timeline 51 * - Wait for a given point to appear and/or be signaled 52 * - Import and export from/to a given point of a timeline 53 * 54 * At it's core, a syncobj is simply a wrapper around a pointer to a struct 55 * &dma_fence which may be NULL. 56 * When a syncobj is first created, its pointer is either NULL or a pointer 57 * to an already signaled fence depending on whether the 58 * &DRM_SYNCOBJ_CREATE_SIGNALED flag is passed to 59 * &DRM_IOCTL_SYNCOBJ_CREATE. 60 * 61 * If the syncobj is considered as a binary (its state is either signaled or 62 * unsignaled) primitive, when GPU work is enqueued in a DRM driver to signal 63 * the syncobj, the syncobj's fence is replaced with a fence which will be 64 * signaled by the completion of that work. 65 * If the syncobj is considered as a timeline primitive, when GPU work is 66 * enqueued in a DRM driver to signal the a given point of the syncobj, a new 67 * struct &dma_fence_chain pointing to the DRM driver's fence and also 68 * pointing to the previous fence that was in the syncobj. The new struct 69 * &dma_fence_chain fence replace the syncobj's fence and will be signaled by 70 * completion of the DRM driver's work and also any work associated with the 71 * fence previously in the syncobj. 72 * 73 * When GPU work which waits on a syncobj is enqueued in a DRM driver, at the 74 * time the work is enqueued, it waits on the syncobj's fence before 75 * submitting the work to hardware. That fence is either : 76 * 77 * - The syncobj's current fence if the syncobj is considered as a binary 78 * primitive. 79 * - The struct &dma_fence associated with a given point if the syncobj is 80 * considered as a timeline primitive. 81 * 82 * If the syncobj's fence is NULL or not present in the syncobj's timeline, 83 * the enqueue operation is expected to fail. 84 * 85 * With binary syncobj, all manipulation of the syncobjs's fence happens in 86 * terms of the current fence at the time the ioctl is called by userspace 87 * regardless of whether that operation is an immediate host-side operation 88 * (signal or reset) or or an operation which is enqueued in some driver 89 * queue. &DRM_IOCTL_SYNCOBJ_RESET and &DRM_IOCTL_SYNCOBJ_SIGNAL can be used 90 * to manipulate a syncobj from the host by resetting its pointer to NULL or 91 * setting its pointer to a fence which is already signaled. 92 * 93 * With a timeline syncobj, all manipulation of the synobj's fence happens in 94 * terms of a u64 value referring to point in the timeline. See 95 * dma_fence_chain_find_seqno() to see how a given point is found in the 96 * timeline. 97 * 98 * Note that applications should be careful to always use timeline set of 99 * ioctl() when dealing with syncobj considered as timeline. Using a binary 100 * set of ioctl() with a syncobj considered as timeline could result incorrect 101 * synchronization. The use of binary syncobj is supported through the 102 * timeline set of ioctl() by using a point value of 0, this will reproduce 103 * the behavior of the binary set of ioctl() (for example replace the 104 * syncobj's fence when signaling). 105 * 106 * 107 * Host-side wait on syncobjs 108 * -------------------------- 109 * 110 * &DRM_IOCTL_SYNCOBJ_WAIT takes an array of syncobj handles and does a 111 * host-side wait on all of the syncobj fences simultaneously. 112 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL is set, the wait ioctl will wait on 113 * all of the syncobj fences to be signaled before it returns. 114 * Otherwise, it returns once at least one syncobj fence has been signaled 115 * and the index of a signaled fence is written back to the client. 116 * 117 * Unlike the enqueued GPU work dependencies which fail if they see a NULL 118 * fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set, 119 * the host-side wait will first wait for the syncobj to receive a non-NULL 120 * fence and then wait on that fence. 121 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is not set and any one of the 122 * syncobjs in the array has a NULL fence, -EINVAL will be returned. 123 * Assuming the syncobj starts off with a NULL fence, this allows a client 124 * to do a host wait in one thread (or process) which waits on GPU work 125 * submitted in another thread (or process) without having to manually 126 * synchronize between the two. 127 * This requirement is inherited from the Vulkan fence API. 128 * 129 * Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj 130 * handles as well as an array of u64 points and does a host-side wait on all 131 * of syncobj fences at the given points simultaneously. 132 * 133 * &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT also adds the ability to wait for a given 134 * fence to materialize on the timeline without waiting for the fence to be 135 * signaled by using the &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE flag. This 136 * requirement is inherited from the wait-before-signal behavior required by 137 * the Vulkan timeline semaphore API. 138 * 139 * Alternatively, &DRM_IOCTL_SYNCOBJ_EVENTFD can be used to wait without 140 * blocking: an eventfd will be signaled when the syncobj is. This is useful to 141 * integrate the wait in an event loop. 142 * 143 * 144 * Import/export of syncobjs 145 * ------------------------- 146 * 147 * &DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE and &DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD 148 * provide two mechanisms for import/export of syncobjs. 149 * 150 * The first lets the client import or export an entire syncobj to a file 151 * descriptor. 152 * These fd's are opaque and have no other use case, except passing the 153 * syncobj between processes. 154 * All exported file descriptors and any syncobj handles created as a 155 * result of importing those file descriptors own a reference to the 156 * same underlying struct &drm_syncobj and the syncobj can be used 157 * persistently across all the processes with which it is shared. 158 * The syncobj is freed only once the last reference is dropped. 159 * Unlike dma-buf, importing a syncobj creates a new handle (with its own 160 * reference) for every import instead of de-duplicating. 161 * The primary use-case of this persistent import/export is for shared 162 * Vulkan fences and semaphores. 163 * 164 * The second import/export mechanism, which is indicated by 165 * &DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE or 166 * &DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE lets the client 167 * import/export the syncobj's current fence from/to a &sync_file. 168 * When a syncobj is exported to a sync file, that sync file wraps the 169 * sycnobj's fence at the time of export and any later signal or reset 170 * operations on the syncobj will not affect the exported sync file. 171 * When a sync file is imported into a syncobj, the syncobj's fence is set 172 * to the fence wrapped by that sync file. 173 * Because sync files are immutable, resetting or signaling the syncobj 174 * will not affect any sync files whose fences have been imported into the 175 * syncobj. 176 * 177 * 178 * Import/export of timeline points in timeline syncobjs 179 * ----------------------------------------------------- 180 * 181 * &DRM_IOCTL_SYNCOBJ_TRANSFER provides a mechanism to transfer a struct 182 * &dma_fence_chain of a syncobj at a given u64 point to another u64 point 183 * into another syncobj. 184 * 185 * Note that if you want to transfer a struct &dma_fence_chain from a given 186 * point on a timeline syncobj from/into a binary syncobj, you can use the 187 * point 0 to mean take/replace the fence in the syncobj. 188 */ 189 190 #include <linux/anon_inodes.h> 191 #include <linux/dma-fence-unwrap.h> 192 #include <linux/eventfd.h> 193 #include <linux/file.h> 194 #include <linux/fs.h> 195 #include <linux/sched/signal.h> 196 #include <linux/sync_file.h> 197 #include <linux/uaccess.h> 198 199 #include <drm/drm.h> 200 #include <drm/drm_drv.h> 201 #include <drm/drm_file.h> 202 #include <drm/drm_gem.h> 203 #include <drm/drm_print.h> 204 #include <drm/drm_syncobj.h> 205 #include <drm/drm_utils.h> 206 207 #include "drm_internal.h" 208 209 struct syncobj_wait_entry { 210 struct list_head node; 211 struct task_struct *task; 212 struct dma_fence *fence; 213 struct dma_fence_cb fence_cb; 214 u64 point; 215 }; 216 217 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj, 218 struct syncobj_wait_entry *wait); 219 220 struct syncobj_eventfd_entry { 221 struct list_head node; 222 struct dma_fence *fence; 223 struct dma_fence_cb fence_cb; 224 struct drm_syncobj *syncobj; 225 struct eventfd_ctx *ev_fd_ctx; 226 u64 point; 227 u32 flags; 228 }; 229 230 static void 231 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj, 232 struct syncobj_eventfd_entry *entry); 233 234 /** 235 * drm_syncobj_find - lookup and reference a sync object. 236 * @file_private: drm file private pointer 237 * @handle: sync object handle to lookup. 238 * 239 * Returns a reference to the syncobj pointed to by handle or NULL. The 240 * reference must be released by calling drm_syncobj_put(). 241 */ 242 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private, 243 u32 handle) 244 { 245 struct drm_syncobj *syncobj; 246 247 spin_lock(&file_private->syncobj_table_lock); 248 249 /* Check if we currently have a reference on the object */ 250 syncobj = idr_find(&file_private->syncobj_idr, handle); 251 if (syncobj) 252 drm_syncobj_get(syncobj); 253 254 spin_unlock(&file_private->syncobj_table_lock); 255 256 return syncobj; 257 } 258 EXPORT_SYMBOL(drm_syncobj_find); 259 260 static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj, 261 struct syncobj_wait_entry *wait) 262 { 263 struct dma_fence *fence; 264 265 if (wait->fence) 266 return; 267 268 spin_lock(&syncobj->lock); 269 /* We've already tried once to get a fence and failed. Now that we 270 * have the lock, try one more time just to be sure we don't add a 271 * callback when a fence has already been set. 272 */ 273 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1)); 274 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) { 275 dma_fence_put(fence); 276 list_add_tail(&wait->node, &syncobj->cb_list); 277 } else if (!fence) { 278 wait->fence = dma_fence_get_stub(); 279 } else { 280 wait->fence = fence; 281 } 282 spin_unlock(&syncobj->lock); 283 } 284 285 static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj, 286 struct syncobj_wait_entry *wait) 287 { 288 if (!wait->node.next) 289 return; 290 291 spin_lock(&syncobj->lock); 292 list_del_init(&wait->node); 293 spin_unlock(&syncobj->lock); 294 } 295 296 static void 297 syncobj_eventfd_entry_free(struct syncobj_eventfd_entry *entry) 298 { 299 eventfd_ctx_put(entry->ev_fd_ctx); 300 dma_fence_put(entry->fence); 301 /* This happens either inside the syncobj lock, or after the node has 302 * already been removed from the list. 303 */ 304 list_del(&entry->node); 305 kfree(entry); 306 } 307 308 static void 309 drm_syncobj_add_eventfd(struct drm_syncobj *syncobj, 310 struct syncobj_eventfd_entry *entry) 311 { 312 spin_lock(&syncobj->lock); 313 list_add_tail(&entry->node, &syncobj->ev_fd_list); 314 syncobj_eventfd_entry_func(syncobj, entry); 315 spin_unlock(&syncobj->lock); 316 } 317 318 /** 319 * drm_syncobj_add_point - add new timeline point to the syncobj 320 * @syncobj: sync object to add timeline point do 321 * @chain: chain node to use to add the point 322 * @fence: fence to encapsulate in the chain node 323 * @point: sequence number to use for the point 324 * 325 * Add the chain node as new timeline point to the syncobj. 326 */ 327 void drm_syncobj_add_point(struct drm_syncobj *syncobj, 328 struct dma_fence_chain *chain, 329 struct dma_fence *fence, 330 uint64_t point) 331 { 332 struct syncobj_wait_entry *wait_cur, *wait_tmp; 333 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 334 struct dma_fence *prev; 335 336 dma_fence_get(fence); 337 338 spin_lock(&syncobj->lock); 339 340 prev = drm_syncobj_fence_get(syncobj); 341 /* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */ 342 if (prev && prev->seqno >= point) 343 DRM_DEBUG("You are adding an unorder point to timeline!\n"); 344 dma_fence_chain_init(chain, prev, fence, point); 345 rcu_assign_pointer(syncobj->fence, &chain->base); 346 347 list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node) 348 syncobj_wait_syncobj_func(syncobj, wait_cur); 349 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 350 syncobj_eventfd_entry_func(syncobj, ev_fd_cur); 351 spin_unlock(&syncobj->lock); 352 353 /* Walk the chain once to trigger garbage collection */ 354 dma_fence_chain_for_each(fence, prev); 355 dma_fence_put(prev); 356 } 357 EXPORT_SYMBOL(drm_syncobj_add_point); 358 359 /** 360 * drm_syncobj_replace_fence - replace fence in a sync object. 361 * @syncobj: Sync object to replace fence in 362 * @fence: fence to install in sync file. 363 * 364 * This replaces the fence on a sync object. 365 */ 366 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj, 367 struct dma_fence *fence) 368 { 369 struct dma_fence *old_fence; 370 struct syncobj_wait_entry *wait_cur, *wait_tmp; 371 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 372 373 if (fence) 374 dma_fence_get(fence); 375 376 spin_lock(&syncobj->lock); 377 378 old_fence = rcu_dereference_protected(syncobj->fence, 379 lockdep_is_held(&syncobj->lock)); 380 rcu_assign_pointer(syncobj->fence, fence); 381 382 if (fence != old_fence) { 383 list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node) 384 syncobj_wait_syncobj_func(syncobj, wait_cur); 385 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 386 syncobj_eventfd_entry_func(syncobj, ev_fd_cur); 387 } 388 389 spin_unlock(&syncobj->lock); 390 391 dma_fence_put(old_fence); 392 } 393 EXPORT_SYMBOL(drm_syncobj_replace_fence); 394 395 /** 396 * drm_syncobj_assign_null_handle - assign a stub fence to the sync object 397 * @syncobj: sync object to assign the fence on 398 * 399 * Assign a already signaled stub fence to the sync object. 400 */ 401 static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj) 402 { 403 struct dma_fence *fence = dma_fence_allocate_private_stub(ktime_get()); 404 405 if (!fence) 406 return -ENOMEM; 407 408 drm_syncobj_replace_fence(syncobj, fence); 409 dma_fence_put(fence); 410 return 0; 411 } 412 413 /* 5s default for wait submission */ 414 #define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL 415 /** 416 * drm_syncobj_find_fence - lookup and reference the fence in a sync object 417 * @file_private: drm file private pointer 418 * @handle: sync object handle to lookup. 419 * @point: timeline point 420 * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not 421 * @fence: out parameter for the fence 422 * 423 * This is just a convenience function that combines drm_syncobj_find() and 424 * drm_syncobj_fence_get(). 425 * 426 * Returns 0 on success or a negative error value on failure. On success @fence 427 * contains a reference to the fence, which must be released by calling 428 * dma_fence_put(). 429 */ 430 int drm_syncobj_find_fence(struct drm_file *file_private, 431 u32 handle, u64 point, u64 flags, 432 struct dma_fence **fence) 433 { 434 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle); 435 struct syncobj_wait_entry wait; 436 u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT); 437 int ret; 438 439 if (!syncobj) 440 return -ENOENT; 441 442 /* Waiting for userspace with locks help is illegal cause that can 443 * trivial deadlock with page faults for example. Make lockdep complain 444 * about it early on. 445 */ 446 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) { 447 might_sleep(); 448 lockdep_assert_none_held_once(); 449 } 450 451 *fence = drm_syncobj_fence_get(syncobj); 452 453 if (*fence) { 454 ret = dma_fence_chain_find_seqno(fence, point); 455 if (!ret) { 456 /* If the requested seqno is already signaled 457 * drm_syncobj_find_fence may return a NULL 458 * fence. To make sure the recipient gets 459 * signalled, use a new fence instead. 460 */ 461 if (!*fence) 462 *fence = dma_fence_get_stub(); 463 464 goto out; 465 } 466 dma_fence_put(*fence); 467 } else { 468 ret = -EINVAL; 469 } 470 471 if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)) 472 goto out; 473 474 memset(&wait, 0, sizeof(wait)); 475 wait.task = current; 476 wait.point = point; 477 drm_syncobj_fence_add_wait(syncobj, &wait); 478 479 do { 480 set_current_state(TASK_INTERRUPTIBLE); 481 if (wait.fence) { 482 ret = 0; 483 break; 484 } 485 if (timeout == 0) { 486 ret = -ETIME; 487 break; 488 } 489 490 if (signal_pending(current)) { 491 ret = -ERESTARTSYS; 492 break; 493 } 494 495 timeout = schedule_timeout(timeout); 496 } while (1); 497 498 __set_current_state(TASK_RUNNING); 499 *fence = wait.fence; 500 501 if (wait.node.next) 502 drm_syncobj_remove_wait(syncobj, &wait); 503 504 out: 505 drm_syncobj_put(syncobj); 506 507 return ret; 508 } 509 EXPORT_SYMBOL(drm_syncobj_find_fence); 510 511 /** 512 * drm_syncobj_free - free a sync object. 513 * @kref: kref to free. 514 * 515 * Only to be called from kref_put in drm_syncobj_put. 516 */ 517 void drm_syncobj_free(struct kref *kref) 518 { 519 struct drm_syncobj *syncobj = container_of(kref, 520 struct drm_syncobj, 521 refcount); 522 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 523 524 drm_syncobj_replace_fence(syncobj, NULL); 525 526 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 527 syncobj_eventfd_entry_free(ev_fd_cur); 528 529 kfree(syncobj); 530 } 531 EXPORT_SYMBOL(drm_syncobj_free); 532 533 /** 534 * drm_syncobj_create - create a new syncobj 535 * @out_syncobj: returned syncobj 536 * @flags: DRM_SYNCOBJ_* flags 537 * @fence: if non-NULL, the syncobj will represent this fence 538 * 539 * This is the first function to create a sync object. After creating, drivers 540 * probably want to make it available to userspace, either through 541 * drm_syncobj_get_handle() or drm_syncobj_get_fd(). 542 * 543 * Returns 0 on success or a negative error value on failure. 544 */ 545 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags, 546 struct dma_fence *fence) 547 { 548 int ret; 549 struct drm_syncobj *syncobj; 550 551 syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL); 552 if (!syncobj) 553 return -ENOMEM; 554 555 kref_init(&syncobj->refcount); 556 INIT_LIST_HEAD(&syncobj->cb_list); 557 INIT_LIST_HEAD(&syncobj->ev_fd_list); 558 spin_lock_init(&syncobj->lock); 559 560 if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) { 561 ret = drm_syncobj_assign_null_handle(syncobj); 562 if (ret < 0) { 563 drm_syncobj_put(syncobj); 564 return ret; 565 } 566 } 567 568 if (fence) 569 drm_syncobj_replace_fence(syncobj, fence); 570 571 *out_syncobj = syncobj; 572 return 0; 573 } 574 EXPORT_SYMBOL(drm_syncobj_create); 575 576 /** 577 * drm_syncobj_get_handle - get a handle from a syncobj 578 * @file_private: drm file private pointer 579 * @syncobj: Sync object to export 580 * @handle: out parameter with the new handle 581 * 582 * Exports a sync object created with drm_syncobj_create() as a handle on 583 * @file_private to userspace. 584 * 585 * Returns 0 on success or a negative error value on failure. 586 */ 587 int drm_syncobj_get_handle(struct drm_file *file_private, 588 struct drm_syncobj *syncobj, u32 *handle) 589 { 590 int ret; 591 592 /* take a reference to put in the idr */ 593 drm_syncobj_get(syncobj); 594 595 idr_preload(GFP_KERNEL); 596 spin_lock(&file_private->syncobj_table_lock); 597 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT); 598 spin_unlock(&file_private->syncobj_table_lock); 599 600 idr_preload_end(); 601 602 if (ret < 0) { 603 drm_syncobj_put(syncobj); 604 return ret; 605 } 606 607 *handle = ret; 608 return 0; 609 } 610 EXPORT_SYMBOL(drm_syncobj_get_handle); 611 612 static int drm_syncobj_create_as_handle(struct drm_file *file_private, 613 u32 *handle, uint32_t flags) 614 { 615 int ret; 616 struct drm_syncobj *syncobj; 617 618 ret = drm_syncobj_create(&syncobj, flags, NULL); 619 if (ret) 620 return ret; 621 622 ret = drm_syncobj_get_handle(file_private, syncobj, handle); 623 drm_syncobj_put(syncobj); 624 return ret; 625 } 626 627 static int drm_syncobj_destroy(struct drm_file *file_private, 628 u32 handle) 629 { 630 struct drm_syncobj *syncobj; 631 632 spin_lock(&file_private->syncobj_table_lock); 633 syncobj = idr_remove(&file_private->syncobj_idr, handle); 634 spin_unlock(&file_private->syncobj_table_lock); 635 636 if (!syncobj) 637 return -EINVAL; 638 639 drm_syncobj_put(syncobj); 640 return 0; 641 } 642 643 static int drm_syncobj_file_release(struct inode *inode, struct file *file) 644 { 645 struct drm_syncobj *syncobj = file->private_data; 646 647 drm_syncobj_put(syncobj); 648 return 0; 649 } 650 651 static const struct file_operations drm_syncobj_file_fops = { 652 .release = drm_syncobj_file_release, 653 }; 654 655 /** 656 * drm_syncobj_get_fd - get a file descriptor from a syncobj 657 * @syncobj: Sync object to export 658 * @p_fd: out parameter with the new file descriptor 659 * 660 * Exports a sync object created with drm_syncobj_create() as a file descriptor. 661 * 662 * Returns 0 on success or a negative error value on failure. 663 */ 664 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd) 665 { 666 struct file *file; 667 int fd; 668 669 fd = get_unused_fd_flags(O_CLOEXEC); 670 if (fd < 0) 671 return fd; 672 673 file = anon_inode_getfile("syncobj_file", 674 &drm_syncobj_file_fops, 675 syncobj, 0); 676 if (IS_ERR(file)) { 677 put_unused_fd(fd); 678 return PTR_ERR(file); 679 } 680 681 drm_syncobj_get(syncobj); 682 fd_install(fd, file); 683 684 *p_fd = fd; 685 return 0; 686 } 687 EXPORT_SYMBOL(drm_syncobj_get_fd); 688 689 static int drm_syncobj_handle_to_fd(struct drm_file *file_private, 690 u32 handle, int *p_fd) 691 { 692 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle); 693 int ret; 694 695 if (!syncobj) 696 return -EINVAL; 697 698 ret = drm_syncobj_get_fd(syncobj, p_fd); 699 drm_syncobj_put(syncobj); 700 return ret; 701 } 702 703 static int drm_syncobj_fd_to_handle(struct drm_file *file_private, 704 int fd, u32 *handle) 705 { 706 struct drm_syncobj *syncobj; 707 struct fd f = fdget(fd); 708 int ret; 709 710 if (!f.file) 711 return -EINVAL; 712 713 if (f.file->f_op != &drm_syncobj_file_fops) { 714 fdput(f); 715 return -EINVAL; 716 } 717 718 /* take a reference to put in the idr */ 719 syncobj = f.file->private_data; 720 drm_syncobj_get(syncobj); 721 722 idr_preload(GFP_KERNEL); 723 spin_lock(&file_private->syncobj_table_lock); 724 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT); 725 spin_unlock(&file_private->syncobj_table_lock); 726 idr_preload_end(); 727 728 if (ret > 0) { 729 *handle = ret; 730 ret = 0; 731 } else 732 drm_syncobj_put(syncobj); 733 734 fdput(f); 735 return ret; 736 } 737 738 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private, 739 int fd, int handle) 740 { 741 struct dma_fence *fence = sync_file_get_fence(fd); 742 struct drm_syncobj *syncobj; 743 744 if (!fence) 745 return -EINVAL; 746 747 syncobj = drm_syncobj_find(file_private, handle); 748 if (!syncobj) { 749 dma_fence_put(fence); 750 return -ENOENT; 751 } 752 753 drm_syncobj_replace_fence(syncobj, fence); 754 dma_fence_put(fence); 755 drm_syncobj_put(syncobj); 756 return 0; 757 } 758 759 static int drm_syncobj_export_sync_file(struct drm_file *file_private, 760 int handle, int *p_fd) 761 { 762 int ret; 763 struct dma_fence *fence; 764 struct sync_file *sync_file; 765 int fd = get_unused_fd_flags(O_CLOEXEC); 766 767 if (fd < 0) 768 return fd; 769 770 ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence); 771 if (ret) 772 goto err_put_fd; 773 774 sync_file = sync_file_create(fence); 775 776 dma_fence_put(fence); 777 778 if (!sync_file) { 779 ret = -EINVAL; 780 goto err_put_fd; 781 } 782 783 fd_install(fd, sync_file->file); 784 785 *p_fd = fd; 786 return 0; 787 err_put_fd: 788 put_unused_fd(fd); 789 return ret; 790 } 791 /** 792 * drm_syncobj_open - initializes syncobj file-private structures at devnode open time 793 * @file_private: drm file-private structure to set up 794 * 795 * Called at device open time, sets up the structure for handling refcounting 796 * of sync objects. 797 */ 798 void 799 drm_syncobj_open(struct drm_file *file_private) 800 { 801 idr_init_base(&file_private->syncobj_idr, 1); 802 spin_lock_init(&file_private->syncobj_table_lock); 803 } 804 805 static int 806 drm_syncobj_release_handle(int id, void *ptr, void *data) 807 { 808 struct drm_syncobj *syncobj = ptr; 809 810 drm_syncobj_put(syncobj); 811 return 0; 812 } 813 814 /** 815 * drm_syncobj_release - release file-private sync object resources 816 * @file_private: drm file-private structure to clean up 817 * 818 * Called at close time when the filp is going away. 819 * 820 * Releases any remaining references on objects by this filp. 821 */ 822 void 823 drm_syncobj_release(struct drm_file *file_private) 824 { 825 idr_for_each(&file_private->syncobj_idr, 826 &drm_syncobj_release_handle, file_private); 827 idr_destroy(&file_private->syncobj_idr); 828 } 829 830 int 831 drm_syncobj_create_ioctl(struct drm_device *dev, void *data, 832 struct drm_file *file_private) 833 { 834 struct drm_syncobj_create *args = data; 835 836 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 837 return -EOPNOTSUPP; 838 839 /* no valid flags yet */ 840 if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED) 841 return -EINVAL; 842 843 return drm_syncobj_create_as_handle(file_private, 844 &args->handle, args->flags); 845 } 846 847 int 848 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data, 849 struct drm_file *file_private) 850 { 851 struct drm_syncobj_destroy *args = data; 852 853 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 854 return -EOPNOTSUPP; 855 856 /* make sure padding is empty */ 857 if (args->pad) 858 return -EINVAL; 859 return drm_syncobj_destroy(file_private, args->handle); 860 } 861 862 int 863 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data, 864 struct drm_file *file_private) 865 { 866 struct drm_syncobj_handle *args = data; 867 868 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 869 return -EOPNOTSUPP; 870 871 if (args->pad) 872 return -EINVAL; 873 874 if (args->flags != 0 && 875 args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE) 876 return -EINVAL; 877 878 if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE) 879 return drm_syncobj_export_sync_file(file_private, args->handle, 880 &args->fd); 881 882 return drm_syncobj_handle_to_fd(file_private, args->handle, 883 &args->fd); 884 } 885 886 int 887 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data, 888 struct drm_file *file_private) 889 { 890 struct drm_syncobj_handle *args = data; 891 892 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 893 return -EOPNOTSUPP; 894 895 if (args->pad) 896 return -EINVAL; 897 898 if (args->flags != 0 && 899 args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE) 900 return -EINVAL; 901 902 if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE) 903 return drm_syncobj_import_sync_file_fence(file_private, 904 args->fd, 905 args->handle); 906 907 return drm_syncobj_fd_to_handle(file_private, args->fd, 908 &args->handle); 909 } 910 911 static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private, 912 struct drm_syncobj_transfer *args) 913 { 914 struct drm_syncobj *timeline_syncobj = NULL; 915 struct dma_fence *fence, *tmp; 916 struct dma_fence_chain *chain; 917 int ret; 918 919 timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle); 920 if (!timeline_syncobj) { 921 return -ENOENT; 922 } 923 ret = drm_syncobj_find_fence(file_private, args->src_handle, 924 args->src_point, args->flags, 925 &tmp); 926 if (ret) 927 goto err_put_timeline; 928 929 fence = dma_fence_unwrap_merge(tmp); 930 dma_fence_put(tmp); 931 if (!fence) { 932 ret = -ENOMEM; 933 goto err_put_timeline; 934 } 935 936 chain = dma_fence_chain_alloc(); 937 if (!chain) { 938 ret = -ENOMEM; 939 goto err_free_fence; 940 } 941 942 drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point); 943 err_free_fence: 944 dma_fence_put(fence); 945 err_put_timeline: 946 drm_syncobj_put(timeline_syncobj); 947 948 return ret; 949 } 950 951 static int 952 drm_syncobj_transfer_to_binary(struct drm_file *file_private, 953 struct drm_syncobj_transfer *args) 954 { 955 struct drm_syncobj *binary_syncobj = NULL; 956 struct dma_fence *fence; 957 int ret; 958 959 binary_syncobj = drm_syncobj_find(file_private, args->dst_handle); 960 if (!binary_syncobj) 961 return -ENOENT; 962 ret = drm_syncobj_find_fence(file_private, args->src_handle, 963 args->src_point, args->flags, &fence); 964 if (ret) 965 goto err; 966 drm_syncobj_replace_fence(binary_syncobj, fence); 967 dma_fence_put(fence); 968 err: 969 drm_syncobj_put(binary_syncobj); 970 971 return ret; 972 } 973 int 974 drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data, 975 struct drm_file *file_private) 976 { 977 struct drm_syncobj_transfer *args = data; 978 int ret; 979 980 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 981 return -EOPNOTSUPP; 982 983 if (args->pad) 984 return -EINVAL; 985 986 if (args->dst_point) 987 ret = drm_syncobj_transfer_to_timeline(file_private, args); 988 else 989 ret = drm_syncobj_transfer_to_binary(file_private, args); 990 991 return ret; 992 } 993 994 static void syncobj_wait_fence_func(struct dma_fence *fence, 995 struct dma_fence_cb *cb) 996 { 997 struct syncobj_wait_entry *wait = 998 container_of(cb, struct syncobj_wait_entry, fence_cb); 999 1000 wake_up_process(wait->task); 1001 } 1002 1003 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj, 1004 struct syncobj_wait_entry *wait) 1005 { 1006 struct dma_fence *fence; 1007 1008 /* This happens inside the syncobj lock */ 1009 fence = rcu_dereference_protected(syncobj->fence, 1010 lockdep_is_held(&syncobj->lock)); 1011 dma_fence_get(fence); 1012 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) { 1013 dma_fence_put(fence); 1014 return; 1015 } else if (!fence) { 1016 wait->fence = dma_fence_get_stub(); 1017 } else { 1018 wait->fence = fence; 1019 } 1020 1021 wake_up_process(wait->task); 1022 list_del_init(&wait->node); 1023 } 1024 1025 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs, 1026 void __user *user_points, 1027 uint32_t count, 1028 uint32_t flags, 1029 signed long timeout, 1030 uint32_t *idx) 1031 { 1032 struct syncobj_wait_entry *entries; 1033 struct dma_fence *fence; 1034 uint64_t *points; 1035 uint32_t signaled_count, i; 1036 1037 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) 1038 lockdep_assert_none_held_once(); 1039 1040 points = kmalloc_array(count, sizeof(*points), GFP_KERNEL); 1041 if (points == NULL) 1042 return -ENOMEM; 1043 1044 if (!user_points) { 1045 memset(points, 0, count * sizeof(uint64_t)); 1046 1047 } else if (copy_from_user(points, user_points, 1048 sizeof(uint64_t) * count)) { 1049 timeout = -EFAULT; 1050 goto err_free_points; 1051 } 1052 1053 entries = kcalloc(count, sizeof(*entries), GFP_KERNEL); 1054 if (!entries) { 1055 timeout = -ENOMEM; 1056 goto err_free_points; 1057 } 1058 /* Walk the list of sync objects and initialize entries. We do 1059 * this up-front so that we can properly return -EINVAL if there is 1060 * a syncobj with a missing fence and then never have the chance of 1061 * returning -EINVAL again. 1062 */ 1063 signaled_count = 0; 1064 for (i = 0; i < count; ++i) { 1065 struct dma_fence *fence; 1066 1067 entries[i].task = current; 1068 entries[i].point = points[i]; 1069 fence = drm_syncobj_fence_get(syncobjs[i]); 1070 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) { 1071 dma_fence_put(fence); 1072 if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1073 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) { 1074 continue; 1075 } else { 1076 timeout = -EINVAL; 1077 goto cleanup_entries; 1078 } 1079 } 1080 1081 if (fence) 1082 entries[i].fence = fence; 1083 else 1084 entries[i].fence = dma_fence_get_stub(); 1085 1086 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) || 1087 dma_fence_is_signaled(entries[i].fence)) { 1088 if (signaled_count == 0 && idx) 1089 *idx = i; 1090 signaled_count++; 1091 } 1092 } 1093 1094 if (signaled_count == count || 1095 (signaled_count > 0 && 1096 !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL))) 1097 goto cleanup_entries; 1098 1099 /* There's a very annoying laxness in the dma_fence API here, in 1100 * that backends are not required to automatically report when a 1101 * fence is signaled prior to fence->ops->enable_signaling() being 1102 * called. So here if we fail to match signaled_count, we need to 1103 * fallthough and try a 0 timeout wait! 1104 */ 1105 1106 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) { 1107 for (i = 0; i < count; ++i) 1108 drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]); 1109 } 1110 1111 do { 1112 set_current_state(TASK_INTERRUPTIBLE); 1113 1114 signaled_count = 0; 1115 for (i = 0; i < count; ++i) { 1116 fence = entries[i].fence; 1117 if (!fence) 1118 continue; 1119 1120 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) || 1121 dma_fence_is_signaled(fence) || 1122 (!entries[i].fence_cb.func && 1123 dma_fence_add_callback(fence, 1124 &entries[i].fence_cb, 1125 syncobj_wait_fence_func))) { 1126 /* The fence has been signaled */ 1127 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) { 1128 signaled_count++; 1129 } else { 1130 if (idx) 1131 *idx = i; 1132 goto done_waiting; 1133 } 1134 } 1135 } 1136 1137 if (signaled_count == count) 1138 goto done_waiting; 1139 1140 if (timeout == 0) { 1141 timeout = -ETIME; 1142 goto done_waiting; 1143 } 1144 1145 if (signal_pending(current)) { 1146 timeout = -ERESTARTSYS; 1147 goto done_waiting; 1148 } 1149 1150 timeout = schedule_timeout(timeout); 1151 } while (1); 1152 1153 done_waiting: 1154 __set_current_state(TASK_RUNNING); 1155 1156 cleanup_entries: 1157 for (i = 0; i < count; ++i) { 1158 drm_syncobj_remove_wait(syncobjs[i], &entries[i]); 1159 if (entries[i].fence_cb.func) 1160 dma_fence_remove_callback(entries[i].fence, 1161 &entries[i].fence_cb); 1162 dma_fence_put(entries[i].fence); 1163 } 1164 kfree(entries); 1165 1166 err_free_points: 1167 kfree(points); 1168 1169 return timeout; 1170 } 1171 1172 /** 1173 * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value 1174 * 1175 * @timeout_nsec: timeout nsec component in ns, 0 for poll 1176 * 1177 * Calculate the timeout in jiffies from an absolute time in sec/nsec. 1178 */ 1179 signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec) 1180 { 1181 ktime_t abs_timeout, now; 1182 u64 timeout_ns, timeout_jiffies64; 1183 1184 /* make 0 timeout means poll - absolute 0 doesn't seem valid */ 1185 if (timeout_nsec == 0) 1186 return 0; 1187 1188 abs_timeout = ns_to_ktime(timeout_nsec); 1189 now = ktime_get(); 1190 1191 if (!ktime_after(abs_timeout, now)) 1192 return 0; 1193 1194 timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now)); 1195 1196 timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns); 1197 /* clamp timeout to avoid infinite timeout */ 1198 if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1) 1199 return MAX_SCHEDULE_TIMEOUT - 1; 1200 1201 return timeout_jiffies64 + 1; 1202 } 1203 EXPORT_SYMBOL(drm_timeout_abs_to_jiffies); 1204 1205 static int drm_syncobj_array_wait(struct drm_device *dev, 1206 struct drm_file *file_private, 1207 struct drm_syncobj_wait *wait, 1208 struct drm_syncobj_timeline_wait *timeline_wait, 1209 struct drm_syncobj **syncobjs, bool timeline) 1210 { 1211 signed long timeout = 0; 1212 uint32_t first = ~0; 1213 1214 if (!timeline) { 1215 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec); 1216 timeout = drm_syncobj_array_wait_timeout(syncobjs, 1217 NULL, 1218 wait->count_handles, 1219 wait->flags, 1220 timeout, &first); 1221 if (timeout < 0) 1222 return timeout; 1223 wait->first_signaled = first; 1224 } else { 1225 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec); 1226 timeout = drm_syncobj_array_wait_timeout(syncobjs, 1227 u64_to_user_ptr(timeline_wait->points), 1228 timeline_wait->count_handles, 1229 timeline_wait->flags, 1230 timeout, &first); 1231 if (timeout < 0) 1232 return timeout; 1233 timeline_wait->first_signaled = first; 1234 } 1235 return 0; 1236 } 1237 1238 static int drm_syncobj_array_find(struct drm_file *file_private, 1239 void __user *user_handles, 1240 uint32_t count_handles, 1241 struct drm_syncobj ***syncobjs_out) 1242 { 1243 uint32_t i, *handles; 1244 struct drm_syncobj **syncobjs; 1245 int ret; 1246 1247 handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL); 1248 if (handles == NULL) 1249 return -ENOMEM; 1250 1251 if (copy_from_user(handles, user_handles, 1252 sizeof(uint32_t) * count_handles)) { 1253 ret = -EFAULT; 1254 goto err_free_handles; 1255 } 1256 1257 syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL); 1258 if (syncobjs == NULL) { 1259 ret = -ENOMEM; 1260 goto err_free_handles; 1261 } 1262 1263 for (i = 0; i < count_handles; i++) { 1264 syncobjs[i] = drm_syncobj_find(file_private, handles[i]); 1265 if (!syncobjs[i]) { 1266 ret = -ENOENT; 1267 goto err_put_syncobjs; 1268 } 1269 } 1270 1271 kfree(handles); 1272 *syncobjs_out = syncobjs; 1273 return 0; 1274 1275 err_put_syncobjs: 1276 while (i-- > 0) 1277 drm_syncobj_put(syncobjs[i]); 1278 kfree(syncobjs); 1279 err_free_handles: 1280 kfree(handles); 1281 1282 return ret; 1283 } 1284 1285 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs, 1286 uint32_t count) 1287 { 1288 uint32_t i; 1289 1290 for (i = 0; i < count; i++) 1291 drm_syncobj_put(syncobjs[i]); 1292 kfree(syncobjs); 1293 } 1294 1295 int 1296 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data, 1297 struct drm_file *file_private) 1298 { 1299 struct drm_syncobj_wait *args = data; 1300 struct drm_syncobj **syncobjs; 1301 int ret = 0; 1302 1303 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1304 return -EOPNOTSUPP; 1305 1306 if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL | 1307 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)) 1308 return -EINVAL; 1309 1310 if (args->count_handles == 0) 1311 return -EINVAL; 1312 1313 ret = drm_syncobj_array_find(file_private, 1314 u64_to_user_ptr(args->handles), 1315 args->count_handles, 1316 &syncobjs); 1317 if (ret < 0) 1318 return ret; 1319 1320 ret = drm_syncobj_array_wait(dev, file_private, 1321 args, NULL, syncobjs, false); 1322 1323 drm_syncobj_array_free(syncobjs, args->count_handles); 1324 1325 return ret; 1326 } 1327 1328 int 1329 drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data, 1330 struct drm_file *file_private) 1331 { 1332 struct drm_syncobj_timeline_wait *args = data; 1333 struct drm_syncobj **syncobjs; 1334 int ret = 0; 1335 1336 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1337 return -EOPNOTSUPP; 1338 1339 if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL | 1340 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1341 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) 1342 return -EINVAL; 1343 1344 if (args->count_handles == 0) 1345 return -EINVAL; 1346 1347 ret = drm_syncobj_array_find(file_private, 1348 u64_to_user_ptr(args->handles), 1349 args->count_handles, 1350 &syncobjs); 1351 if (ret < 0) 1352 return ret; 1353 1354 ret = drm_syncobj_array_wait(dev, file_private, 1355 NULL, args, syncobjs, true); 1356 1357 drm_syncobj_array_free(syncobjs, args->count_handles); 1358 1359 return ret; 1360 } 1361 1362 static void syncobj_eventfd_entry_fence_func(struct dma_fence *fence, 1363 struct dma_fence_cb *cb) 1364 { 1365 struct syncobj_eventfd_entry *entry = 1366 container_of(cb, struct syncobj_eventfd_entry, fence_cb); 1367 1368 eventfd_signal(entry->ev_fd_ctx, 1); 1369 syncobj_eventfd_entry_free(entry); 1370 } 1371 1372 static void 1373 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj, 1374 struct syncobj_eventfd_entry *entry) 1375 { 1376 int ret; 1377 struct dma_fence *fence; 1378 1379 /* This happens inside the syncobj lock */ 1380 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1)); 1381 ret = dma_fence_chain_find_seqno(&fence, entry->point); 1382 if (ret != 0 || !fence) { 1383 dma_fence_put(fence); 1384 return; 1385 } 1386 1387 list_del_init(&entry->node); 1388 entry->fence = fence; 1389 1390 if (entry->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) { 1391 eventfd_signal(entry->ev_fd_ctx, 1); 1392 syncobj_eventfd_entry_free(entry); 1393 } else { 1394 ret = dma_fence_add_callback(fence, &entry->fence_cb, 1395 syncobj_eventfd_entry_fence_func); 1396 if (ret == -ENOENT) { 1397 eventfd_signal(entry->ev_fd_ctx, 1); 1398 syncobj_eventfd_entry_free(entry); 1399 } 1400 } 1401 } 1402 1403 int 1404 drm_syncobj_eventfd_ioctl(struct drm_device *dev, void *data, 1405 struct drm_file *file_private) 1406 { 1407 struct drm_syncobj_eventfd *args = data; 1408 struct drm_syncobj *syncobj; 1409 struct eventfd_ctx *ev_fd_ctx; 1410 struct syncobj_eventfd_entry *entry; 1411 1412 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1413 return -EOPNOTSUPP; 1414 1415 if (args->flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) 1416 return -EINVAL; 1417 1418 if (args->pad) 1419 return -EINVAL; 1420 1421 syncobj = drm_syncobj_find(file_private, args->handle); 1422 if (!syncobj) 1423 return -ENOENT; 1424 1425 ev_fd_ctx = eventfd_ctx_fdget(args->fd); 1426 if (IS_ERR(ev_fd_ctx)) 1427 return PTR_ERR(ev_fd_ctx); 1428 1429 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 1430 if (!entry) { 1431 eventfd_ctx_put(ev_fd_ctx); 1432 return -ENOMEM; 1433 } 1434 entry->syncobj = syncobj; 1435 entry->ev_fd_ctx = ev_fd_ctx; 1436 entry->point = args->point; 1437 entry->flags = args->flags; 1438 1439 drm_syncobj_add_eventfd(syncobj, entry); 1440 drm_syncobj_put(syncobj); 1441 1442 return 0; 1443 } 1444 1445 int 1446 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data, 1447 struct drm_file *file_private) 1448 { 1449 struct drm_syncobj_array *args = data; 1450 struct drm_syncobj **syncobjs; 1451 uint32_t i; 1452 int ret; 1453 1454 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1455 return -EOPNOTSUPP; 1456 1457 if (args->pad != 0) 1458 return -EINVAL; 1459 1460 if (args->count_handles == 0) 1461 return -EINVAL; 1462 1463 ret = drm_syncobj_array_find(file_private, 1464 u64_to_user_ptr(args->handles), 1465 args->count_handles, 1466 &syncobjs); 1467 if (ret < 0) 1468 return ret; 1469 1470 for (i = 0; i < args->count_handles; i++) 1471 drm_syncobj_replace_fence(syncobjs[i], NULL); 1472 1473 drm_syncobj_array_free(syncobjs, args->count_handles); 1474 1475 return 0; 1476 } 1477 1478 int 1479 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data, 1480 struct drm_file *file_private) 1481 { 1482 struct drm_syncobj_array *args = data; 1483 struct drm_syncobj **syncobjs; 1484 uint32_t i; 1485 int ret; 1486 1487 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1488 return -EOPNOTSUPP; 1489 1490 if (args->pad != 0) 1491 return -EINVAL; 1492 1493 if (args->count_handles == 0) 1494 return -EINVAL; 1495 1496 ret = drm_syncobj_array_find(file_private, 1497 u64_to_user_ptr(args->handles), 1498 args->count_handles, 1499 &syncobjs); 1500 if (ret < 0) 1501 return ret; 1502 1503 for (i = 0; i < args->count_handles; i++) { 1504 ret = drm_syncobj_assign_null_handle(syncobjs[i]); 1505 if (ret < 0) 1506 break; 1507 } 1508 1509 drm_syncobj_array_free(syncobjs, args->count_handles); 1510 1511 return ret; 1512 } 1513 1514 int 1515 drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data, 1516 struct drm_file *file_private) 1517 { 1518 struct drm_syncobj_timeline_array *args = data; 1519 struct drm_syncobj **syncobjs; 1520 struct dma_fence_chain **chains; 1521 uint64_t *points; 1522 uint32_t i, j; 1523 int ret; 1524 1525 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1526 return -EOPNOTSUPP; 1527 1528 if (args->flags != 0) 1529 return -EINVAL; 1530 1531 if (args->count_handles == 0) 1532 return -EINVAL; 1533 1534 ret = drm_syncobj_array_find(file_private, 1535 u64_to_user_ptr(args->handles), 1536 args->count_handles, 1537 &syncobjs); 1538 if (ret < 0) 1539 return ret; 1540 1541 points = kmalloc_array(args->count_handles, sizeof(*points), 1542 GFP_KERNEL); 1543 if (!points) { 1544 ret = -ENOMEM; 1545 goto out; 1546 } 1547 if (!u64_to_user_ptr(args->points)) { 1548 memset(points, 0, args->count_handles * sizeof(uint64_t)); 1549 } else if (copy_from_user(points, u64_to_user_ptr(args->points), 1550 sizeof(uint64_t) * args->count_handles)) { 1551 ret = -EFAULT; 1552 goto err_points; 1553 } 1554 1555 chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL); 1556 if (!chains) { 1557 ret = -ENOMEM; 1558 goto err_points; 1559 } 1560 for (i = 0; i < args->count_handles; i++) { 1561 chains[i] = dma_fence_chain_alloc(); 1562 if (!chains[i]) { 1563 for (j = 0; j < i; j++) 1564 dma_fence_chain_free(chains[j]); 1565 ret = -ENOMEM; 1566 goto err_chains; 1567 } 1568 } 1569 1570 for (i = 0; i < args->count_handles; i++) { 1571 struct dma_fence *fence = dma_fence_get_stub(); 1572 1573 drm_syncobj_add_point(syncobjs[i], chains[i], 1574 fence, points[i]); 1575 dma_fence_put(fence); 1576 } 1577 err_chains: 1578 kfree(chains); 1579 err_points: 1580 kfree(points); 1581 out: 1582 drm_syncobj_array_free(syncobjs, args->count_handles); 1583 1584 return ret; 1585 } 1586 1587 int drm_syncobj_query_ioctl(struct drm_device *dev, void *data, 1588 struct drm_file *file_private) 1589 { 1590 struct drm_syncobj_timeline_array *args = data; 1591 struct drm_syncobj **syncobjs; 1592 uint64_t __user *points = u64_to_user_ptr(args->points); 1593 uint32_t i; 1594 int ret; 1595 1596 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1597 return -EOPNOTSUPP; 1598 1599 if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) 1600 return -EINVAL; 1601 1602 if (args->count_handles == 0) 1603 return -EINVAL; 1604 1605 ret = drm_syncobj_array_find(file_private, 1606 u64_to_user_ptr(args->handles), 1607 args->count_handles, 1608 &syncobjs); 1609 if (ret < 0) 1610 return ret; 1611 1612 for (i = 0; i < args->count_handles; i++) { 1613 struct dma_fence_chain *chain; 1614 struct dma_fence *fence; 1615 uint64_t point; 1616 1617 fence = drm_syncobj_fence_get(syncobjs[i]); 1618 chain = to_dma_fence_chain(fence); 1619 if (chain) { 1620 struct dma_fence *iter, *last_signaled = 1621 dma_fence_get(fence); 1622 1623 if (args->flags & 1624 DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) { 1625 point = fence->seqno; 1626 } else { 1627 dma_fence_chain_for_each(iter, fence) { 1628 if (iter->context != fence->context) { 1629 dma_fence_put(iter); 1630 /* It is most likely that timeline has 1631 * unorder points. */ 1632 break; 1633 } 1634 dma_fence_put(last_signaled); 1635 last_signaled = dma_fence_get(iter); 1636 } 1637 point = dma_fence_is_signaled(last_signaled) ? 1638 last_signaled->seqno : 1639 to_dma_fence_chain(last_signaled)->prev_seqno; 1640 } 1641 dma_fence_put(last_signaled); 1642 } else { 1643 point = 0; 1644 } 1645 dma_fence_put(fence); 1646 ret = copy_to_user(&points[i], &point, sizeof(uint64_t)); 1647 ret = ret ? -EFAULT : 0; 1648 if (ret) 1649 break; 1650 } 1651 drm_syncobj_array_free(syncobjs, args->count_handles); 1652 1653 return ret; 1654 } 1655