1 /* 2 * Copyright 2017 Red Hat 3 * Parts ported from amdgpu (fence wait code). 4 * Copyright 2016 Advanced Micro Devices, Inc. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 * IN THE SOFTWARE. 24 * 25 * Authors: 26 * 27 */ 28 29 /** 30 * DOC: Overview 31 * 32 * DRM synchronisation objects (syncobj, see struct &drm_syncobj) provide a 33 * container for a synchronization primitive which can be used by userspace 34 * to explicitly synchronize GPU commands, can be shared between userspace 35 * processes, and can be shared between different DRM drivers. 36 * Their primary use-case is to implement Vulkan fences and semaphores. 37 * The syncobj userspace API provides ioctls for several operations: 38 * 39 * - Creation and destruction of syncobjs 40 * - Import and export of syncobjs to/from a syncobj file descriptor 41 * - Import and export a syncobj's underlying fence to/from a sync file 42 * - Reset a syncobj (set its fence to NULL) 43 * - Signal a syncobj (set a trivially signaled fence) 44 * - Wait for a syncobj's fence to appear and be signaled 45 * 46 * The syncobj userspace API also provides operations to manipulate a syncobj 47 * in terms of a timeline of struct &dma_fence_chain rather than a single 48 * struct &dma_fence, through the following operations: 49 * 50 * - Signal a given point on the timeline 51 * - Wait for a given point to appear and/or be signaled 52 * - Import and export from/to a given point of a timeline 53 * 54 * At it's core, a syncobj is simply a wrapper around a pointer to a struct 55 * &dma_fence which may be NULL. 56 * When a syncobj is first created, its pointer is either NULL or a pointer 57 * to an already signaled fence depending on whether the 58 * &DRM_SYNCOBJ_CREATE_SIGNALED flag is passed to 59 * &DRM_IOCTL_SYNCOBJ_CREATE. 60 * 61 * If the syncobj is considered as a binary (its state is either signaled or 62 * unsignaled) primitive, when GPU work is enqueued in a DRM driver to signal 63 * the syncobj, the syncobj's fence is replaced with a fence which will be 64 * signaled by the completion of that work. 65 * If the syncobj is considered as a timeline primitive, when GPU work is 66 * enqueued in a DRM driver to signal the a given point of the syncobj, a new 67 * struct &dma_fence_chain pointing to the DRM driver's fence and also 68 * pointing to the previous fence that was in the syncobj. The new struct 69 * &dma_fence_chain fence replace the syncobj's fence and will be signaled by 70 * completion of the DRM driver's work and also any work associated with the 71 * fence previously in the syncobj. 72 * 73 * When GPU work which waits on a syncobj is enqueued in a DRM driver, at the 74 * time the work is enqueued, it waits on the syncobj's fence before 75 * submitting the work to hardware. That fence is either : 76 * 77 * - The syncobj's current fence if the syncobj is considered as a binary 78 * primitive. 79 * - The struct &dma_fence associated with a given point if the syncobj is 80 * considered as a timeline primitive. 81 * 82 * If the syncobj's fence is NULL or not present in the syncobj's timeline, 83 * the enqueue operation is expected to fail. 84 * 85 * With binary syncobj, all manipulation of the syncobjs's fence happens in 86 * terms of the current fence at the time the ioctl is called by userspace 87 * regardless of whether that operation is an immediate host-side operation 88 * (signal or reset) or or an operation which is enqueued in some driver 89 * queue. &DRM_IOCTL_SYNCOBJ_RESET and &DRM_IOCTL_SYNCOBJ_SIGNAL can be used 90 * to manipulate a syncobj from the host by resetting its pointer to NULL or 91 * setting its pointer to a fence which is already signaled. 92 * 93 * With a timeline syncobj, all manipulation of the synobj's fence happens in 94 * terms of a u64 value referring to point in the timeline. See 95 * dma_fence_chain_find_seqno() to see how a given point is found in the 96 * timeline. 97 * 98 * Note that applications should be careful to always use timeline set of 99 * ioctl() when dealing with syncobj considered as timeline. Using a binary 100 * set of ioctl() with a syncobj considered as timeline could result incorrect 101 * synchronization. The use of binary syncobj is supported through the 102 * timeline set of ioctl() by using a point value of 0, this will reproduce 103 * the behavior of the binary set of ioctl() (for example replace the 104 * syncobj's fence when signaling). 105 * 106 * 107 * Host-side wait on syncobjs 108 * -------------------------- 109 * 110 * &DRM_IOCTL_SYNCOBJ_WAIT takes an array of syncobj handles and does a 111 * host-side wait on all of the syncobj fences simultaneously. 112 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL is set, the wait ioctl will wait on 113 * all of the syncobj fences to be signaled before it returns. 114 * Otherwise, it returns once at least one syncobj fence has been signaled 115 * and the index of a signaled fence is written back to the client. 116 * 117 * Unlike the enqueued GPU work dependencies which fail if they see a NULL 118 * fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set, 119 * the host-side wait will first wait for the syncobj to receive a non-NULL 120 * fence and then wait on that fence. 121 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is not set and any one of the 122 * syncobjs in the array has a NULL fence, -EINVAL will be returned. 123 * Assuming the syncobj starts off with a NULL fence, this allows a client 124 * to do a host wait in one thread (or process) which waits on GPU work 125 * submitted in another thread (or process) without having to manually 126 * synchronize between the two. 127 * This requirement is inherited from the Vulkan fence API. 128 * 129 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE is set, the ioctl will also set 130 * a fence deadline hint on the backing fences before waiting, to provide the 131 * fence signaler with an appropriate sense of urgency. The deadline is 132 * specified as an absolute &CLOCK_MONOTONIC value in units of ns. 133 * 134 * Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj 135 * handles as well as an array of u64 points and does a host-side wait on all 136 * of syncobj fences at the given points simultaneously. 137 * 138 * &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT also adds the ability to wait for a given 139 * fence to materialize on the timeline without waiting for the fence to be 140 * signaled by using the &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE flag. This 141 * requirement is inherited from the wait-before-signal behavior required by 142 * the Vulkan timeline semaphore API. 143 * 144 * Alternatively, &DRM_IOCTL_SYNCOBJ_EVENTFD can be used to wait without 145 * blocking: an eventfd will be signaled when the syncobj is. This is useful to 146 * integrate the wait in an event loop. 147 * 148 * 149 * Import/export of syncobjs 150 * ------------------------- 151 * 152 * &DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE and &DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD 153 * provide two mechanisms for import/export of syncobjs. 154 * 155 * The first lets the client import or export an entire syncobj to a file 156 * descriptor. 157 * These fd's are opaque and have no other use case, except passing the 158 * syncobj between processes. 159 * All exported file descriptors and any syncobj handles created as a 160 * result of importing those file descriptors own a reference to the 161 * same underlying struct &drm_syncobj and the syncobj can be used 162 * persistently across all the processes with which it is shared. 163 * The syncobj is freed only once the last reference is dropped. 164 * Unlike dma-buf, importing a syncobj creates a new handle (with its own 165 * reference) for every import instead of de-duplicating. 166 * The primary use-case of this persistent import/export is for shared 167 * Vulkan fences and semaphores. 168 * 169 * The second import/export mechanism, which is indicated by 170 * &DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE or 171 * &DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE lets the client 172 * import/export the syncobj's current fence from/to a &sync_file. 173 * When a syncobj is exported to a sync file, that sync file wraps the 174 * sycnobj's fence at the time of export and any later signal or reset 175 * operations on the syncobj will not affect the exported sync file. 176 * When a sync file is imported into a syncobj, the syncobj's fence is set 177 * to the fence wrapped by that sync file. 178 * Because sync files are immutable, resetting or signaling the syncobj 179 * will not affect any sync files whose fences have been imported into the 180 * syncobj. 181 * 182 * 183 * Import/export of timeline points in timeline syncobjs 184 * ----------------------------------------------------- 185 * 186 * &DRM_IOCTL_SYNCOBJ_TRANSFER provides a mechanism to transfer a struct 187 * &dma_fence_chain of a syncobj at a given u64 point to another u64 point 188 * into another syncobj. 189 * 190 * Note that if you want to transfer a struct &dma_fence_chain from a given 191 * point on a timeline syncobj from/into a binary syncobj, you can use the 192 * point 0 to mean take/replace the fence in the syncobj. 193 */ 194 195 #include <linux/anon_inodes.h> 196 #include <linux/dma-fence-unwrap.h> 197 #include <linux/eventfd.h> 198 #include <linux/export.h> 199 #include <linux/file.h> 200 #include <linux/fs.h> 201 #include <linux/sched/signal.h> 202 #include <linux/sync_file.h> 203 #include <linux/uaccess.h> 204 205 #include <drm/drm.h> 206 #include <drm/drm_drv.h> 207 #include <drm/drm_file.h> 208 #include <drm/drm_gem.h> 209 #include <drm/drm_print.h> 210 #include <drm/drm_syncobj.h> 211 #include <drm/drm_utils.h> 212 213 #include "drm_internal.h" 214 215 struct syncobj_wait_entry { 216 struct list_head node; 217 struct task_struct *task; 218 struct dma_fence *fence; 219 struct dma_fence_cb fence_cb; 220 u64 point; 221 }; 222 223 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj, 224 struct syncobj_wait_entry *wait); 225 226 struct syncobj_eventfd_entry { 227 struct list_head node; 228 struct dma_fence *fence; 229 struct dma_fence_cb fence_cb; 230 struct drm_syncobj *syncobj; 231 struct eventfd_ctx *ev_fd_ctx; 232 u64 point; 233 u32 flags; 234 }; 235 236 static void 237 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj, 238 struct syncobj_eventfd_entry *entry); 239 240 /** 241 * drm_syncobj_find - lookup and reference a sync object. 242 * @file_private: drm file private pointer 243 * @handle: sync object handle to lookup. 244 * 245 * Returns a reference to the syncobj pointed to by handle or NULL. The 246 * reference must be released by calling drm_syncobj_put(). 247 */ 248 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private, 249 u32 handle) 250 { 251 struct drm_syncobj *syncobj; 252 253 xa_lock(&file_private->syncobj_xa); 254 255 /* Check if we currently have a reference on the object */ 256 syncobj = xa_load(&file_private->syncobj_xa, handle); 257 if (syncobj) 258 drm_syncobj_get(syncobj); 259 260 xa_unlock(&file_private->syncobj_xa); 261 262 return syncobj; 263 } 264 EXPORT_SYMBOL(drm_syncobj_find); 265 266 static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj, 267 struct syncobj_wait_entry *wait) 268 { 269 struct dma_fence *fence; 270 271 if (wait->fence) 272 return; 273 274 spin_lock(&syncobj->lock); 275 /* We've already tried once to get a fence and failed. Now that we 276 * have the lock, try one more time just to be sure we don't add a 277 * callback when a fence has already been set. 278 */ 279 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1)); 280 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) { 281 dma_fence_put(fence); 282 list_add_tail(&wait->node, &syncobj->cb_list); 283 } else if (!fence) { 284 wait->fence = dma_fence_get_stub(); 285 } else { 286 wait->fence = fence; 287 } 288 spin_unlock(&syncobj->lock); 289 } 290 291 static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj, 292 struct syncobj_wait_entry *wait) 293 { 294 if (!wait->node.next) 295 return; 296 297 spin_lock(&syncobj->lock); 298 list_del_init(&wait->node); 299 spin_unlock(&syncobj->lock); 300 } 301 302 static void 303 syncobj_eventfd_entry_free(struct syncobj_eventfd_entry *entry) 304 { 305 eventfd_ctx_put(entry->ev_fd_ctx); 306 dma_fence_put(entry->fence); 307 /* This happens either inside the syncobj lock, or after the node has 308 * already been removed from the list. 309 */ 310 list_del(&entry->node); 311 kfree(entry); 312 } 313 314 static void 315 drm_syncobj_add_eventfd(struct drm_syncobj *syncobj, 316 struct syncobj_eventfd_entry *entry) 317 { 318 spin_lock(&syncobj->lock); 319 list_add_tail(&entry->node, &syncobj->ev_fd_list); 320 syncobj_eventfd_entry_func(syncobj, entry); 321 spin_unlock(&syncobj->lock); 322 } 323 324 /** 325 * drm_syncobj_add_point - add new timeline point to the syncobj 326 * @syncobj: sync object to add timeline point do 327 * @chain: chain node to use to add the point 328 * @fence: fence to encapsulate in the chain node 329 * @point: sequence number to use for the point 330 * 331 * Add the chain node as new timeline point to the syncobj. 332 */ 333 void drm_syncobj_add_point(struct drm_syncobj *syncobj, 334 struct dma_fence_chain *chain, 335 struct dma_fence *fence, 336 uint64_t point) 337 { 338 struct syncobj_wait_entry *wait_cur, *wait_tmp; 339 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 340 struct dma_fence *prev; 341 342 dma_fence_get(fence); 343 344 spin_lock(&syncobj->lock); 345 346 prev = drm_syncobj_fence_get(syncobj); 347 /* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */ 348 if (prev && prev->seqno >= point) 349 DRM_DEBUG("You are adding an unorder point to timeline!\n"); 350 dma_fence_chain_init(chain, prev, fence, point); 351 rcu_assign_pointer(syncobj->fence, &chain->base); 352 353 list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node) 354 syncobj_wait_syncobj_func(syncobj, wait_cur); 355 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 356 syncobj_eventfd_entry_func(syncobj, ev_fd_cur); 357 spin_unlock(&syncobj->lock); 358 359 /* Walk the chain once to trigger garbage collection */ 360 dma_fence_chain_for_each(fence, prev); 361 dma_fence_put(prev); 362 } 363 EXPORT_SYMBOL(drm_syncobj_add_point); 364 365 /** 366 * drm_syncobj_replace_fence - replace fence in a sync object. 367 * @syncobj: Sync object to replace fence in 368 * @fence: fence to install in sync file. 369 * 370 * This replaces the fence on a sync object. 371 */ 372 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj, 373 struct dma_fence *fence) 374 { 375 struct dma_fence *old_fence; 376 struct syncobj_wait_entry *wait_cur, *wait_tmp; 377 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 378 379 if (fence) 380 dma_fence_get(fence); 381 382 spin_lock(&syncobj->lock); 383 384 old_fence = rcu_dereference_protected(syncobj->fence, 385 lockdep_is_held(&syncobj->lock)); 386 rcu_assign_pointer(syncobj->fence, fence); 387 388 if (fence != old_fence) { 389 list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node) 390 syncobj_wait_syncobj_func(syncobj, wait_cur); 391 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 392 syncobj_eventfd_entry_func(syncobj, ev_fd_cur); 393 } 394 395 spin_unlock(&syncobj->lock); 396 397 dma_fence_put(old_fence); 398 } 399 EXPORT_SYMBOL(drm_syncobj_replace_fence); 400 401 /** 402 * drm_syncobj_assign_null_handle - assign a stub fence to the sync object 403 * @syncobj: sync object to assign the fence on 404 * 405 * Assign a already signaled stub fence to the sync object. 406 */ 407 static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj) 408 { 409 struct dma_fence *fence = dma_fence_allocate_private_stub(ktime_get()); 410 411 if (!fence) 412 return -ENOMEM; 413 414 drm_syncobj_replace_fence(syncobj, fence); 415 dma_fence_put(fence); 416 return 0; 417 } 418 419 /* 5s default for wait submission */ 420 #define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL 421 /** 422 * drm_syncobj_find_fence - lookup and reference the fence in a sync object 423 * @file_private: drm file private pointer 424 * @handle: sync object handle to lookup. 425 * @point: timeline point 426 * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not 427 * @fence: out parameter for the fence 428 * 429 * This is just a convenience function that combines drm_syncobj_find() and 430 * drm_syncobj_fence_get(). 431 * 432 * Returns 0 on success or a negative error value on failure. On success @fence 433 * contains a reference to the fence, which must be released by calling 434 * dma_fence_put(). 435 */ 436 int drm_syncobj_find_fence(struct drm_file *file_private, 437 u32 handle, u64 point, u64 flags, 438 struct dma_fence **fence) 439 { 440 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle); 441 struct syncobj_wait_entry wait; 442 u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT); 443 int ret; 444 445 if (!syncobj) 446 return -ENOENT; 447 448 if (flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) { 449 ret = -EINVAL; 450 goto out; 451 } 452 453 /* Waiting for userspace with locks held is illegal cause that can 454 * trivial deadlock with page faults for example. Make lockdep complain 455 * about it early on. 456 */ 457 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) { 458 might_sleep(); 459 lockdep_assert_none_held_once(); 460 } 461 462 *fence = drm_syncobj_fence_get(syncobj); 463 464 if (*fence) { 465 ret = dma_fence_chain_find_seqno(fence, point); 466 if (!ret) { 467 /* If the requested seqno is already signaled 468 * drm_syncobj_find_fence may return a NULL 469 * fence. To make sure the recipient gets 470 * signalled, use a new fence instead. 471 */ 472 if (!*fence) 473 *fence = dma_fence_get_stub(); 474 475 goto out; 476 } 477 dma_fence_put(*fence); 478 } else { 479 ret = -EINVAL; 480 } 481 482 if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)) 483 goto out; 484 485 memset(&wait, 0, sizeof(wait)); 486 wait.task = current; 487 wait.point = point; 488 drm_syncobj_fence_add_wait(syncobj, &wait); 489 490 do { 491 set_current_state(TASK_INTERRUPTIBLE); 492 if (wait.fence) { 493 ret = 0; 494 break; 495 } 496 if (timeout == 0) { 497 ret = -ETIME; 498 break; 499 } 500 501 if (signal_pending(current)) { 502 ret = -ERESTARTSYS; 503 break; 504 } 505 506 timeout = schedule_timeout(timeout); 507 } while (1); 508 509 __set_current_state(TASK_RUNNING); 510 *fence = wait.fence; 511 512 if (wait.node.next) 513 drm_syncobj_remove_wait(syncobj, &wait); 514 515 out: 516 drm_syncobj_put(syncobj); 517 518 return ret; 519 } 520 EXPORT_SYMBOL(drm_syncobj_find_fence); 521 522 /** 523 * drm_syncobj_free - free a sync object. 524 * @kref: kref to free. 525 * 526 * Only to be called from kref_put in drm_syncobj_put. 527 */ 528 void drm_syncobj_free(struct kref *kref) 529 { 530 struct drm_syncobj *syncobj = container_of(kref, 531 struct drm_syncobj, 532 refcount); 533 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 534 535 drm_syncobj_replace_fence(syncobj, NULL); 536 537 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 538 syncobj_eventfd_entry_free(ev_fd_cur); 539 540 kfree(syncobj); 541 } 542 EXPORT_SYMBOL(drm_syncobj_free); 543 544 /** 545 * drm_syncobj_create - create a new syncobj 546 * @out_syncobj: returned syncobj 547 * @flags: DRM_SYNCOBJ_* flags 548 * @fence: if non-NULL, the syncobj will represent this fence 549 * 550 * This is the first function to create a sync object. After creating, drivers 551 * probably want to make it available to userspace, either through 552 * drm_syncobj_get_handle() or drm_syncobj_get_fd(). 553 * 554 * Returns 0 on success or a negative error value on failure. 555 */ 556 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags, 557 struct dma_fence *fence) 558 { 559 int ret; 560 struct drm_syncobj *syncobj; 561 562 syncobj = kzalloc_obj(struct drm_syncobj); 563 if (!syncobj) 564 return -ENOMEM; 565 566 kref_init(&syncobj->refcount); 567 INIT_LIST_HEAD(&syncobj->cb_list); 568 INIT_LIST_HEAD(&syncobj->ev_fd_list); 569 spin_lock_init(&syncobj->lock); 570 571 if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) { 572 ret = drm_syncobj_assign_null_handle(syncobj); 573 if (ret < 0) { 574 drm_syncobj_put(syncobj); 575 return ret; 576 } 577 } 578 579 if (fence) 580 drm_syncobj_replace_fence(syncobj, fence); 581 582 *out_syncobj = syncobj; 583 return 0; 584 } 585 EXPORT_SYMBOL(drm_syncobj_create); 586 587 /** 588 * drm_syncobj_get_handle - get a handle from a syncobj 589 * @file_private: drm file private pointer 590 * @syncobj: Sync object to export 591 * @handle: out parameter with the new handle 592 * 593 * Exports a sync object created with drm_syncobj_create() as a handle on 594 * @file_private to userspace. 595 * 596 * Returns 0 on success or a negative error value on failure. 597 */ 598 int drm_syncobj_get_handle(struct drm_file *file_private, 599 struct drm_syncobj *syncobj, u32 *handle) 600 { 601 int ret; 602 603 /* take a reference to put in the xarray */ 604 drm_syncobj_get(syncobj); 605 606 ret = xa_alloc(&file_private->syncobj_xa, handle, syncobj, xa_limit_32b, 607 GFP_KERNEL); 608 if (ret) 609 drm_syncobj_put(syncobj); 610 611 return ret; 612 } 613 EXPORT_SYMBOL(drm_syncobj_get_handle); 614 615 static int drm_syncobj_create_as_handle(struct drm_file *file_private, 616 u32 *handle, uint32_t flags) 617 { 618 int ret; 619 struct drm_syncobj *syncobj; 620 621 ret = drm_syncobj_create(&syncobj, flags, NULL); 622 if (ret) 623 return ret; 624 625 ret = drm_syncobj_get_handle(file_private, syncobj, handle); 626 drm_syncobj_put(syncobj); 627 return ret; 628 } 629 630 static int drm_syncobj_destroy(struct drm_file *file_private, 631 u32 handle) 632 { 633 struct drm_syncobj *syncobj; 634 635 syncobj = xa_erase(&file_private->syncobj_xa, handle); 636 if (!syncobj) 637 return -EINVAL; 638 639 drm_syncobj_put(syncobj); 640 return 0; 641 } 642 643 static int drm_syncobj_file_release(struct inode *inode, struct file *file) 644 { 645 struct drm_syncobj *syncobj = file->private_data; 646 647 drm_syncobj_put(syncobj); 648 return 0; 649 } 650 651 static const struct file_operations drm_syncobj_file_fops = { 652 .release = drm_syncobj_file_release, 653 }; 654 655 /** 656 * drm_syncobj_get_fd - get a file descriptor from a syncobj 657 * @syncobj: Sync object to export 658 * @p_fd: out parameter with the new file descriptor 659 * 660 * Exports a sync object created with drm_syncobj_create() as a file descriptor. 661 * 662 * Returns 0 on success or a negative error value on failure. 663 */ 664 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd) 665 { 666 struct file *file; 667 int fd; 668 669 fd = get_unused_fd_flags(O_CLOEXEC); 670 if (fd < 0) 671 return fd; 672 673 file = anon_inode_getfile("syncobj_file", 674 &drm_syncobj_file_fops, 675 syncobj, 0); 676 if (IS_ERR(file)) { 677 put_unused_fd(fd); 678 return PTR_ERR(file); 679 } 680 681 drm_syncobj_get(syncobj); 682 fd_install(fd, file); 683 684 *p_fd = fd; 685 return 0; 686 } 687 EXPORT_SYMBOL(drm_syncobj_get_fd); 688 689 static int drm_syncobj_handle_to_fd(struct drm_file *file_private, 690 u32 handle, int *p_fd) 691 { 692 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle); 693 int ret; 694 695 if (!syncobj) 696 return -EINVAL; 697 698 ret = drm_syncobj_get_fd(syncobj, p_fd); 699 drm_syncobj_put(syncobj); 700 return ret; 701 } 702 703 static int drm_syncobj_fd_to_handle(struct drm_file *file_private, 704 int fd, u32 *handle) 705 { 706 struct drm_syncobj *syncobj; 707 CLASS(fd, f)(fd); 708 int ret; 709 710 if (fd_empty(f)) 711 return -EINVAL; 712 713 if (fd_file(f)->f_op != &drm_syncobj_file_fops) 714 return -EINVAL; 715 716 /* take a reference to put in the xarray */ 717 syncobj = fd_file(f)->private_data; 718 drm_syncobj_get(syncobj); 719 720 ret = xa_alloc(&file_private->syncobj_xa, handle, syncobj, xa_limit_32b, 721 GFP_KERNEL); 722 if (ret) 723 drm_syncobj_put(syncobj); 724 725 return ret; 726 } 727 728 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private, 729 int fd, int handle, u64 point) 730 { 731 struct dma_fence *fence = sync_file_get_fence(fd); 732 struct drm_syncobj *syncobj; 733 734 if (!fence) 735 return -EINVAL; 736 737 syncobj = drm_syncobj_find(file_private, handle); 738 if (!syncobj) { 739 dma_fence_put(fence); 740 return -ENOENT; 741 } 742 743 if (point) { 744 struct dma_fence_chain *chain = dma_fence_chain_alloc(); 745 746 if (!chain) 747 return -ENOMEM; 748 749 drm_syncobj_add_point(syncobj, chain, fence, point); 750 } else { 751 drm_syncobj_replace_fence(syncobj, fence); 752 } 753 754 dma_fence_put(fence); 755 drm_syncobj_put(syncobj); 756 return 0; 757 } 758 759 static int drm_syncobj_export_sync_file(struct drm_file *file_private, 760 int handle, u64 point, int *p_fd) 761 { 762 int ret; 763 struct dma_fence *fence; 764 struct sync_file *sync_file; 765 int fd = get_unused_fd_flags(O_CLOEXEC); 766 767 if (fd < 0) 768 return fd; 769 770 ret = drm_syncobj_find_fence(file_private, handle, point, 0, &fence); 771 if (ret) 772 goto err_put_fd; 773 774 sync_file = sync_file_create(fence); 775 776 dma_fence_put(fence); 777 778 if (!sync_file) { 779 ret = -EINVAL; 780 goto err_put_fd; 781 } 782 783 fd_install(fd, sync_file->file); 784 785 *p_fd = fd; 786 return 0; 787 err_put_fd: 788 put_unused_fd(fd); 789 return ret; 790 } 791 /** 792 * drm_syncobj_open - initializes syncobj file-private structures at devnode open time 793 * @file_private: drm file-private structure to set up 794 * 795 * Called at device open time, sets up the structure for handling refcounting 796 * of sync objects. 797 */ 798 void 799 drm_syncobj_open(struct drm_file *file_private) 800 { 801 xa_init_flags(&file_private->syncobj_xa, XA_FLAGS_ALLOC1); 802 } 803 804 /** 805 * drm_syncobj_release - release file-private sync object resources 806 * @file_private: drm file-private structure to clean up 807 * 808 * Called at close time when the filp is going away. 809 * 810 * Releases any remaining references on objects by this filp. 811 */ 812 void 813 drm_syncobj_release(struct drm_file *file_private) 814 { 815 struct drm_syncobj *syncobj; 816 unsigned long handle; 817 818 xa_for_each(&file_private->syncobj_xa, handle, syncobj) 819 drm_syncobj_put(syncobj); 820 xa_destroy(&file_private->syncobj_xa); 821 } 822 823 int 824 drm_syncobj_create_ioctl(struct drm_device *dev, void *data, 825 struct drm_file *file_private) 826 { 827 struct drm_syncobj_create *args = data; 828 829 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 830 return -EOPNOTSUPP; 831 832 /* no valid flags yet */ 833 if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED) 834 return -EINVAL; 835 836 return drm_syncobj_create_as_handle(file_private, 837 &args->handle, args->flags); 838 } 839 840 int 841 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data, 842 struct drm_file *file_private) 843 { 844 struct drm_syncobj_destroy *args = data; 845 846 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 847 return -EOPNOTSUPP; 848 849 /* make sure padding is empty */ 850 if (args->pad) 851 return -EINVAL; 852 return drm_syncobj_destroy(file_private, args->handle); 853 } 854 855 int 856 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data, 857 struct drm_file *file_private) 858 { 859 struct drm_syncobj_handle *args = data; 860 unsigned int valid_flags = DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_TIMELINE | 861 DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE; 862 u64 point = 0; 863 864 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 865 return -EOPNOTSUPP; 866 867 if (args->pad) 868 return -EINVAL; 869 870 if (args->flags & ~valid_flags) 871 return -EINVAL; 872 873 if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_TIMELINE) 874 point = args->point; 875 876 if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE) 877 return drm_syncobj_export_sync_file(file_private, args->handle, 878 point, &args->fd); 879 880 if (args->point) 881 return -EINVAL; 882 883 return drm_syncobj_handle_to_fd(file_private, args->handle, 884 &args->fd); 885 } 886 887 int 888 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data, 889 struct drm_file *file_private) 890 { 891 struct drm_syncobj_handle *args = data; 892 unsigned int valid_flags = DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_TIMELINE | 893 DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE; 894 u64 point = 0; 895 896 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 897 return -EOPNOTSUPP; 898 899 if (args->pad) 900 return -EINVAL; 901 902 if (args->flags & ~valid_flags) 903 return -EINVAL; 904 905 if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_TIMELINE) 906 point = args->point; 907 908 if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE) 909 return drm_syncobj_import_sync_file_fence(file_private, 910 args->fd, 911 args->handle, 912 point); 913 914 if (args->point) 915 return -EINVAL; 916 917 return drm_syncobj_fd_to_handle(file_private, args->fd, 918 &args->handle); 919 } 920 921 static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private, 922 struct drm_syncobj_transfer *args) 923 { 924 struct drm_syncobj *timeline_syncobj = NULL; 925 struct dma_fence *fence, *tmp; 926 struct dma_fence_chain *chain; 927 int ret; 928 929 timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle); 930 if (!timeline_syncobj) { 931 return -ENOENT; 932 } 933 ret = drm_syncobj_find_fence(file_private, args->src_handle, 934 args->src_point, args->flags, 935 &tmp); 936 if (ret) 937 goto err_put_timeline; 938 939 fence = dma_fence_unwrap_merge(tmp); 940 dma_fence_put(tmp); 941 if (!fence) { 942 ret = -ENOMEM; 943 goto err_put_timeline; 944 } 945 946 chain = dma_fence_chain_alloc(); 947 if (!chain) { 948 ret = -ENOMEM; 949 goto err_free_fence; 950 } 951 952 drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point); 953 err_free_fence: 954 dma_fence_put(fence); 955 err_put_timeline: 956 drm_syncobj_put(timeline_syncobj); 957 958 return ret; 959 } 960 961 static int 962 drm_syncobj_transfer_to_binary(struct drm_file *file_private, 963 struct drm_syncobj_transfer *args) 964 { 965 struct drm_syncobj *binary_syncobj = NULL; 966 struct dma_fence *fence; 967 int ret; 968 969 binary_syncobj = drm_syncobj_find(file_private, args->dst_handle); 970 if (!binary_syncobj) 971 return -ENOENT; 972 ret = drm_syncobj_find_fence(file_private, args->src_handle, 973 args->src_point, args->flags, &fence); 974 if (ret) 975 goto err; 976 drm_syncobj_replace_fence(binary_syncobj, fence); 977 dma_fence_put(fence); 978 err: 979 drm_syncobj_put(binary_syncobj); 980 981 return ret; 982 } 983 int 984 drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data, 985 struct drm_file *file_private) 986 { 987 struct drm_syncobj_transfer *args = data; 988 int ret; 989 990 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 991 return -EOPNOTSUPP; 992 993 if (args->pad) 994 return -EINVAL; 995 996 if (args->dst_point) 997 ret = drm_syncobj_transfer_to_timeline(file_private, args); 998 else 999 ret = drm_syncobj_transfer_to_binary(file_private, args); 1000 1001 return ret; 1002 } 1003 1004 static void syncobj_wait_fence_func(struct dma_fence *fence, 1005 struct dma_fence_cb *cb) 1006 { 1007 struct syncobj_wait_entry *wait = 1008 container_of(cb, struct syncobj_wait_entry, fence_cb); 1009 1010 wake_up_process(wait->task); 1011 } 1012 1013 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj, 1014 struct syncobj_wait_entry *wait) 1015 { 1016 struct dma_fence *fence; 1017 1018 /* This happens inside the syncobj lock */ 1019 fence = rcu_dereference_protected(syncobj->fence, 1020 lockdep_is_held(&syncobj->lock)); 1021 dma_fence_get(fence); 1022 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) { 1023 dma_fence_put(fence); 1024 return; 1025 } else if (!fence) { 1026 wait->fence = dma_fence_get_stub(); 1027 } else { 1028 wait->fence = fence; 1029 } 1030 1031 wake_up_process(wait->task); 1032 list_del_init(&wait->node); 1033 } 1034 1035 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs, 1036 void __user *user_points, 1037 uint32_t count, 1038 uint32_t flags, 1039 signed long timeout, 1040 uint32_t *idx, 1041 ktime_t *deadline) 1042 { 1043 struct syncobj_wait_entry *entries; 1044 struct dma_fence *fence; 1045 uint64_t *points; 1046 uint32_t signaled_count, i; 1047 1048 if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1049 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) { 1050 might_sleep(); 1051 lockdep_assert_none_held_once(); 1052 } 1053 1054 points = kmalloc_array(count, sizeof(*points), GFP_KERNEL); 1055 if (points == NULL) 1056 return -ENOMEM; 1057 1058 if (!user_points) { 1059 memset(points, 0, count * sizeof(uint64_t)); 1060 1061 } else if (copy_from_user(points, user_points, 1062 sizeof(uint64_t) * count)) { 1063 timeout = -EFAULT; 1064 goto err_free_points; 1065 } 1066 1067 entries = kzalloc_objs(*entries, count); 1068 if (!entries) { 1069 timeout = -ENOMEM; 1070 goto err_free_points; 1071 } 1072 /* Walk the list of sync objects and initialize entries. We do 1073 * this up-front so that we can properly return -EINVAL if there is 1074 * a syncobj with a missing fence and then never have the chance of 1075 * returning -EINVAL again. 1076 */ 1077 signaled_count = 0; 1078 for (i = 0; i < count; ++i) { 1079 struct dma_fence *fence; 1080 1081 entries[i].task = current; 1082 entries[i].point = points[i]; 1083 fence = drm_syncobj_fence_get(syncobjs[i]); 1084 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) { 1085 dma_fence_put(fence); 1086 if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1087 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) { 1088 continue; 1089 } else { 1090 timeout = -EINVAL; 1091 goto cleanup_entries; 1092 } 1093 } 1094 1095 if (fence) 1096 entries[i].fence = fence; 1097 else 1098 entries[i].fence = dma_fence_get_stub(); 1099 1100 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) || 1101 dma_fence_is_signaled(entries[i].fence)) { 1102 if (signaled_count == 0 && idx) 1103 *idx = i; 1104 signaled_count++; 1105 } 1106 } 1107 1108 if (signaled_count == count || 1109 (signaled_count > 0 && 1110 !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL))) 1111 goto cleanup_entries; 1112 1113 /* There's a very annoying laxness in the dma_fence API here, in 1114 * that backends are not required to automatically report when a 1115 * fence is signaled prior to fence->ops->enable_signaling() being 1116 * called. So here if we fail to match signaled_count, we need to 1117 * fallthough and try a 0 timeout wait! 1118 */ 1119 1120 if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1121 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) { 1122 for (i = 0; i < count; ++i) 1123 drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]); 1124 } 1125 1126 if (deadline) { 1127 for (i = 0; i < count; ++i) { 1128 fence = entries[i].fence; 1129 if (!fence) 1130 continue; 1131 dma_fence_set_deadline(fence, *deadline); 1132 } 1133 } 1134 1135 do { 1136 set_current_state(TASK_INTERRUPTIBLE); 1137 1138 signaled_count = 0; 1139 for (i = 0; i < count; ++i) { 1140 fence = entries[i].fence; 1141 if (!fence) 1142 continue; 1143 1144 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) || 1145 dma_fence_is_signaled(fence) || 1146 (!entries[i].fence_cb.func && 1147 dma_fence_add_callback(fence, 1148 &entries[i].fence_cb, 1149 syncobj_wait_fence_func))) { 1150 /* The fence has been signaled */ 1151 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) { 1152 signaled_count++; 1153 } else { 1154 if (idx) 1155 *idx = i; 1156 goto done_waiting; 1157 } 1158 } 1159 } 1160 1161 if (signaled_count == count) 1162 goto done_waiting; 1163 1164 if (timeout == 0) { 1165 timeout = -ETIME; 1166 goto done_waiting; 1167 } 1168 1169 if (signal_pending(current)) { 1170 timeout = -ERESTARTSYS; 1171 goto done_waiting; 1172 } 1173 1174 timeout = schedule_timeout(timeout); 1175 } while (1); 1176 1177 done_waiting: 1178 __set_current_state(TASK_RUNNING); 1179 1180 cleanup_entries: 1181 for (i = 0; i < count; ++i) { 1182 drm_syncobj_remove_wait(syncobjs[i], &entries[i]); 1183 if (entries[i].fence_cb.func) 1184 dma_fence_remove_callback(entries[i].fence, 1185 &entries[i].fence_cb); 1186 dma_fence_put(entries[i].fence); 1187 } 1188 kfree(entries); 1189 1190 err_free_points: 1191 kfree(points); 1192 1193 return timeout; 1194 } 1195 1196 /** 1197 * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value 1198 * 1199 * @timeout_nsec: timeout nsec component in ns, 0 for poll 1200 * 1201 * Calculate the timeout in jiffies from an absolute time in sec/nsec. 1202 */ 1203 signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec) 1204 { 1205 ktime_t abs_timeout, now; 1206 u64 timeout_ns, timeout_jiffies64; 1207 1208 /* make 0 timeout means poll - absolute 0 doesn't seem valid */ 1209 if (timeout_nsec == 0) 1210 return 0; 1211 1212 abs_timeout = ns_to_ktime(timeout_nsec); 1213 now = ktime_get(); 1214 1215 if (!ktime_after(abs_timeout, now)) 1216 return 0; 1217 1218 timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now)); 1219 1220 timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns); 1221 /* clamp timeout to avoid infinite timeout */ 1222 if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1) 1223 return MAX_SCHEDULE_TIMEOUT - 1; 1224 1225 return timeout_jiffies64 + 1; 1226 } 1227 EXPORT_SYMBOL(drm_timeout_abs_to_jiffies); 1228 1229 static int drm_syncobj_array_wait(struct drm_device *dev, 1230 struct drm_file *file_private, 1231 struct drm_syncobj_wait *wait, 1232 struct drm_syncobj_timeline_wait *timeline_wait, 1233 struct drm_syncobj **syncobjs, bool timeline, 1234 ktime_t *deadline) 1235 { 1236 signed long timeout = 0; 1237 uint32_t first = ~0; 1238 1239 if (!timeline) { 1240 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec); 1241 timeout = drm_syncobj_array_wait_timeout(syncobjs, 1242 NULL, 1243 wait->count_handles, 1244 wait->flags, 1245 timeout, &first, 1246 deadline); 1247 if (timeout < 0) 1248 return timeout; 1249 wait->first_signaled = first; 1250 } else { 1251 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec); 1252 timeout = drm_syncobj_array_wait_timeout(syncobjs, 1253 u64_to_user_ptr(timeline_wait->points), 1254 timeline_wait->count_handles, 1255 timeline_wait->flags, 1256 timeout, &first, 1257 deadline); 1258 if (timeout < 0) 1259 return timeout; 1260 timeline_wait->first_signaled = first; 1261 } 1262 return 0; 1263 } 1264 1265 static int drm_syncobj_array_find(struct drm_file *file_private, 1266 void __user *user_handles, 1267 uint32_t count_handles, 1268 struct drm_syncobj ***syncobjs_out) 1269 { 1270 uint32_t i, *handles; 1271 struct drm_syncobj **syncobjs; 1272 int ret; 1273 1274 handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL); 1275 if (handles == NULL) 1276 return -ENOMEM; 1277 1278 if (copy_from_user(handles, user_handles, 1279 sizeof(uint32_t) * count_handles)) { 1280 ret = -EFAULT; 1281 goto err_free_handles; 1282 } 1283 1284 syncobjs = kmalloc_objs(*syncobjs, count_handles); 1285 if (syncobjs == NULL) { 1286 ret = -ENOMEM; 1287 goto err_free_handles; 1288 } 1289 1290 for (i = 0; i < count_handles; i++) { 1291 syncobjs[i] = drm_syncobj_find(file_private, handles[i]); 1292 if (!syncobjs[i]) { 1293 ret = -ENOENT; 1294 goto err_put_syncobjs; 1295 } 1296 } 1297 1298 kfree(handles); 1299 *syncobjs_out = syncobjs; 1300 return 0; 1301 1302 err_put_syncobjs: 1303 while (i-- > 0) 1304 drm_syncobj_put(syncobjs[i]); 1305 kfree(syncobjs); 1306 err_free_handles: 1307 kfree(handles); 1308 1309 return ret; 1310 } 1311 1312 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs, 1313 uint32_t count) 1314 { 1315 uint32_t i; 1316 1317 for (i = 0; i < count; i++) 1318 drm_syncobj_put(syncobjs[i]); 1319 kfree(syncobjs); 1320 } 1321 1322 int 1323 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data, 1324 struct drm_file *file_private) 1325 { 1326 struct drm_syncobj_wait *args = data; 1327 struct drm_syncobj **syncobjs; 1328 unsigned int possible_flags; 1329 ktime_t t, *tp = NULL; 1330 int ret = 0; 1331 1332 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1333 return -EOPNOTSUPP; 1334 1335 possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL | 1336 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1337 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE; 1338 1339 if (args->flags & ~possible_flags) 1340 return -EINVAL; 1341 1342 if (args->count_handles == 0) 1343 return 0; 1344 1345 ret = drm_syncobj_array_find(file_private, 1346 u64_to_user_ptr(args->handles), 1347 args->count_handles, 1348 &syncobjs); 1349 if (ret < 0) 1350 return ret; 1351 1352 if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) { 1353 t = ns_to_ktime(args->deadline_nsec); 1354 tp = &t; 1355 } 1356 1357 ret = drm_syncobj_array_wait(dev, file_private, 1358 args, NULL, syncobjs, false, tp); 1359 1360 drm_syncobj_array_free(syncobjs, args->count_handles); 1361 1362 return ret; 1363 } 1364 1365 int 1366 drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data, 1367 struct drm_file *file_private) 1368 { 1369 struct drm_syncobj_timeline_wait *args = data; 1370 struct drm_syncobj **syncobjs; 1371 unsigned int possible_flags; 1372 ktime_t t, *tp = NULL; 1373 int ret = 0; 1374 1375 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1376 return -EOPNOTSUPP; 1377 1378 possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL | 1379 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1380 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE | 1381 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE; 1382 1383 if (args->flags & ~possible_flags) 1384 return -EINVAL; 1385 1386 if (args->count_handles == 0) 1387 return 0; 1388 1389 ret = drm_syncobj_array_find(file_private, 1390 u64_to_user_ptr(args->handles), 1391 args->count_handles, 1392 &syncobjs); 1393 if (ret < 0) 1394 return ret; 1395 1396 if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) { 1397 t = ns_to_ktime(args->deadline_nsec); 1398 tp = &t; 1399 } 1400 1401 ret = drm_syncobj_array_wait(dev, file_private, 1402 NULL, args, syncobjs, true, tp); 1403 1404 drm_syncobj_array_free(syncobjs, args->count_handles); 1405 1406 return ret; 1407 } 1408 1409 static void syncobj_eventfd_entry_fence_func(struct dma_fence *fence, 1410 struct dma_fence_cb *cb) 1411 { 1412 struct syncobj_eventfd_entry *entry = 1413 container_of(cb, struct syncobj_eventfd_entry, fence_cb); 1414 1415 eventfd_signal(entry->ev_fd_ctx); 1416 syncobj_eventfd_entry_free(entry); 1417 } 1418 1419 static void 1420 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj, 1421 struct syncobj_eventfd_entry *entry) 1422 { 1423 int ret; 1424 struct dma_fence *fence; 1425 1426 /* This happens inside the syncobj lock */ 1427 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1)); 1428 if (!fence) 1429 return; 1430 1431 ret = dma_fence_chain_find_seqno(&fence, entry->point); 1432 if (ret != 0) { 1433 /* The given seqno has not been submitted yet. */ 1434 dma_fence_put(fence); 1435 return; 1436 } else if (!fence) { 1437 /* If dma_fence_chain_find_seqno returns 0 but sets the fence 1438 * to NULL, it implies that the given seqno is signaled and a 1439 * later seqno has already been submitted. Assign a stub fence 1440 * so that the eventfd still gets signaled below. 1441 */ 1442 fence = dma_fence_get_stub(); 1443 } 1444 1445 list_del_init(&entry->node); 1446 entry->fence = fence; 1447 1448 if (entry->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) { 1449 eventfd_signal(entry->ev_fd_ctx); 1450 syncobj_eventfd_entry_free(entry); 1451 } else { 1452 ret = dma_fence_add_callback(fence, &entry->fence_cb, 1453 syncobj_eventfd_entry_fence_func); 1454 if (ret == -ENOENT) { 1455 eventfd_signal(entry->ev_fd_ctx); 1456 syncobj_eventfd_entry_free(entry); 1457 } 1458 } 1459 } 1460 1461 int 1462 drm_syncobj_eventfd_ioctl(struct drm_device *dev, void *data, 1463 struct drm_file *file_private) 1464 { 1465 struct drm_syncobj_eventfd *args = data; 1466 struct drm_syncobj *syncobj; 1467 struct eventfd_ctx *ev_fd_ctx; 1468 struct syncobj_eventfd_entry *entry; 1469 int ret; 1470 1471 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1472 return -EOPNOTSUPP; 1473 1474 if (args->flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) 1475 return -EINVAL; 1476 1477 if (args->pad) 1478 return -EINVAL; 1479 1480 syncobj = drm_syncobj_find(file_private, args->handle); 1481 if (!syncobj) 1482 return -ENOENT; 1483 1484 ev_fd_ctx = eventfd_ctx_fdget(args->fd); 1485 if (IS_ERR(ev_fd_ctx)) { 1486 ret = PTR_ERR(ev_fd_ctx); 1487 goto err_fdget; 1488 } 1489 1490 entry = kzalloc_obj(*entry); 1491 if (!entry) { 1492 ret = -ENOMEM; 1493 goto err_kzalloc; 1494 } 1495 entry->syncobj = syncobj; 1496 entry->ev_fd_ctx = ev_fd_ctx; 1497 entry->point = args->point; 1498 entry->flags = args->flags; 1499 1500 drm_syncobj_add_eventfd(syncobj, entry); 1501 drm_syncobj_put(syncobj); 1502 1503 return 0; 1504 1505 err_kzalloc: 1506 eventfd_ctx_put(ev_fd_ctx); 1507 err_fdget: 1508 drm_syncobj_put(syncobj); 1509 return ret; 1510 } 1511 1512 int 1513 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data, 1514 struct drm_file *file_private) 1515 { 1516 struct drm_syncobj_array *args = data; 1517 struct drm_syncobj **syncobjs; 1518 uint32_t i; 1519 int ret; 1520 1521 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1522 return -EOPNOTSUPP; 1523 1524 if (args->pad != 0) 1525 return -EINVAL; 1526 1527 if (args->count_handles == 0) 1528 return -EINVAL; 1529 1530 ret = drm_syncobj_array_find(file_private, 1531 u64_to_user_ptr(args->handles), 1532 args->count_handles, 1533 &syncobjs); 1534 if (ret < 0) 1535 return ret; 1536 1537 for (i = 0; i < args->count_handles; i++) 1538 drm_syncobj_replace_fence(syncobjs[i], NULL); 1539 1540 drm_syncobj_array_free(syncobjs, args->count_handles); 1541 1542 return 0; 1543 } 1544 1545 int 1546 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data, 1547 struct drm_file *file_private) 1548 { 1549 struct drm_syncobj_array *args = data; 1550 struct drm_syncobj **syncobjs; 1551 uint32_t i; 1552 int ret; 1553 1554 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1555 return -EOPNOTSUPP; 1556 1557 if (args->pad != 0) 1558 return -EINVAL; 1559 1560 if (args->count_handles == 0) 1561 return -EINVAL; 1562 1563 ret = drm_syncobj_array_find(file_private, 1564 u64_to_user_ptr(args->handles), 1565 args->count_handles, 1566 &syncobjs); 1567 if (ret < 0) 1568 return ret; 1569 1570 for (i = 0; i < args->count_handles; i++) { 1571 ret = drm_syncobj_assign_null_handle(syncobjs[i]); 1572 if (ret < 0) 1573 break; 1574 } 1575 1576 drm_syncobj_array_free(syncobjs, args->count_handles); 1577 1578 return ret; 1579 } 1580 1581 int 1582 drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data, 1583 struct drm_file *file_private) 1584 { 1585 struct drm_syncobj_timeline_array *args = data; 1586 struct drm_syncobj **syncobjs; 1587 struct dma_fence_chain **chains; 1588 uint64_t *points; 1589 uint32_t i, j; 1590 int ret; 1591 1592 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1593 return -EOPNOTSUPP; 1594 1595 if (args->flags != 0) 1596 return -EINVAL; 1597 1598 if (args->count_handles == 0) 1599 return -EINVAL; 1600 1601 ret = drm_syncobj_array_find(file_private, 1602 u64_to_user_ptr(args->handles), 1603 args->count_handles, 1604 &syncobjs); 1605 if (ret < 0) 1606 return ret; 1607 1608 points = kmalloc_array(args->count_handles, sizeof(*points), 1609 GFP_KERNEL); 1610 if (!points) { 1611 ret = -ENOMEM; 1612 goto out; 1613 } 1614 if (!u64_to_user_ptr(args->points)) { 1615 memset(points, 0, args->count_handles * sizeof(uint64_t)); 1616 } else if (copy_from_user(points, u64_to_user_ptr(args->points), 1617 sizeof(uint64_t) * args->count_handles)) { 1618 ret = -EFAULT; 1619 goto err_points; 1620 } 1621 1622 chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL); 1623 if (!chains) { 1624 ret = -ENOMEM; 1625 goto err_points; 1626 } 1627 for (i = 0; i < args->count_handles; i++) { 1628 chains[i] = dma_fence_chain_alloc(); 1629 if (!chains[i]) { 1630 for (j = 0; j < i; j++) 1631 dma_fence_chain_free(chains[j]); 1632 ret = -ENOMEM; 1633 goto err_chains; 1634 } 1635 } 1636 1637 for (i = 0; i < args->count_handles; i++) { 1638 struct dma_fence *fence = dma_fence_get_stub(); 1639 1640 drm_syncobj_add_point(syncobjs[i], chains[i], 1641 fence, points[i]); 1642 dma_fence_put(fence); 1643 } 1644 err_chains: 1645 kfree(chains); 1646 err_points: 1647 kfree(points); 1648 out: 1649 drm_syncobj_array_free(syncobjs, args->count_handles); 1650 1651 return ret; 1652 } 1653 1654 int drm_syncobj_query_ioctl(struct drm_device *dev, void *data, 1655 struct drm_file *file_private) 1656 { 1657 struct drm_syncobj_timeline_array *args = data; 1658 struct drm_syncobj **syncobjs; 1659 uint64_t __user *points = u64_to_user_ptr(args->points); 1660 uint32_t i; 1661 int ret; 1662 1663 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1664 return -EOPNOTSUPP; 1665 1666 if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) 1667 return -EINVAL; 1668 1669 if (args->count_handles == 0) 1670 return -EINVAL; 1671 1672 ret = drm_syncobj_array_find(file_private, 1673 u64_to_user_ptr(args->handles), 1674 args->count_handles, 1675 &syncobjs); 1676 if (ret < 0) 1677 return ret; 1678 1679 for (i = 0; i < args->count_handles; i++) { 1680 struct dma_fence_chain *chain; 1681 struct dma_fence *fence; 1682 uint64_t point; 1683 1684 fence = drm_syncobj_fence_get(syncobjs[i]); 1685 chain = to_dma_fence_chain(fence); 1686 if (chain) { 1687 struct dma_fence *iter, *last_signaled = 1688 dma_fence_get(fence); 1689 1690 if (args->flags & 1691 DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) { 1692 point = fence->seqno; 1693 } else { 1694 dma_fence_chain_for_each(iter, fence) { 1695 if (iter->context != fence->context) { 1696 dma_fence_put(iter); 1697 /* It is most likely that timeline has 1698 * unorder points. */ 1699 break; 1700 } 1701 dma_fence_put(last_signaled); 1702 last_signaled = dma_fence_get(iter); 1703 } 1704 point = dma_fence_is_signaled(last_signaled) ? 1705 last_signaled->seqno : 1706 to_dma_fence_chain(last_signaled)->prev_seqno; 1707 } 1708 dma_fence_put(last_signaled); 1709 } else { 1710 point = 0; 1711 } 1712 dma_fence_put(fence); 1713 ret = copy_to_user(&points[i], &point, sizeof(uint64_t)); 1714 ret = ret ? -EFAULT : 0; 1715 if (ret) 1716 break; 1717 } 1718 drm_syncobj_array_free(syncobjs, args->count_handles); 1719 1720 return ret; 1721 } 1722