1 /* 2 * Copyright © 2008 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 */ 27 28 #include <linux/dma-buf.h> 29 #include <linux/export.h> 30 #include <linux/file.h> 31 #include <linux/fs.h> 32 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 33 #include <linux/fs_context.h> 34 #endif 35 #include <linux/iosys-map.h> 36 #include <linux/mem_encrypt.h> 37 #include <linux/mm.h> 38 #include <linux/mman.h> 39 #include <linux/module.h> 40 #include <linux/pagemap.h> 41 #include <linux/pagevec.h> 42 #include <linux/sched/mm.h> 43 #include <linux/shmem_fs.h> 44 #include <linux/slab.h> 45 #include <linux/string_helpers.h> 46 #include <linux/types.h> 47 #include <linux/uaccess.h> 48 49 #include <drm/drm.h> 50 #include <drm/drm_device.h> 51 #include <drm/drm_drv.h> 52 #include <drm/drm_file.h> 53 #include <drm/drm_gem.h> 54 #include <drm/drm_managed.h> 55 #include <drm/drm_print.h> 56 #include <drm/drm_vma_manager.h> 57 58 #include "drm_internal.h" 59 60 /** @file drm_gem.c 61 * 62 * This file provides some of the base ioctls and library routines for 63 * the graphics memory manager implemented by each device driver. 64 * 65 * Because various devices have different requirements in terms of 66 * synchronization and migration strategies, implementing that is left up to 67 * the driver, and all that the general API provides should be generic -- 68 * allocating objects, reading/writing data with the cpu, freeing objects. 69 * Even there, platform-dependent optimizations for reading/writing data with 70 * the CPU mean we'll likely hook those out to driver-specific calls. However, 71 * the DRI2 implementation wants to have at least allocate/mmap be generic. 72 * 73 * The goal was to have swap-backed object allocation managed through 74 * struct file. However, file descriptors as handles to a struct file have 75 * two major failings: 76 * - Process limits prevent more than 1024 or so being used at a time by 77 * default. 78 * - Inability to allocate high fds will aggravate the X Server's select() 79 * handling, and likely that of many GL client applications as well. 80 * 81 * This led to a plan of using our own integer IDs (called handles, following 82 * DRM terminology) to mimic fds, and implement the fd syscalls we need as 83 * ioctls. The objects themselves will still include the struct file so 84 * that we can transition to fds if the required kernel infrastructure shows 85 * up at a later date, and as our interface with shmfs for memory allocation. 86 */ 87 88 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 89 static void drm_gem_huge_mnt_free(struct drm_device *dev, void *data) 90 { 91 kern_unmount(dev->huge_mnt); 92 } 93 94 /** 95 * drm_gem_huge_mnt_create - Create, mount and use a huge tmpfs mountpoint 96 * @dev: DRM device that will use the huge tmpfs mountpoint 97 * @value: huge tmpfs mount option value 98 * 99 * This function creates and mounts a dedicated huge tmpfs mountpoint for the 100 * lifetime of the DRM device @dev which is used at GEM object initialization 101 * with drm_gem_object_init(). 102 * 103 * The most common option for @value is "within_size" which only allocates huge 104 * pages if the page will be fully within the GEM object size. "always", 105 * "advise" and "never" are supported too but the latter would just create a 106 * mountpoint similar to the default one (`shm_mnt`). See shmemfs and 107 * Transparent Hugepage for more information. 108 * 109 * Returns: 110 * 0 on success or a negative error code on failure. 111 */ 112 int drm_gem_huge_mnt_create(struct drm_device *dev, const char *value) 113 { 114 struct file_system_type *type; 115 struct fs_context *fc; 116 int ret; 117 118 if (unlikely(drm_gem_get_huge_mnt(dev))) 119 return 0; 120 121 type = get_fs_type("tmpfs"); 122 if (unlikely(!type)) 123 return -EOPNOTSUPP; 124 fc = fs_context_for_mount(type, SB_KERNMOUNT); 125 if (IS_ERR(fc)) 126 return PTR_ERR(fc); 127 ret = vfs_parse_fs_string(fc, "source", "tmpfs"); 128 if (unlikely(ret)) 129 return -ENOPARAM; 130 ret = vfs_parse_fs_string(fc, "huge", value); 131 if (unlikely(ret)) 132 return -ENOPARAM; 133 134 dev->huge_mnt = fc_mount_longterm(fc); 135 put_fs_context(fc); 136 137 return drmm_add_action_or_reset(dev, drm_gem_huge_mnt_free, NULL); 138 } 139 EXPORT_SYMBOL_GPL(drm_gem_huge_mnt_create); 140 #endif 141 142 static void 143 drm_gem_init_release(struct drm_device *dev, void *ptr) 144 { 145 drm_vma_offset_manager_destroy(dev->vma_offset_manager); 146 } 147 148 /** 149 * drm_gem_init - Initialize the GEM device fields 150 * @dev: drm_devic structure to initialize 151 */ 152 int 153 drm_gem_init(struct drm_device *dev) 154 { 155 struct drm_vma_offset_manager *vma_offset_manager; 156 157 mutex_init(&dev->object_name_lock); 158 idr_init_base(&dev->object_name_idr, 1); 159 160 vma_offset_manager = drmm_kzalloc(dev, sizeof(*vma_offset_manager), 161 GFP_KERNEL); 162 if (!vma_offset_manager) 163 return -ENOMEM; 164 165 dev->vma_offset_manager = vma_offset_manager; 166 drm_vma_offset_manager_init(vma_offset_manager, 167 DRM_FILE_PAGE_OFFSET_START, 168 DRM_FILE_PAGE_OFFSET_SIZE); 169 170 return drmm_add_action(dev, drm_gem_init_release, NULL); 171 } 172 173 /** 174 * drm_gem_object_init - initialize an allocated shmem-backed GEM object 175 * 176 * @dev: drm_device the object should be initialized for 177 * @obj: drm_gem_object to initialize 178 * @size: object size 179 * 180 * Initialize an already allocated GEM object of the specified size with 181 * shmfs backing store. A huge mountpoint can be used by calling 182 * drm_gem_huge_mnt_create() beforehand. 183 */ 184 int drm_gem_object_init(struct drm_device *dev, struct drm_gem_object *obj, 185 size_t size) 186 { 187 struct vfsmount *huge_mnt; 188 struct file *filp; 189 190 drm_gem_private_object_init(dev, obj, size); 191 192 huge_mnt = drm_gem_get_huge_mnt(dev); 193 if (huge_mnt) 194 filp = shmem_file_setup_with_mnt(huge_mnt, "drm mm object", 195 size, VM_NORESERVE); 196 else 197 filp = shmem_file_setup("drm mm object", size, VM_NORESERVE); 198 199 if (IS_ERR(filp)) 200 return PTR_ERR(filp); 201 202 obj->filp = filp; 203 204 return 0; 205 } 206 EXPORT_SYMBOL(drm_gem_object_init); 207 208 /** 209 * drm_gem_private_object_init - initialize an allocated private GEM object 210 * @dev: drm_device the object should be initialized for 211 * @obj: drm_gem_object to initialize 212 * @size: object size 213 * 214 * Initialize an already allocated GEM object of the specified size with 215 * no GEM provided backing store. Instead the caller is responsible for 216 * backing the object and handling it. 217 */ 218 void drm_gem_private_object_init(struct drm_device *dev, 219 struct drm_gem_object *obj, size_t size) 220 { 221 BUG_ON((size & (PAGE_SIZE - 1)) != 0); 222 223 obj->dev = dev; 224 obj->filp = NULL; 225 226 kref_init(&obj->refcount); 227 obj->handle_count = 0; 228 obj->size = size; 229 mutex_init(&obj->gpuva.lock); 230 dma_resv_init(&obj->_resv); 231 if (!obj->resv) 232 obj->resv = &obj->_resv; 233 234 if (drm_core_check_feature(dev, DRIVER_GEM_GPUVA)) 235 drm_gem_gpuva_init(obj); 236 237 drm_vma_node_reset(&obj->vma_node); 238 INIT_LIST_HEAD(&obj->lru_node); 239 } 240 EXPORT_SYMBOL(drm_gem_private_object_init); 241 242 /** 243 * drm_gem_private_object_fini - Finalize a failed drm_gem_object 244 * @obj: drm_gem_object 245 * 246 * Uninitialize an already allocated GEM object when it initialized failed 247 */ 248 void drm_gem_private_object_fini(struct drm_gem_object *obj) 249 { 250 WARN_ON(obj->dma_buf); 251 252 dma_resv_fini(&obj->_resv); 253 mutex_destroy(&obj->gpuva.lock); 254 } 255 EXPORT_SYMBOL(drm_gem_private_object_fini); 256 257 static void drm_gem_object_handle_get(struct drm_gem_object *obj) 258 { 259 struct drm_device *dev = obj->dev; 260 261 drm_WARN_ON(dev, !mutex_is_locked(&dev->object_name_lock)); 262 263 if (obj->handle_count++ == 0) 264 drm_gem_object_get(obj); 265 } 266 267 /** 268 * drm_gem_object_handle_get_if_exists_unlocked - acquire reference on user-space handle, if any 269 * @obj: GEM object 270 * 271 * Acquires a reference on the GEM buffer object's handle. Required to keep 272 * the GEM object alive. Call drm_gem_object_handle_put_if_exists_unlocked() 273 * to release the reference. Does nothing if the buffer object has no handle. 274 * 275 * Returns: 276 * True if a handle exists, or false otherwise 277 */ 278 bool drm_gem_object_handle_get_if_exists_unlocked(struct drm_gem_object *obj) 279 { 280 struct drm_device *dev = obj->dev; 281 282 guard(mutex)(&dev->object_name_lock); 283 284 /* 285 * First ref taken during GEM object creation, if any. Some 286 * drivers set up internal framebuffers with GEM objects that 287 * do not have a GEM handle. Hence, this counter can be zero. 288 */ 289 if (!obj->handle_count) 290 return false; 291 292 drm_gem_object_handle_get(obj); 293 294 return true; 295 } 296 297 /** 298 * drm_gem_object_handle_free - release resources bound to userspace handles 299 * @obj: GEM object to clean up. 300 * 301 * Called after the last handle to the object has been closed 302 * 303 * Removes any name for the object. Note that this must be 304 * called before drm_gem_object_free or we'll be touching 305 * freed memory 306 */ 307 static void drm_gem_object_handle_free(struct drm_gem_object *obj) 308 { 309 struct drm_device *dev = obj->dev; 310 311 /* Remove any name for this object */ 312 if (obj->name) { 313 idr_remove(&dev->object_name_idr, obj->name); 314 obj->name = 0; 315 } 316 } 317 318 static void drm_gem_object_exported_dma_buf_free(struct drm_gem_object *obj) 319 { 320 /* Unbreak the reference cycle if we have an exported dma_buf. */ 321 if (obj->dma_buf) { 322 dma_buf_put(obj->dma_buf); 323 obj->dma_buf = NULL; 324 } 325 } 326 327 /** 328 * drm_gem_object_handle_put_unlocked - releases reference on user-space handle 329 * @obj: GEM object 330 * 331 * Releases a reference on the GEM buffer object's handle. Possibly releases 332 * the GEM buffer object and associated dma-buf objects. 333 */ 334 void drm_gem_object_handle_put_unlocked(struct drm_gem_object *obj) 335 { 336 struct drm_device *dev = obj->dev; 337 bool final = false; 338 339 if (drm_WARN_ON(dev, READ_ONCE(obj->handle_count) == 0)) 340 return; 341 342 /* 343 * Must bump handle count first as this may be the last 344 * ref, in which case the object would disappear before 345 * we checked for a name. 346 */ 347 348 mutex_lock(&dev->object_name_lock); 349 if (--obj->handle_count == 0) { 350 drm_gem_object_handle_free(obj); 351 drm_gem_object_exported_dma_buf_free(obj); 352 final = true; 353 } 354 mutex_unlock(&dev->object_name_lock); 355 356 if (final) 357 drm_gem_object_put(obj); 358 } 359 360 /* 361 * Called at device or object close to release the file's 362 * handle references on objects. 363 */ 364 static int 365 drm_gem_object_release_handle(int id, void *ptr, void *data) 366 { 367 struct drm_file *file_priv = data; 368 struct drm_gem_object *obj = ptr; 369 370 if (drm_WARN_ON(obj->dev, !data)) 371 return 0; 372 373 if (obj->funcs->close) 374 obj->funcs->close(obj, file_priv); 375 376 mutex_lock(&file_priv->prime.lock); 377 378 drm_prime_remove_buf_handle(&file_priv->prime, id); 379 380 mutex_unlock(&file_priv->prime.lock); 381 382 drm_vma_node_revoke(&obj->vma_node, file_priv); 383 384 drm_gem_object_handle_put_unlocked(obj); 385 386 return 0; 387 } 388 389 /** 390 * drm_gem_handle_delete - deletes the given file-private handle 391 * @filp: drm file-private structure to use for the handle look up 392 * @handle: userspace handle to delete 393 * 394 * Removes the GEM handle from the @filp lookup table which has been added with 395 * drm_gem_handle_create(). If this is the last handle also cleans up linked 396 * resources like GEM names. 397 */ 398 int 399 drm_gem_handle_delete(struct drm_file *filp, u32 handle) 400 { 401 struct drm_gem_object *obj; 402 403 spin_lock(&filp->table_lock); 404 405 /* Check if we currently have a reference on the object */ 406 obj = idr_replace(&filp->object_idr, NULL, handle); 407 spin_unlock(&filp->table_lock); 408 if (IS_ERR_OR_NULL(obj)) 409 return -EINVAL; 410 411 /* Release driver's reference and decrement refcount. */ 412 drm_gem_object_release_handle(handle, obj, filp); 413 414 /* And finally make the handle available for future allocations. */ 415 spin_lock(&filp->table_lock); 416 idr_remove(&filp->object_idr, handle); 417 spin_unlock(&filp->table_lock); 418 419 return 0; 420 } 421 EXPORT_SYMBOL(drm_gem_handle_delete); 422 423 /** 424 * drm_gem_dumb_map_offset - return the fake mmap offset for a gem object 425 * @file: drm file-private structure containing the gem object 426 * @dev: corresponding drm_device 427 * @handle: gem object handle 428 * @offset: return location for the fake mmap offset 429 * 430 * This implements the &drm_driver.dumb_map_offset kms driver callback for 431 * drivers which use gem to manage their backing storage. 432 * 433 * Returns: 434 * 0 on success or a negative error code on failure. 435 */ 436 int drm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev, 437 u32 handle, u64 *offset) 438 { 439 struct drm_gem_object *obj; 440 int ret; 441 442 obj = drm_gem_object_lookup(file, handle); 443 if (!obj) 444 return -ENOENT; 445 446 /* Don't allow imported objects to be mapped */ 447 if (drm_gem_is_imported(obj)) { 448 ret = -EINVAL; 449 goto out; 450 } 451 452 ret = drm_gem_create_mmap_offset(obj); 453 if (ret) 454 goto out; 455 456 *offset = drm_vma_node_offset_addr(&obj->vma_node); 457 out: 458 drm_gem_object_put(obj); 459 460 return ret; 461 } 462 EXPORT_SYMBOL_GPL(drm_gem_dumb_map_offset); 463 464 /** 465 * drm_gem_handle_create_tail - internal functions to create a handle 466 * @file_priv: drm file-private structure to register the handle for 467 * @obj: object to register 468 * @handlep: pointer to return the created handle to the caller 469 * 470 * This expects the &drm_device.object_name_lock to be held already and will 471 * drop it before returning. Used to avoid races in establishing new handles 472 * when importing an object from either an flink name or a dma-buf. 473 * 474 * Handles must be release again through drm_gem_handle_delete(). This is done 475 * when userspace closes @file_priv for all attached handles, or through the 476 * GEM_CLOSE ioctl for individual handles. 477 */ 478 int 479 drm_gem_handle_create_tail(struct drm_file *file_priv, 480 struct drm_gem_object *obj, 481 u32 *handlep) 482 { 483 struct drm_device *dev = obj->dev; 484 u32 handle; 485 int ret; 486 487 WARN_ON(!mutex_is_locked(&dev->object_name_lock)); 488 489 drm_gem_object_handle_get(obj); 490 491 /* 492 * Get the user-visible handle using idr. Preload and perform 493 * allocation under our spinlock. 494 */ 495 idr_preload(GFP_KERNEL); 496 spin_lock(&file_priv->table_lock); 497 498 ret = idr_alloc(&file_priv->object_idr, NULL, 1, 0, GFP_NOWAIT); 499 500 spin_unlock(&file_priv->table_lock); 501 idr_preload_end(); 502 503 mutex_unlock(&dev->object_name_lock); 504 if (ret < 0) 505 goto err_unref; 506 507 handle = ret; 508 509 ret = drm_vma_node_allow(&obj->vma_node, file_priv); 510 if (ret) 511 goto err_remove; 512 513 if (obj->funcs->open) { 514 ret = obj->funcs->open(obj, file_priv); 515 if (ret) 516 goto err_revoke; 517 } 518 519 /* mirrors drm_gem_handle_delete to avoid races */ 520 spin_lock(&file_priv->table_lock); 521 obj = idr_replace(&file_priv->object_idr, obj, handle); 522 WARN_ON(obj != NULL); 523 spin_unlock(&file_priv->table_lock); 524 *handlep = handle; 525 return 0; 526 527 err_revoke: 528 drm_vma_node_revoke(&obj->vma_node, file_priv); 529 err_remove: 530 spin_lock(&file_priv->table_lock); 531 idr_remove(&file_priv->object_idr, handle); 532 spin_unlock(&file_priv->table_lock); 533 err_unref: 534 drm_gem_object_handle_put_unlocked(obj); 535 return ret; 536 } 537 538 /** 539 * drm_gem_handle_create - create a gem handle for an object 540 * @file_priv: drm file-private structure to register the handle for 541 * @obj: object to register 542 * @handlep: pointer to return the created handle to the caller 543 * 544 * Create a handle for this object. This adds a handle reference to the object, 545 * which includes a regular reference count. Callers will likely want to 546 * dereference the object afterwards. 547 * 548 * Since this publishes @obj to userspace it must be fully set up by this point, 549 * drivers must call this last in their buffer object creation callbacks. 550 */ 551 int drm_gem_handle_create(struct drm_file *file_priv, 552 struct drm_gem_object *obj, 553 u32 *handlep) 554 { 555 mutex_lock(&obj->dev->object_name_lock); 556 557 return drm_gem_handle_create_tail(file_priv, obj, handlep); 558 } 559 EXPORT_SYMBOL(drm_gem_handle_create); 560 561 562 /** 563 * drm_gem_free_mmap_offset - release a fake mmap offset for an object 564 * @obj: obj in question 565 * 566 * This routine frees fake offsets allocated by drm_gem_create_mmap_offset(). 567 * 568 * Note that drm_gem_object_release() already calls this function, so drivers 569 * don't have to take care of releasing the mmap offset themselves when freeing 570 * the GEM object. 571 */ 572 void 573 drm_gem_free_mmap_offset(struct drm_gem_object *obj) 574 { 575 struct drm_device *dev = obj->dev; 576 577 drm_vma_offset_remove(dev->vma_offset_manager, &obj->vma_node); 578 } 579 EXPORT_SYMBOL(drm_gem_free_mmap_offset); 580 581 /** 582 * drm_gem_create_mmap_offset_size - create a fake mmap offset for an object 583 * @obj: obj in question 584 * @size: the virtual size 585 * 586 * GEM memory mapping works by handing back to userspace a fake mmap offset 587 * it can use in a subsequent mmap(2) call. The DRM core code then looks 588 * up the object based on the offset and sets up the various memory mapping 589 * structures. 590 * 591 * This routine allocates and attaches a fake offset for @obj, in cases where 592 * the virtual size differs from the physical size (ie. &drm_gem_object.size). 593 * Otherwise just use drm_gem_create_mmap_offset(). 594 * 595 * This function is idempotent and handles an already allocated mmap offset 596 * transparently. Drivers do not need to check for this case. 597 */ 598 int 599 drm_gem_create_mmap_offset_size(struct drm_gem_object *obj, size_t size) 600 { 601 struct drm_device *dev = obj->dev; 602 603 return drm_vma_offset_add(dev->vma_offset_manager, &obj->vma_node, 604 size / PAGE_SIZE); 605 } 606 EXPORT_SYMBOL(drm_gem_create_mmap_offset_size); 607 608 /** 609 * drm_gem_create_mmap_offset - create a fake mmap offset for an object 610 * @obj: obj in question 611 * 612 * GEM memory mapping works by handing back to userspace a fake mmap offset 613 * it can use in a subsequent mmap(2) call. The DRM core code then looks 614 * up the object based on the offset and sets up the various memory mapping 615 * structures. 616 * 617 * This routine allocates and attaches a fake offset for @obj. 618 * 619 * Drivers can call drm_gem_free_mmap_offset() before freeing @obj to release 620 * the fake offset again. 621 */ 622 int drm_gem_create_mmap_offset(struct drm_gem_object *obj) 623 { 624 return drm_gem_create_mmap_offset_size(obj, obj->size); 625 } 626 EXPORT_SYMBOL(drm_gem_create_mmap_offset); 627 628 /* 629 * Move folios to appropriate lru and release the folios, decrementing the 630 * ref count of those folios. 631 */ 632 static void drm_gem_check_release_batch(struct folio_batch *fbatch) 633 { 634 check_move_unevictable_folios(fbatch); 635 __folio_batch_release(fbatch); 636 cond_resched(); 637 } 638 639 /** 640 * drm_gem_get_pages - helper to allocate backing pages for a GEM object 641 * from shmem 642 * @obj: obj in question 643 * 644 * This reads the page-array of the shmem-backing storage of the given gem 645 * object. An array of pages is returned. If a page is not allocated or 646 * swapped-out, this will allocate/swap-in the required pages. Note that the 647 * whole object is covered by the page-array and pinned in memory. 648 * 649 * Use drm_gem_put_pages() to release the array and unpin all pages. 650 * 651 * This uses the GFP-mask set on the shmem-mapping (see mapping_set_gfp_mask()). 652 * If you require other GFP-masks, you have to do those allocations yourself. 653 * 654 * Note that you are not allowed to change gfp-zones during runtime. That is, 655 * shmem_read_mapping_page_gfp() must be called with the same gfp_zone(gfp) as 656 * set during initialization. If you have special zone constraints, set them 657 * after drm_gem_object_init() via mapping_set_gfp_mask(). shmem-core takes care 658 * to keep pages in the required zone during swap-in. 659 * 660 * This function is only valid on objects initialized with 661 * drm_gem_object_init(), but not for those initialized with 662 * drm_gem_private_object_init() only. 663 */ 664 struct page **drm_gem_get_pages(struct drm_gem_object *obj) 665 { 666 struct address_space *mapping; 667 struct page **pages; 668 struct folio *folio; 669 struct folio_batch fbatch; 670 unsigned long i, j, npages; 671 672 if (WARN_ON(!obj->filp)) 673 return ERR_PTR(-EINVAL); 674 675 /* This is the shared memory object that backs the GEM resource */ 676 mapping = obj->filp->f_mapping; 677 678 /* We already BUG_ON() for non-page-aligned sizes in 679 * drm_gem_object_init(), so we should never hit this unless 680 * driver author is doing something really wrong: 681 */ 682 WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0); 683 684 npages = obj->size >> PAGE_SHIFT; 685 686 pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL); 687 if (pages == NULL) 688 return ERR_PTR(-ENOMEM); 689 690 mapping_set_unevictable(mapping); 691 692 i = 0; 693 while (i < npages) { 694 unsigned long nr; 695 folio = shmem_read_folio_gfp(mapping, i, 696 mapping_gfp_mask(mapping)); 697 if (IS_ERR(folio)) 698 goto fail; 699 nr = min(npages - i, folio_nr_pages(folio)); 700 for (j = 0; j < nr; j++, i++) 701 pages[i] = folio_file_page(folio, i); 702 703 /* Make sure shmem keeps __GFP_DMA32 allocated pages in the 704 * correct region during swapin. Note that this requires 705 * __GFP_DMA32 to be set in mapping_gfp_mask(inode->i_mapping) 706 * so shmem can relocate pages during swapin if required. 707 */ 708 BUG_ON(mapping_gfp_constraint(mapping, __GFP_DMA32) && 709 (folio_pfn(folio) >= 0x00100000UL)); 710 } 711 712 return pages; 713 714 fail: 715 mapping_clear_unevictable(mapping); 716 folio_batch_init(&fbatch); 717 j = 0; 718 while (j < i) { 719 struct folio *f = page_folio(pages[j]); 720 if (!folio_batch_add(&fbatch, f)) 721 drm_gem_check_release_batch(&fbatch); 722 j += folio_nr_pages(f); 723 } 724 if (fbatch.nr) 725 drm_gem_check_release_batch(&fbatch); 726 727 kvfree(pages); 728 return ERR_CAST(folio); 729 } 730 EXPORT_SYMBOL(drm_gem_get_pages); 731 732 /** 733 * drm_gem_put_pages - helper to free backing pages for a GEM object 734 * @obj: obj in question 735 * @pages: pages to free 736 * @dirty: if true, pages will be marked as dirty 737 * @accessed: if true, the pages will be marked as accessed 738 */ 739 void drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages, 740 bool dirty, bool accessed) 741 { 742 int i, npages; 743 struct address_space *mapping; 744 struct folio_batch fbatch; 745 746 mapping = file_inode(obj->filp)->i_mapping; 747 mapping_clear_unevictable(mapping); 748 749 /* We already BUG_ON() for non-page-aligned sizes in 750 * drm_gem_object_init(), so we should never hit this unless 751 * driver author is doing something really wrong: 752 */ 753 WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0); 754 755 npages = obj->size >> PAGE_SHIFT; 756 757 folio_batch_init(&fbatch); 758 for (i = 0; i < npages; i++) { 759 struct folio *folio; 760 761 if (!pages[i]) 762 continue; 763 folio = page_folio(pages[i]); 764 765 if (dirty) 766 folio_mark_dirty(folio); 767 768 if (accessed) 769 folio_mark_accessed(folio); 770 771 /* Undo the reference we took when populating the table */ 772 if (!folio_batch_add(&fbatch, folio)) 773 drm_gem_check_release_batch(&fbatch); 774 i += folio_nr_pages(folio) - 1; 775 } 776 if (folio_batch_count(&fbatch)) 777 drm_gem_check_release_batch(&fbatch); 778 779 kvfree(pages); 780 } 781 EXPORT_SYMBOL(drm_gem_put_pages); 782 783 static int objects_lookup(struct drm_file *filp, u32 *handle, int count, 784 struct drm_gem_object **objs) 785 { 786 int i, ret = 0; 787 struct drm_gem_object *obj; 788 789 spin_lock(&filp->table_lock); 790 791 for (i = 0; i < count; i++) { 792 /* Check if we currently have a reference on the object */ 793 obj = idr_find(&filp->object_idr, handle[i]); 794 if (!obj) { 795 ret = -ENOENT; 796 break; 797 } 798 drm_gem_object_get(obj); 799 objs[i] = obj; 800 } 801 spin_unlock(&filp->table_lock); 802 803 return ret; 804 } 805 806 /** 807 * drm_gem_objects_lookup - look up GEM objects from an array of handles 808 * @filp: DRM file private date 809 * @bo_handles: user pointer to array of userspace handle 810 * @count: size of handle array 811 * @objs_out: returned pointer to array of drm_gem_object pointers 812 * 813 * Takes an array of userspace handles and returns a newly allocated array of 814 * GEM objects. 815 * 816 * For a single handle lookup, use drm_gem_object_lookup(). 817 * 818 * Returns: 819 * @objs filled in with GEM object pointers. Returned GEM objects need to be 820 * released with drm_gem_object_put(). -ENOENT is returned on a lookup 821 * failure. 0 is returned on success. 822 * 823 */ 824 int drm_gem_objects_lookup(struct drm_file *filp, void __user *bo_handles, 825 int count, struct drm_gem_object ***objs_out) 826 { 827 struct drm_gem_object **objs; 828 u32 *handles; 829 int ret; 830 831 if (!count) 832 return 0; 833 834 objs = kvmalloc_array(count, sizeof(struct drm_gem_object *), 835 GFP_KERNEL | __GFP_ZERO); 836 if (!objs) 837 return -ENOMEM; 838 839 *objs_out = objs; 840 841 handles = vmemdup_array_user(bo_handles, count, sizeof(u32)); 842 if (IS_ERR(handles)) 843 return PTR_ERR(handles); 844 845 ret = objects_lookup(filp, handles, count, objs); 846 kvfree(handles); 847 return ret; 848 849 } 850 EXPORT_SYMBOL(drm_gem_objects_lookup); 851 852 /** 853 * drm_gem_object_lookup - look up a GEM object from its handle 854 * @filp: DRM file private date 855 * @handle: userspace handle 856 * 857 * If looking up an array of handles, use drm_gem_objects_lookup(). 858 * 859 * Returns: 860 * A reference to the object named by the handle if such exists on @filp, NULL 861 * otherwise. 862 */ 863 struct drm_gem_object * 864 drm_gem_object_lookup(struct drm_file *filp, u32 handle) 865 { 866 struct drm_gem_object *obj = NULL; 867 868 objects_lookup(filp, &handle, 1, &obj); 869 return obj; 870 } 871 EXPORT_SYMBOL(drm_gem_object_lookup); 872 873 /** 874 * drm_gem_dma_resv_wait - Wait on GEM object's reservation's objects 875 * shared and/or exclusive fences. 876 * @filep: DRM file private date 877 * @handle: userspace handle 878 * @wait_all: if true, wait on all fences, else wait on just exclusive fence 879 * @timeout: timeout value in jiffies or zero to return immediately 880 * 881 * Returns: 882 * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or 883 * greater than 0 on success. 884 */ 885 long drm_gem_dma_resv_wait(struct drm_file *filep, u32 handle, 886 bool wait_all, unsigned long timeout) 887 { 888 struct drm_device *dev = filep->minor->dev; 889 struct drm_gem_object *obj; 890 long ret; 891 892 obj = drm_gem_object_lookup(filep, handle); 893 if (!obj) { 894 drm_dbg_core(dev, "Failed to look up GEM BO %d\n", handle); 895 return -EINVAL; 896 } 897 898 ret = dma_resv_wait_timeout(obj->resv, dma_resv_usage_rw(wait_all), 899 true, timeout); 900 if (ret == 0) 901 ret = -ETIME; 902 else if (ret > 0) 903 ret = 0; 904 905 drm_gem_object_put(obj); 906 907 return ret; 908 } 909 EXPORT_SYMBOL(drm_gem_dma_resv_wait); 910 911 int 912 drm_gem_close_ioctl(struct drm_device *dev, void *data, 913 struct drm_file *file_priv) 914 { 915 struct drm_gem_close *args = data; 916 int ret; 917 918 if (!drm_core_check_feature(dev, DRIVER_GEM)) 919 return -EOPNOTSUPP; 920 921 ret = drm_gem_handle_delete(file_priv, args->handle); 922 923 return ret; 924 } 925 926 int 927 drm_gem_flink_ioctl(struct drm_device *dev, void *data, 928 struct drm_file *file_priv) 929 { 930 struct drm_gem_flink *args = data; 931 struct drm_gem_object *obj; 932 int ret; 933 934 if (!drm_core_check_feature(dev, DRIVER_GEM)) 935 return -EOPNOTSUPP; 936 937 obj = drm_gem_object_lookup(file_priv, args->handle); 938 if (obj == NULL) 939 return -ENOENT; 940 941 mutex_lock(&dev->object_name_lock); 942 /* prevent races with concurrent gem_close. */ 943 if (obj->handle_count == 0) { 944 ret = -ENOENT; 945 goto err; 946 } 947 948 if (!obj->name) { 949 ret = idr_alloc(&dev->object_name_idr, obj, 1, 0, GFP_KERNEL); 950 if (ret < 0) 951 goto err; 952 953 obj->name = ret; 954 } 955 956 args->name = (uint64_t) obj->name; 957 ret = 0; 958 959 err: 960 mutex_unlock(&dev->object_name_lock); 961 drm_gem_object_put(obj); 962 return ret; 963 } 964 965 int 966 drm_gem_open_ioctl(struct drm_device *dev, void *data, 967 struct drm_file *file_priv) 968 { 969 struct drm_gem_open *args = data; 970 struct drm_gem_object *obj; 971 int ret; 972 u32 handle; 973 974 if (!drm_core_check_feature(dev, DRIVER_GEM)) 975 return -EOPNOTSUPP; 976 977 mutex_lock(&dev->object_name_lock); 978 obj = idr_find(&dev->object_name_idr, (int) args->name); 979 if (obj) { 980 drm_gem_object_get(obj); 981 } else { 982 mutex_unlock(&dev->object_name_lock); 983 return -ENOENT; 984 } 985 986 /* drm_gem_handle_create_tail unlocks dev->object_name_lock. */ 987 ret = drm_gem_handle_create_tail(file_priv, obj, &handle); 988 if (ret) 989 goto err; 990 991 args->handle = handle; 992 args->size = obj->size; 993 994 err: 995 drm_gem_object_put(obj); 996 return ret; 997 } 998 999 int drm_gem_change_handle_ioctl(struct drm_device *dev, void *data, 1000 struct drm_file *file_priv) 1001 { 1002 struct drm_gem_change_handle *args = data; 1003 struct drm_gem_object *obj; 1004 int ret; 1005 1006 if (!drm_core_check_feature(dev, DRIVER_GEM)) 1007 return -EOPNOTSUPP; 1008 1009 obj = drm_gem_object_lookup(file_priv, args->handle); 1010 if (!obj) 1011 return -ENOENT; 1012 1013 if (args->handle == args->new_handle) { 1014 ret = 0; 1015 goto out; 1016 } 1017 1018 mutex_lock(&file_priv->prime.lock); 1019 1020 spin_lock(&file_priv->table_lock); 1021 ret = idr_alloc(&file_priv->object_idr, obj, 1022 args->new_handle, args->new_handle + 1, GFP_NOWAIT); 1023 spin_unlock(&file_priv->table_lock); 1024 1025 if (ret < 0) 1026 goto out_unlock; 1027 1028 if (obj->dma_buf) { 1029 ret = drm_prime_add_buf_handle(&file_priv->prime, obj->dma_buf, args->new_handle); 1030 if (ret < 0) { 1031 spin_lock(&file_priv->table_lock); 1032 idr_remove(&file_priv->object_idr, args->new_handle); 1033 spin_unlock(&file_priv->table_lock); 1034 goto out_unlock; 1035 } 1036 1037 drm_prime_remove_buf_handle(&file_priv->prime, args->handle); 1038 } 1039 1040 ret = 0; 1041 1042 spin_lock(&file_priv->table_lock); 1043 idr_remove(&file_priv->object_idr, args->handle); 1044 spin_unlock(&file_priv->table_lock); 1045 1046 out_unlock: 1047 mutex_unlock(&file_priv->prime.lock); 1048 out: 1049 drm_gem_object_put(obj); 1050 1051 return ret; 1052 } 1053 1054 /** 1055 * drm_gem_open - initializes GEM file-private structures at devnode open time 1056 * @dev: drm_device which is being opened by userspace 1057 * @file_private: drm file-private structure to set up 1058 * 1059 * Called at device open time, sets up the structure for handling refcounting 1060 * of mm objects. 1061 */ 1062 void 1063 drm_gem_open(struct drm_device *dev, struct drm_file *file_private) 1064 { 1065 idr_init_base(&file_private->object_idr, 1); 1066 spin_lock_init(&file_private->table_lock); 1067 } 1068 1069 /** 1070 * drm_gem_release - release file-private GEM resources 1071 * @dev: drm_device which is being closed by userspace 1072 * @file_private: drm file-private structure to clean up 1073 * 1074 * Called at close time when the filp is going away. 1075 * 1076 * Releases any remaining references on objects by this filp. 1077 */ 1078 void 1079 drm_gem_release(struct drm_device *dev, struct drm_file *file_private) 1080 { 1081 idr_for_each(&file_private->object_idr, 1082 &drm_gem_object_release_handle, file_private); 1083 idr_destroy(&file_private->object_idr); 1084 } 1085 1086 /** 1087 * drm_gem_object_release - release GEM buffer object resources 1088 * @obj: GEM buffer object 1089 * 1090 * This releases any structures and resources used by @obj and is the inverse of 1091 * drm_gem_object_init(). 1092 */ 1093 void 1094 drm_gem_object_release(struct drm_gem_object *obj) 1095 { 1096 if (obj->filp) 1097 fput(obj->filp); 1098 1099 drm_gem_private_object_fini(obj); 1100 1101 drm_gem_free_mmap_offset(obj); 1102 drm_gem_lru_remove(obj); 1103 } 1104 EXPORT_SYMBOL(drm_gem_object_release); 1105 1106 /** 1107 * drm_gem_object_free - free a GEM object 1108 * @kref: kref of the object to free 1109 * 1110 * Called after the last reference to the object has been lost. 1111 * 1112 * Frees the object 1113 */ 1114 void 1115 drm_gem_object_free(struct kref *kref) 1116 { 1117 struct drm_gem_object *obj = 1118 container_of(kref, struct drm_gem_object, refcount); 1119 1120 if (WARN_ON(!obj->funcs->free)) 1121 return; 1122 1123 obj->funcs->free(obj); 1124 } 1125 EXPORT_SYMBOL(drm_gem_object_free); 1126 1127 /** 1128 * drm_gem_vm_open - vma->ops->open implementation for GEM 1129 * @vma: VM area structure 1130 * 1131 * This function implements the #vm_operations_struct open() callback for GEM 1132 * drivers. This must be used together with drm_gem_vm_close(). 1133 */ 1134 void drm_gem_vm_open(struct vm_area_struct *vma) 1135 { 1136 struct drm_gem_object *obj = vma->vm_private_data; 1137 1138 drm_gem_object_get(obj); 1139 } 1140 EXPORT_SYMBOL(drm_gem_vm_open); 1141 1142 /** 1143 * drm_gem_vm_close - vma->ops->close implementation for GEM 1144 * @vma: VM area structure 1145 * 1146 * This function implements the #vm_operations_struct close() callback for GEM 1147 * drivers. This must be used together with drm_gem_vm_open(). 1148 */ 1149 void drm_gem_vm_close(struct vm_area_struct *vma) 1150 { 1151 struct drm_gem_object *obj = vma->vm_private_data; 1152 1153 drm_gem_object_put(obj); 1154 } 1155 EXPORT_SYMBOL(drm_gem_vm_close); 1156 1157 /** 1158 * drm_gem_mmap_obj - memory map a GEM object 1159 * @obj: the GEM object to map 1160 * @obj_size: the object size to be mapped, in bytes 1161 * @vma: VMA for the area to be mapped 1162 * 1163 * Set up the VMA to prepare mapping of the GEM object using the GEM object's 1164 * vm_ops. Depending on their requirements, GEM objects can either 1165 * provide a fault handler in their vm_ops (in which case any accesses to 1166 * the object will be trapped, to perform migration, GTT binding, surface 1167 * register allocation, or performance monitoring), or mmap the buffer memory 1168 * synchronously after calling drm_gem_mmap_obj. 1169 * 1170 * This function is mainly intended to implement the DMABUF mmap operation, when 1171 * the GEM object is not looked up based on its fake offset. To implement the 1172 * DRM mmap operation, drivers should use the drm_gem_mmap() function. 1173 * 1174 * drm_gem_mmap_obj() assumes the user is granted access to the buffer while 1175 * drm_gem_mmap() prevents unprivileged users from mapping random objects. So 1176 * callers must verify access restrictions before calling this helper. 1177 * 1178 * Return 0 or success or -EINVAL if the object size is smaller than the VMA 1179 * size, or if no vm_ops are provided. 1180 */ 1181 int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size, 1182 struct vm_area_struct *vma) 1183 { 1184 int ret; 1185 1186 /* Check for valid size. */ 1187 if (obj_size < vma->vm_end - vma->vm_start) 1188 return -EINVAL; 1189 1190 /* Take a ref for this mapping of the object, so that the fault 1191 * handler can dereference the mmap offset's pointer to the object. 1192 * This reference is cleaned up by the corresponding vm_close 1193 * (which should happen whether the vma was created by this call, or 1194 * by a vm_open due to mremap or partial unmap or whatever). 1195 */ 1196 drm_gem_object_get(obj); 1197 1198 vma->vm_private_data = obj; 1199 vma->vm_ops = obj->funcs->vm_ops; 1200 1201 if (obj->funcs->mmap) { 1202 ret = obj->funcs->mmap(obj, vma); 1203 if (ret) 1204 goto err_drm_gem_object_put; 1205 WARN_ON(!(vma->vm_flags & VM_DONTEXPAND)); 1206 } else { 1207 if (!vma->vm_ops) { 1208 ret = -EINVAL; 1209 goto err_drm_gem_object_put; 1210 } 1211 1212 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); 1213 vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); 1214 vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); 1215 } 1216 1217 return 0; 1218 1219 err_drm_gem_object_put: 1220 drm_gem_object_put(obj); 1221 return ret; 1222 } 1223 EXPORT_SYMBOL(drm_gem_mmap_obj); 1224 1225 /* 1226 * Look up a GEM object in offset space based on the exact start address. The 1227 * caller must be granted access to the object. Returns a GEM object on success 1228 * or a negative error code on failure. The returned GEM object needs to be 1229 * released with drm_gem_object_put(). 1230 */ 1231 static struct drm_gem_object * 1232 drm_gem_object_lookup_at_offset(struct file *filp, unsigned long start, 1233 unsigned long pages) 1234 { 1235 struct drm_file *priv = filp->private_data; 1236 struct drm_device *dev = priv->minor->dev; 1237 struct drm_gem_object *obj = NULL; 1238 struct drm_vma_offset_node *node; 1239 1240 if (drm_dev_is_unplugged(dev)) 1241 return ERR_PTR(-ENODEV); 1242 1243 drm_vma_offset_lock_lookup(dev->vma_offset_manager); 1244 node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager, 1245 start, pages); 1246 if (likely(node)) { 1247 obj = container_of(node, struct drm_gem_object, vma_node); 1248 /* 1249 * When the object is being freed, after it hits 0-refcnt it 1250 * proceeds to tear down the object. In the process it will 1251 * attempt to remove the VMA offset and so acquire this 1252 * mgr->vm_lock. Therefore if we find an object with a 0-refcnt 1253 * that matches our range, we know it is in the process of being 1254 * destroyed and will be freed as soon as we release the lock - 1255 * so we have to check for the 0-refcnted object and treat it as 1256 * invalid. 1257 */ 1258 if (!kref_get_unless_zero(&obj->refcount)) 1259 obj = NULL; 1260 } 1261 drm_vma_offset_unlock_lookup(dev->vma_offset_manager); 1262 1263 if (!obj) 1264 return ERR_PTR(-EINVAL); 1265 1266 if (!drm_vma_node_is_allowed(node, priv)) { 1267 drm_gem_object_put(obj); 1268 return ERR_PTR(-EACCES); 1269 } 1270 1271 return obj; 1272 } 1273 1274 #ifdef CONFIG_MMU 1275 /** 1276 * drm_gem_get_unmapped_area - get memory mapping region routine for GEM objects 1277 * @filp: DRM file pointer 1278 * @uaddr: User address hint 1279 * @len: Mapping length 1280 * @pgoff: Offset (in pages) 1281 * @flags: Mapping flags 1282 * 1283 * If a driver supports GEM object mapping, before ending up in drm_gem_mmap(), 1284 * mmap calls on the DRM file descriptor will first try to find a free linear 1285 * address space large enough for a mapping. Since GEM objects are backed by 1286 * shmem buffers, this should preferably be handled by the shmem virtual memory 1287 * filesystem which can appropriately align addresses to huge page sizes when 1288 * needed. 1289 * 1290 * Look up the GEM object based on the offset passed in (vma->vm_pgoff will 1291 * contain the fake offset we created) and call shmem_get_unmapped_area() with 1292 * the right file pointer. 1293 * 1294 * If a GEM object is not available at the given offset or if the caller is not 1295 * granted access to it, fall back to mm_get_unmapped_area(). 1296 */ 1297 unsigned long drm_gem_get_unmapped_area(struct file *filp, unsigned long uaddr, 1298 unsigned long len, unsigned long pgoff, 1299 unsigned long flags) 1300 { 1301 struct drm_gem_object *obj; 1302 unsigned long ret; 1303 1304 obj = drm_gem_object_lookup_at_offset(filp, pgoff, len >> PAGE_SHIFT); 1305 if (IS_ERR(obj)) 1306 obj = NULL; 1307 1308 if (!obj || !obj->filp || !obj->filp->f_op->get_unmapped_area) 1309 ret = mm_get_unmapped_area(filp, uaddr, len, 0, flags); 1310 else 1311 ret = obj->filp->f_op->get_unmapped_area(obj->filp, uaddr, len, 0, flags); 1312 1313 drm_gem_object_put(obj); 1314 1315 return ret; 1316 } 1317 EXPORT_SYMBOL_GPL(drm_gem_get_unmapped_area); 1318 #endif 1319 1320 /** 1321 * drm_gem_mmap - memory map routine for GEM objects 1322 * @filp: DRM file pointer 1323 * @vma: VMA for the area to be mapped 1324 * 1325 * If a driver supports GEM object mapping, mmap calls on the DRM file 1326 * descriptor will end up here. 1327 * 1328 * Look up the GEM object based on the offset passed in (vma->vm_pgoff will 1329 * contain the fake offset we created) and map it with a call to 1330 * drm_gem_mmap_obj(). 1331 * 1332 * If the caller is not granted access to the buffer object, the mmap will fail 1333 * with EACCES. Please see the vma manager for more information. 1334 */ 1335 int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma) 1336 { 1337 struct drm_gem_object *obj; 1338 int ret; 1339 1340 obj = drm_gem_object_lookup_at_offset(filp, vma->vm_pgoff, 1341 vma_pages(vma)); 1342 if (IS_ERR(obj)) 1343 return PTR_ERR(obj); 1344 1345 ret = drm_gem_mmap_obj(obj, 1346 drm_vma_node_size(&obj->vma_node) << PAGE_SHIFT, 1347 vma); 1348 1349 drm_gem_object_put(obj); 1350 1351 return ret; 1352 } 1353 EXPORT_SYMBOL(drm_gem_mmap); 1354 1355 void drm_gem_print_info(struct drm_printer *p, unsigned int indent, 1356 const struct drm_gem_object *obj) 1357 { 1358 drm_printf_indent(p, indent, "name=%d\n", obj->name); 1359 drm_printf_indent(p, indent, "refcount=%u\n", 1360 kref_read(&obj->refcount)); 1361 drm_printf_indent(p, indent, "start=%08lx\n", 1362 drm_vma_node_start(&obj->vma_node)); 1363 drm_printf_indent(p, indent, "size=%zu\n", obj->size); 1364 drm_printf_indent(p, indent, "imported=%s\n", 1365 str_yes_no(drm_gem_is_imported(obj))); 1366 1367 if (obj->funcs->print_info) 1368 obj->funcs->print_info(p, indent, obj); 1369 } 1370 1371 int drm_gem_vmap_locked(struct drm_gem_object *obj, struct iosys_map *map) 1372 { 1373 int ret; 1374 1375 dma_resv_assert_held(obj->resv); 1376 1377 if (!obj->funcs->vmap) 1378 return -EOPNOTSUPP; 1379 1380 ret = obj->funcs->vmap(obj, map); 1381 if (ret) 1382 return ret; 1383 else if (iosys_map_is_null(map)) 1384 return -ENOMEM; 1385 1386 return 0; 1387 } 1388 EXPORT_SYMBOL(drm_gem_vmap_locked); 1389 1390 void drm_gem_vunmap_locked(struct drm_gem_object *obj, struct iosys_map *map) 1391 { 1392 dma_resv_assert_held(obj->resv); 1393 1394 if (iosys_map_is_null(map)) 1395 return; 1396 1397 if (obj->funcs->vunmap) 1398 obj->funcs->vunmap(obj, map); 1399 1400 /* Always set the mapping to NULL. Callers may rely on this. */ 1401 iosys_map_clear(map); 1402 } 1403 EXPORT_SYMBOL(drm_gem_vunmap_locked); 1404 1405 void drm_gem_lock(struct drm_gem_object *obj) 1406 { 1407 dma_resv_lock(obj->resv, NULL); 1408 } 1409 EXPORT_SYMBOL(drm_gem_lock); 1410 1411 void drm_gem_unlock(struct drm_gem_object *obj) 1412 { 1413 dma_resv_unlock(obj->resv); 1414 } 1415 EXPORT_SYMBOL(drm_gem_unlock); 1416 1417 int drm_gem_vmap(struct drm_gem_object *obj, struct iosys_map *map) 1418 { 1419 int ret; 1420 1421 dma_resv_lock(obj->resv, NULL); 1422 ret = drm_gem_vmap_locked(obj, map); 1423 dma_resv_unlock(obj->resv); 1424 1425 return ret; 1426 } 1427 EXPORT_SYMBOL(drm_gem_vmap); 1428 1429 void drm_gem_vunmap(struct drm_gem_object *obj, struct iosys_map *map) 1430 { 1431 dma_resv_lock(obj->resv, NULL); 1432 drm_gem_vunmap_locked(obj, map); 1433 dma_resv_unlock(obj->resv); 1434 } 1435 EXPORT_SYMBOL(drm_gem_vunmap); 1436 1437 /** 1438 * drm_gem_lock_reservations - Sets up the ww context and acquires 1439 * the lock on an array of GEM objects. 1440 * 1441 * Once you've locked your reservations, you'll want to set up space 1442 * for your shared fences (if applicable), submit your job, then 1443 * drm_gem_unlock_reservations(). 1444 * 1445 * @objs: drm_gem_objects to lock 1446 * @count: Number of objects in @objs 1447 * @acquire_ctx: struct ww_acquire_ctx that will be initialized as 1448 * part of tracking this set of locked reservations. 1449 */ 1450 int 1451 drm_gem_lock_reservations(struct drm_gem_object **objs, int count, 1452 struct ww_acquire_ctx *acquire_ctx) 1453 { 1454 int contended = -1; 1455 int i, ret; 1456 1457 ww_acquire_init(acquire_ctx, &reservation_ww_class); 1458 1459 retry: 1460 if (contended != -1) { 1461 struct drm_gem_object *obj = objs[contended]; 1462 1463 ret = dma_resv_lock_slow_interruptible(obj->resv, 1464 acquire_ctx); 1465 if (ret) { 1466 ww_acquire_fini(acquire_ctx); 1467 return ret; 1468 } 1469 } 1470 1471 for (i = 0; i < count; i++) { 1472 if (i == contended) 1473 continue; 1474 1475 ret = dma_resv_lock_interruptible(objs[i]->resv, 1476 acquire_ctx); 1477 if (ret) { 1478 int j; 1479 1480 for (j = 0; j < i; j++) 1481 dma_resv_unlock(objs[j]->resv); 1482 1483 if (contended != -1 && contended >= i) 1484 dma_resv_unlock(objs[contended]->resv); 1485 1486 if (ret == -EDEADLK) { 1487 contended = i; 1488 goto retry; 1489 } 1490 1491 ww_acquire_fini(acquire_ctx); 1492 return ret; 1493 } 1494 } 1495 1496 ww_acquire_done(acquire_ctx); 1497 1498 return 0; 1499 } 1500 EXPORT_SYMBOL(drm_gem_lock_reservations); 1501 1502 void 1503 drm_gem_unlock_reservations(struct drm_gem_object **objs, int count, 1504 struct ww_acquire_ctx *acquire_ctx) 1505 { 1506 int i; 1507 1508 for (i = 0; i < count; i++) 1509 dma_resv_unlock(objs[i]->resv); 1510 1511 ww_acquire_fini(acquire_ctx); 1512 } 1513 EXPORT_SYMBOL(drm_gem_unlock_reservations); 1514 1515 /** 1516 * drm_gem_lru_init - initialize a LRU 1517 * 1518 * @lru: The LRU to initialize 1519 * @lock: The lock protecting the LRU 1520 */ 1521 void 1522 drm_gem_lru_init(struct drm_gem_lru *lru, struct mutex *lock) 1523 { 1524 lru->lock = lock; 1525 lru->count = 0; 1526 INIT_LIST_HEAD(&lru->list); 1527 } 1528 EXPORT_SYMBOL(drm_gem_lru_init); 1529 1530 static void 1531 drm_gem_lru_remove_locked(struct drm_gem_object *obj) 1532 { 1533 obj->lru->count -= obj->size >> PAGE_SHIFT; 1534 WARN_ON(obj->lru->count < 0); 1535 list_del(&obj->lru_node); 1536 obj->lru = NULL; 1537 } 1538 1539 /** 1540 * drm_gem_lru_remove - remove object from whatever LRU it is in 1541 * 1542 * If the object is currently in any LRU, remove it. 1543 * 1544 * @obj: The GEM object to remove from current LRU 1545 */ 1546 void 1547 drm_gem_lru_remove(struct drm_gem_object *obj) 1548 { 1549 struct drm_gem_lru *lru = obj->lru; 1550 1551 if (!lru) 1552 return; 1553 1554 mutex_lock(lru->lock); 1555 drm_gem_lru_remove_locked(obj); 1556 mutex_unlock(lru->lock); 1557 } 1558 EXPORT_SYMBOL(drm_gem_lru_remove); 1559 1560 /** 1561 * drm_gem_lru_move_tail_locked - move the object to the tail of the LRU 1562 * 1563 * Like &drm_gem_lru_move_tail but lru lock must be held 1564 * 1565 * @lru: The LRU to move the object into. 1566 * @obj: The GEM object to move into this LRU 1567 */ 1568 void 1569 drm_gem_lru_move_tail_locked(struct drm_gem_lru *lru, struct drm_gem_object *obj) 1570 { 1571 lockdep_assert_held_once(lru->lock); 1572 1573 if (obj->lru) 1574 drm_gem_lru_remove_locked(obj); 1575 1576 lru->count += obj->size >> PAGE_SHIFT; 1577 list_add_tail(&obj->lru_node, &lru->list); 1578 obj->lru = lru; 1579 } 1580 EXPORT_SYMBOL(drm_gem_lru_move_tail_locked); 1581 1582 /** 1583 * drm_gem_lru_move_tail - move the object to the tail of the LRU 1584 * 1585 * If the object is already in this LRU it will be moved to the 1586 * tail. Otherwise it will be removed from whichever other LRU 1587 * it is in (if any) and moved into this LRU. 1588 * 1589 * @lru: The LRU to move the object into. 1590 * @obj: The GEM object to move into this LRU 1591 */ 1592 void 1593 drm_gem_lru_move_tail(struct drm_gem_lru *lru, struct drm_gem_object *obj) 1594 { 1595 mutex_lock(lru->lock); 1596 drm_gem_lru_move_tail_locked(lru, obj); 1597 mutex_unlock(lru->lock); 1598 } 1599 EXPORT_SYMBOL(drm_gem_lru_move_tail); 1600 1601 /** 1602 * drm_gem_lru_scan - helper to implement shrinker.scan_objects 1603 * 1604 * If the shrink callback succeeds, it is expected that the driver 1605 * move the object out of this LRU. 1606 * 1607 * If the LRU possibly contain active buffers, it is the responsibility 1608 * of the shrink callback to check for this (ie. dma_resv_test_signaled()) 1609 * or if necessary block until the buffer becomes idle. 1610 * 1611 * @lru: The LRU to scan 1612 * @nr_to_scan: The number of pages to try to reclaim 1613 * @remaining: The number of pages left to reclaim, should be initialized by caller 1614 * @shrink: Callback to try to shrink/reclaim the object. 1615 * @ticket: Optional ww_acquire_ctx context to use for locking 1616 */ 1617 unsigned long 1618 drm_gem_lru_scan(struct drm_gem_lru *lru, 1619 unsigned int nr_to_scan, 1620 unsigned long *remaining, 1621 bool (*shrink)(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket), 1622 struct ww_acquire_ctx *ticket) 1623 { 1624 struct drm_gem_lru still_in_lru; 1625 struct drm_gem_object *obj; 1626 unsigned freed = 0; 1627 1628 drm_gem_lru_init(&still_in_lru, lru->lock); 1629 1630 mutex_lock(lru->lock); 1631 1632 while (freed < nr_to_scan) { 1633 obj = list_first_entry_or_null(&lru->list, typeof(*obj), lru_node); 1634 1635 if (!obj) 1636 break; 1637 1638 drm_gem_lru_move_tail_locked(&still_in_lru, obj); 1639 1640 /* 1641 * If it's in the process of being freed, gem_object->free() 1642 * may be blocked on lock waiting to remove it. So just 1643 * skip it. 1644 */ 1645 if (!kref_get_unless_zero(&obj->refcount)) 1646 continue; 1647 1648 /* 1649 * Now that we own a reference, we can drop the lock for the 1650 * rest of the loop body, to reduce contention with other 1651 * code paths that need the LRU lock 1652 */ 1653 mutex_unlock(lru->lock); 1654 1655 if (ticket) 1656 ww_acquire_init(ticket, &reservation_ww_class); 1657 1658 /* 1659 * Note that this still needs to be trylock, since we can 1660 * hit shrinker in response to trying to get backing pages 1661 * for this obj (ie. while it's lock is already held) 1662 */ 1663 if (!ww_mutex_trylock(&obj->resv->lock, ticket)) { 1664 *remaining += obj->size >> PAGE_SHIFT; 1665 goto tail; 1666 } 1667 1668 if (shrink(obj, ticket)) { 1669 freed += obj->size >> PAGE_SHIFT; 1670 1671 /* 1672 * If we succeeded in releasing the object's backing 1673 * pages, we expect the driver to have moved the object 1674 * out of this LRU 1675 */ 1676 WARN_ON(obj->lru == &still_in_lru); 1677 WARN_ON(obj->lru == lru); 1678 } 1679 1680 dma_resv_unlock(obj->resv); 1681 1682 if (ticket) 1683 ww_acquire_fini(ticket); 1684 1685 tail: 1686 drm_gem_object_put(obj); 1687 mutex_lock(lru->lock); 1688 } 1689 1690 /* 1691 * Move objects we've skipped over out of the temporary still_in_lru 1692 * back into this LRU 1693 */ 1694 list_for_each_entry (obj, &still_in_lru.list, lru_node) 1695 obj->lru = lru; 1696 list_splice_tail(&still_in_lru.list, &lru->list); 1697 lru->count += still_in_lru.count; 1698 1699 mutex_unlock(lru->lock); 1700 1701 return freed; 1702 } 1703 EXPORT_SYMBOL(drm_gem_lru_scan); 1704 1705 /** 1706 * drm_gem_evict_locked - helper to evict backing pages for a GEM object 1707 * @obj: obj in question 1708 */ 1709 int drm_gem_evict_locked(struct drm_gem_object *obj) 1710 { 1711 dma_resv_assert_held(obj->resv); 1712 1713 if (!dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ)) 1714 return -EBUSY; 1715 1716 if (obj->funcs->evict) 1717 return obj->funcs->evict(obj); 1718 1719 return 0; 1720 } 1721 EXPORT_SYMBOL(drm_gem_evict_locked); 1722