1 /* 2 * Copyright © 2008 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 */ 27 28 #include <linux/dma-buf.h> 29 #include <linux/export.h> 30 #include <linux/file.h> 31 #include <linux/fs.h> 32 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 33 #include <linux/fs_context.h> 34 #endif 35 #include <linux/iosys-map.h> 36 #include <linux/mem_encrypt.h> 37 #include <linux/mm.h> 38 #include <linux/mman.h> 39 #include <linux/module.h> 40 #include <linux/pagemap.h> 41 #include <linux/pagevec.h> 42 #include <linux/sched/mm.h> 43 #include <linux/shmem_fs.h> 44 #include <linux/slab.h> 45 #include <linux/string_helpers.h> 46 #include <linux/types.h> 47 #include <linux/uaccess.h> 48 49 #include <drm/drm.h> 50 #include <drm/drm_device.h> 51 #include <drm/drm_drv.h> 52 #include <drm/drm_file.h> 53 #include <drm/drm_gem.h> 54 #include <drm/drm_managed.h> 55 #include <drm/drm_print.h> 56 #include <drm/drm_vma_manager.h> 57 58 #include "drm_internal.h" 59 60 /** @file drm_gem.c 61 * 62 * This file provides some of the base ioctls and library routines for 63 * the graphics memory manager implemented by each device driver. 64 * 65 * Because various devices have different requirements in terms of 66 * synchronization and migration strategies, implementing that is left up to 67 * the driver, and all that the general API provides should be generic -- 68 * allocating objects, reading/writing data with the cpu, freeing objects. 69 * Even there, platform-dependent optimizations for reading/writing data with 70 * the CPU mean we'll likely hook those out to driver-specific calls. However, 71 * the DRI2 implementation wants to have at least allocate/mmap be generic. 72 * 73 * The goal was to have swap-backed object allocation managed through 74 * struct file. However, file descriptors as handles to a struct file have 75 * two major failings: 76 * - Process limits prevent more than 1024 or so being used at a time by 77 * default. 78 * - Inability to allocate high fds will aggravate the X Server's select() 79 * handling, and likely that of many GL client applications as well. 80 * 81 * This led to a plan of using our own integer IDs (called handles, following 82 * DRM terminology) to mimic fds, and implement the fd syscalls we need as 83 * ioctls. The objects themselves will still include the struct file so 84 * that we can transition to fds if the required kernel infrastructure shows 85 * up at a later date, and as our interface with shmfs for memory allocation. 86 */ 87 88 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 89 static void drm_gem_huge_mnt_free(struct drm_device *dev, void *data) 90 { 91 kern_unmount(dev->huge_mnt); 92 } 93 94 /** 95 * drm_gem_huge_mnt_create - Create, mount and use a huge tmpfs mountpoint 96 * @dev: DRM device that will use the huge tmpfs mountpoint 97 * @value: huge tmpfs mount option value 98 * 99 * This function creates and mounts a dedicated huge tmpfs mountpoint for the 100 * lifetime of the DRM device @dev which is used at GEM object initialization 101 * with drm_gem_object_init(). 102 * 103 * The most common option for @value is "within_size" which only allocates huge 104 * pages if the page will be fully within the GEM object size. "always", 105 * "advise" and "never" are supported too but the latter would just create a 106 * mountpoint similar to the default one (`shm_mnt`). See shmemfs and 107 * Transparent Hugepage for more information. 108 * 109 * Returns: 110 * 0 on success or a negative error code on failure. 111 */ 112 int drm_gem_huge_mnt_create(struct drm_device *dev, const char *value) 113 { 114 struct file_system_type *type; 115 struct fs_context *fc; 116 int ret; 117 118 if (unlikely(drm_gem_get_huge_mnt(dev))) 119 return 0; 120 121 type = get_fs_type("tmpfs"); 122 if (unlikely(!type)) 123 return -EOPNOTSUPP; 124 fc = fs_context_for_mount(type, SB_KERNMOUNT); 125 if (IS_ERR(fc)) 126 return PTR_ERR(fc); 127 ret = vfs_parse_fs_string(fc, "source", "tmpfs"); 128 if (unlikely(ret)) 129 return -ENOPARAM; 130 ret = vfs_parse_fs_string(fc, "huge", value); 131 if (unlikely(ret)) 132 return -ENOPARAM; 133 134 dev->huge_mnt = fc_mount_longterm(fc); 135 put_fs_context(fc); 136 137 return drmm_add_action_or_reset(dev, drm_gem_huge_mnt_free, NULL); 138 } 139 EXPORT_SYMBOL_GPL(drm_gem_huge_mnt_create); 140 #endif 141 142 static void 143 drm_gem_init_release(struct drm_device *dev, void *ptr) 144 { 145 drm_vma_offset_manager_destroy(dev->vma_offset_manager); 146 } 147 148 /** 149 * drm_gem_init - Initialize the GEM device fields 150 * @dev: drm_devic structure to initialize 151 */ 152 int 153 drm_gem_init(struct drm_device *dev) 154 { 155 struct drm_vma_offset_manager *vma_offset_manager; 156 157 mutex_init(&dev->object_name_lock); 158 idr_init_base(&dev->object_name_idr, 1); 159 160 vma_offset_manager = drmm_kzalloc(dev, sizeof(*vma_offset_manager), 161 GFP_KERNEL); 162 if (!vma_offset_manager) 163 return -ENOMEM; 164 165 dev->vma_offset_manager = vma_offset_manager; 166 drm_vma_offset_manager_init(vma_offset_manager, 167 DRM_FILE_PAGE_OFFSET_START, 168 DRM_FILE_PAGE_OFFSET_SIZE); 169 170 return drmm_add_action(dev, drm_gem_init_release, NULL); 171 } 172 173 /** 174 * drm_gem_object_init - initialize an allocated shmem-backed GEM object 175 * 176 * @dev: drm_device the object should be initialized for 177 * @obj: drm_gem_object to initialize 178 * @size: object size 179 * 180 * Initialize an already allocated GEM object of the specified size with 181 * shmfs backing store. A huge mountpoint can be used by calling 182 * drm_gem_huge_mnt_create() beforehand. 183 */ 184 int drm_gem_object_init(struct drm_device *dev, struct drm_gem_object *obj, 185 size_t size) 186 { 187 struct vfsmount *huge_mnt; 188 struct file *filp; 189 190 drm_gem_private_object_init(dev, obj, size); 191 192 huge_mnt = drm_gem_get_huge_mnt(dev); 193 if (huge_mnt) 194 filp = shmem_file_setup_with_mnt(huge_mnt, "drm mm object", 195 size, VM_NORESERVE); 196 else 197 filp = shmem_file_setup("drm mm object", size, VM_NORESERVE); 198 199 if (IS_ERR(filp)) 200 return PTR_ERR(filp); 201 202 obj->filp = filp; 203 204 return 0; 205 } 206 EXPORT_SYMBOL(drm_gem_object_init); 207 208 /** 209 * drm_gem_private_object_init - initialize an allocated private GEM object 210 * @dev: drm_device the object should be initialized for 211 * @obj: drm_gem_object to initialize 212 * @size: object size 213 * 214 * Initialize an already allocated GEM object of the specified size with 215 * no GEM provided backing store. Instead the caller is responsible for 216 * backing the object and handling it. 217 */ 218 void drm_gem_private_object_init(struct drm_device *dev, 219 struct drm_gem_object *obj, size_t size) 220 { 221 BUG_ON((size & (PAGE_SIZE - 1)) != 0); 222 223 obj->dev = dev; 224 obj->filp = NULL; 225 226 kref_init(&obj->refcount); 227 obj->handle_count = 0; 228 obj->size = size; 229 mutex_init(&obj->gpuva.lock); 230 dma_resv_init(&obj->_resv); 231 if (!obj->resv) 232 obj->resv = &obj->_resv; 233 234 if (drm_core_check_feature(dev, DRIVER_GEM_GPUVA)) 235 drm_gem_gpuva_init(obj); 236 237 drm_vma_node_reset(&obj->vma_node); 238 INIT_LIST_HEAD(&obj->lru_node); 239 } 240 EXPORT_SYMBOL(drm_gem_private_object_init); 241 242 /** 243 * drm_gem_private_object_fini - Finalize a failed drm_gem_object 244 * @obj: drm_gem_object 245 * 246 * Uninitialize an already allocated GEM object when it initialized failed 247 */ 248 void drm_gem_private_object_fini(struct drm_gem_object *obj) 249 { 250 WARN_ON(obj->dma_buf); 251 252 dma_resv_fini(&obj->_resv); 253 mutex_destroy(&obj->gpuva.lock); 254 } 255 EXPORT_SYMBOL(drm_gem_private_object_fini); 256 257 static void drm_gem_object_handle_get(struct drm_gem_object *obj) 258 { 259 struct drm_device *dev = obj->dev; 260 261 drm_WARN_ON(dev, !mutex_is_locked(&dev->object_name_lock)); 262 263 if (obj->handle_count++ == 0) 264 drm_gem_object_get(obj); 265 } 266 267 /** 268 * drm_gem_object_handle_get_if_exists_unlocked - acquire reference on user-space handle, if any 269 * @obj: GEM object 270 * 271 * Acquires a reference on the GEM buffer object's handle. Required to keep 272 * the GEM object alive. Call drm_gem_object_handle_put_if_exists_unlocked() 273 * to release the reference. Does nothing if the buffer object has no handle. 274 * 275 * Returns: 276 * True if a handle exists, or false otherwise 277 */ 278 bool drm_gem_object_handle_get_if_exists_unlocked(struct drm_gem_object *obj) 279 { 280 struct drm_device *dev = obj->dev; 281 282 guard(mutex)(&dev->object_name_lock); 283 284 /* 285 * First ref taken during GEM object creation, if any. Some 286 * drivers set up internal framebuffers with GEM objects that 287 * do not have a GEM handle. Hence, this counter can be zero. 288 */ 289 if (!obj->handle_count) 290 return false; 291 292 drm_gem_object_handle_get(obj); 293 294 return true; 295 } 296 297 /** 298 * drm_gem_object_handle_free - release resources bound to userspace handles 299 * @obj: GEM object to clean up. 300 * 301 * Called after the last handle to the object has been closed 302 * 303 * Removes any name for the object. Note that this must be 304 * called before drm_gem_object_free or we'll be touching 305 * freed memory 306 */ 307 static void drm_gem_object_handle_free(struct drm_gem_object *obj) 308 { 309 struct drm_device *dev = obj->dev; 310 311 /* Remove any name for this object */ 312 if (obj->name) { 313 idr_remove(&dev->object_name_idr, obj->name); 314 obj->name = 0; 315 } 316 } 317 318 static void drm_gem_object_exported_dma_buf_free(struct drm_gem_object *obj) 319 { 320 /* Unbreak the reference cycle if we have an exported dma_buf. */ 321 if (obj->dma_buf) { 322 dma_buf_put(obj->dma_buf); 323 obj->dma_buf = NULL; 324 } 325 } 326 327 /** 328 * drm_gem_object_handle_put_unlocked - releases reference on user-space handle 329 * @obj: GEM object 330 * 331 * Releases a reference on the GEM buffer object's handle. Possibly releases 332 * the GEM buffer object and associated dma-buf objects. 333 */ 334 void drm_gem_object_handle_put_unlocked(struct drm_gem_object *obj) 335 { 336 struct drm_device *dev = obj->dev; 337 bool final = false; 338 339 if (drm_WARN_ON(dev, READ_ONCE(obj->handle_count) == 0)) 340 return; 341 342 /* 343 * Must bump handle count first as this may be the last 344 * ref, in which case the object would disappear before 345 * we checked for a name. 346 */ 347 348 mutex_lock(&dev->object_name_lock); 349 if (--obj->handle_count == 0) { 350 drm_gem_object_handle_free(obj); 351 drm_gem_object_exported_dma_buf_free(obj); 352 final = true; 353 } 354 mutex_unlock(&dev->object_name_lock); 355 356 if (final) 357 drm_gem_object_put(obj); 358 } 359 360 /* 361 * Called at device or object close to release the file's 362 * handle references on objects. 363 */ 364 static int 365 drm_gem_object_release_handle(int id, void *ptr, void *data) 366 { 367 struct drm_file *file_priv = data; 368 struct drm_gem_object *obj = ptr; 369 370 if (drm_WARN_ON(obj->dev, !data)) 371 return 0; 372 373 if (obj->funcs->close) 374 obj->funcs->close(obj, file_priv); 375 376 mutex_lock(&file_priv->prime.lock); 377 378 drm_prime_remove_buf_handle(&file_priv->prime, id); 379 380 mutex_unlock(&file_priv->prime.lock); 381 382 drm_vma_node_revoke(&obj->vma_node, file_priv); 383 384 drm_gem_object_handle_put_unlocked(obj); 385 386 return 0; 387 } 388 389 /** 390 * drm_gem_handle_delete - deletes the given file-private handle 391 * @filp: drm file-private structure to use for the handle look up 392 * @handle: userspace handle to delete 393 * 394 * Removes the GEM handle from the @filp lookup table which has been added with 395 * drm_gem_handle_create(). If this is the last handle also cleans up linked 396 * resources like GEM names. 397 */ 398 int 399 drm_gem_handle_delete(struct drm_file *filp, u32 handle) 400 { 401 struct drm_gem_object *obj; 402 403 spin_lock(&filp->table_lock); 404 405 /* Check if we currently have a reference on the object */ 406 obj = idr_replace(&filp->object_idr, NULL, handle); 407 spin_unlock(&filp->table_lock); 408 if (IS_ERR_OR_NULL(obj)) 409 return -EINVAL; 410 411 /* Release driver's reference and decrement refcount. */ 412 drm_gem_object_release_handle(handle, obj, filp); 413 414 /* And finally make the handle available for future allocations. */ 415 spin_lock(&filp->table_lock); 416 idr_remove(&filp->object_idr, handle); 417 spin_unlock(&filp->table_lock); 418 419 return 0; 420 } 421 EXPORT_SYMBOL(drm_gem_handle_delete); 422 423 /** 424 * drm_gem_dumb_map_offset - return the fake mmap offset for a gem object 425 * @file: drm file-private structure containing the gem object 426 * @dev: corresponding drm_device 427 * @handle: gem object handle 428 * @offset: return location for the fake mmap offset 429 * 430 * This implements the &drm_driver.dumb_map_offset kms driver callback for 431 * drivers which use gem to manage their backing storage. 432 * 433 * Returns: 434 * 0 on success or a negative error code on failure. 435 */ 436 int drm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev, 437 u32 handle, u64 *offset) 438 { 439 struct drm_gem_object *obj; 440 int ret; 441 442 obj = drm_gem_object_lookup(file, handle); 443 if (!obj) 444 return -ENOENT; 445 446 /* Don't allow imported objects to be mapped */ 447 if (drm_gem_is_imported(obj)) { 448 ret = -EINVAL; 449 goto out; 450 } 451 452 ret = drm_gem_create_mmap_offset(obj); 453 if (ret) 454 goto out; 455 456 *offset = drm_vma_node_offset_addr(&obj->vma_node); 457 out: 458 drm_gem_object_put(obj); 459 460 return ret; 461 } 462 EXPORT_SYMBOL_GPL(drm_gem_dumb_map_offset); 463 464 /** 465 * drm_gem_handle_create_tail - internal functions to create a handle 466 * @file_priv: drm file-private structure to register the handle for 467 * @obj: object to register 468 * @handlep: pointer to return the created handle to the caller 469 * 470 * This expects the &drm_device.object_name_lock to be held already and will 471 * drop it before returning. Used to avoid races in establishing new handles 472 * when importing an object from either an flink name or a dma-buf. 473 * 474 * Handles must be release again through drm_gem_handle_delete(). This is done 475 * when userspace closes @file_priv for all attached handles, or through the 476 * GEM_CLOSE ioctl for individual handles. 477 */ 478 int 479 drm_gem_handle_create_tail(struct drm_file *file_priv, 480 struct drm_gem_object *obj, 481 u32 *handlep) 482 { 483 struct drm_device *dev = obj->dev; 484 u32 handle; 485 int ret; 486 487 WARN_ON(!mutex_is_locked(&dev->object_name_lock)); 488 489 drm_gem_object_handle_get(obj); 490 491 /* 492 * Get the user-visible handle using idr. Preload and perform 493 * allocation under our spinlock. 494 */ 495 idr_preload(GFP_KERNEL); 496 spin_lock(&file_priv->table_lock); 497 498 ret = idr_alloc(&file_priv->object_idr, NULL, 1, 0, GFP_NOWAIT); 499 500 spin_unlock(&file_priv->table_lock); 501 idr_preload_end(); 502 503 mutex_unlock(&dev->object_name_lock); 504 if (ret < 0) 505 goto err_unref; 506 507 handle = ret; 508 509 ret = drm_vma_node_allow(&obj->vma_node, file_priv); 510 if (ret) 511 goto err_remove; 512 513 if (obj->funcs->open) { 514 ret = obj->funcs->open(obj, file_priv); 515 if (ret) 516 goto err_revoke; 517 } 518 519 /* mirrors drm_gem_handle_delete to avoid races */ 520 spin_lock(&file_priv->table_lock); 521 obj = idr_replace(&file_priv->object_idr, obj, handle); 522 WARN_ON(obj != NULL); 523 spin_unlock(&file_priv->table_lock); 524 *handlep = handle; 525 return 0; 526 527 err_revoke: 528 drm_vma_node_revoke(&obj->vma_node, file_priv); 529 err_remove: 530 spin_lock(&file_priv->table_lock); 531 idr_remove(&file_priv->object_idr, handle); 532 spin_unlock(&file_priv->table_lock); 533 err_unref: 534 drm_gem_object_handle_put_unlocked(obj); 535 return ret; 536 } 537 538 /** 539 * drm_gem_handle_create - create a gem handle for an object 540 * @file_priv: drm file-private structure to register the handle for 541 * @obj: object to register 542 * @handlep: pointer to return the created handle to the caller 543 * 544 * Create a handle for this object. This adds a handle reference to the object, 545 * which includes a regular reference count. Callers will likely want to 546 * dereference the object afterwards. 547 * 548 * Since this publishes @obj to userspace it must be fully set up by this point, 549 * drivers must call this last in their buffer object creation callbacks. 550 */ 551 int drm_gem_handle_create(struct drm_file *file_priv, 552 struct drm_gem_object *obj, 553 u32 *handlep) 554 { 555 mutex_lock(&obj->dev->object_name_lock); 556 557 return drm_gem_handle_create_tail(file_priv, obj, handlep); 558 } 559 EXPORT_SYMBOL(drm_gem_handle_create); 560 561 562 /** 563 * drm_gem_free_mmap_offset - release a fake mmap offset for an object 564 * @obj: obj in question 565 * 566 * This routine frees fake offsets allocated by drm_gem_create_mmap_offset(). 567 * 568 * Note that drm_gem_object_release() already calls this function, so drivers 569 * don't have to take care of releasing the mmap offset themselves when freeing 570 * the GEM object. 571 */ 572 void 573 drm_gem_free_mmap_offset(struct drm_gem_object *obj) 574 { 575 struct drm_device *dev = obj->dev; 576 577 drm_vma_offset_remove(dev->vma_offset_manager, &obj->vma_node); 578 } 579 EXPORT_SYMBOL(drm_gem_free_mmap_offset); 580 581 /** 582 * drm_gem_create_mmap_offset_size - create a fake mmap offset for an object 583 * @obj: obj in question 584 * @size: the virtual size 585 * 586 * GEM memory mapping works by handing back to userspace a fake mmap offset 587 * it can use in a subsequent mmap(2) call. The DRM core code then looks 588 * up the object based on the offset and sets up the various memory mapping 589 * structures. 590 * 591 * This routine allocates and attaches a fake offset for @obj, in cases where 592 * the virtual size differs from the physical size (ie. &drm_gem_object.size). 593 * Otherwise just use drm_gem_create_mmap_offset(). 594 * 595 * This function is idempotent and handles an already allocated mmap offset 596 * transparently. Drivers do not need to check for this case. 597 */ 598 int 599 drm_gem_create_mmap_offset_size(struct drm_gem_object *obj, size_t size) 600 { 601 struct drm_device *dev = obj->dev; 602 603 return drm_vma_offset_add(dev->vma_offset_manager, &obj->vma_node, 604 size / PAGE_SIZE); 605 } 606 EXPORT_SYMBOL(drm_gem_create_mmap_offset_size); 607 608 /** 609 * drm_gem_create_mmap_offset - create a fake mmap offset for an object 610 * @obj: obj in question 611 * 612 * GEM memory mapping works by handing back to userspace a fake mmap offset 613 * it can use in a subsequent mmap(2) call. The DRM core code then looks 614 * up the object based on the offset and sets up the various memory mapping 615 * structures. 616 * 617 * This routine allocates and attaches a fake offset for @obj. 618 * 619 * Drivers can call drm_gem_free_mmap_offset() before freeing @obj to release 620 * the fake offset again. 621 */ 622 int drm_gem_create_mmap_offset(struct drm_gem_object *obj) 623 { 624 return drm_gem_create_mmap_offset_size(obj, obj->size); 625 } 626 EXPORT_SYMBOL(drm_gem_create_mmap_offset); 627 628 /* 629 * Move folios to appropriate lru and release the folios, decrementing the 630 * ref count of those folios. 631 */ 632 static void drm_gem_check_release_batch(struct folio_batch *fbatch) 633 { 634 check_move_unevictable_folios(fbatch); 635 __folio_batch_release(fbatch); 636 cond_resched(); 637 } 638 639 /** 640 * drm_gem_get_pages - helper to allocate backing pages for a GEM object 641 * from shmem 642 * @obj: obj in question 643 * 644 * This reads the page-array of the shmem-backing storage of the given gem 645 * object. An array of pages is returned. If a page is not allocated or 646 * swapped-out, this will allocate/swap-in the required pages. Note that the 647 * whole object is covered by the page-array and pinned in memory. 648 * 649 * Use drm_gem_put_pages() to release the array and unpin all pages. 650 * 651 * This uses the GFP-mask set on the shmem-mapping (see mapping_set_gfp_mask()). 652 * If you require other GFP-masks, you have to do those allocations yourself. 653 * 654 * Note that you are not allowed to change gfp-zones during runtime. That is, 655 * shmem_read_mapping_page_gfp() must be called with the same gfp_zone(gfp) as 656 * set during initialization. If you have special zone constraints, set them 657 * after drm_gem_object_init() via mapping_set_gfp_mask(). shmem-core takes care 658 * to keep pages in the required zone during swap-in. 659 * 660 * This function is only valid on objects initialized with 661 * drm_gem_object_init(), but not for those initialized with 662 * drm_gem_private_object_init() only. 663 */ 664 struct page **drm_gem_get_pages(struct drm_gem_object *obj) 665 { 666 struct address_space *mapping; 667 struct page **pages; 668 struct folio *folio; 669 struct folio_batch fbatch; 670 unsigned long i, j, npages; 671 672 if (WARN_ON(!obj->filp)) 673 return ERR_PTR(-EINVAL); 674 675 /* This is the shared memory object that backs the GEM resource */ 676 mapping = obj->filp->f_mapping; 677 678 /* We already BUG_ON() for non-page-aligned sizes in 679 * drm_gem_object_init(), so we should never hit this unless 680 * driver author is doing something really wrong: 681 */ 682 WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0); 683 684 npages = obj->size >> PAGE_SHIFT; 685 686 pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL); 687 if (pages == NULL) 688 return ERR_PTR(-ENOMEM); 689 690 mapping_set_unevictable(mapping); 691 692 i = 0; 693 while (i < npages) { 694 unsigned long nr; 695 folio = shmem_read_folio_gfp(mapping, i, 696 mapping_gfp_mask(mapping)); 697 if (IS_ERR(folio)) 698 goto fail; 699 nr = min(npages - i, folio_nr_pages(folio)); 700 for (j = 0; j < nr; j++, i++) 701 pages[i] = folio_file_page(folio, i); 702 703 /* Make sure shmem keeps __GFP_DMA32 allocated pages in the 704 * correct region during swapin. Note that this requires 705 * __GFP_DMA32 to be set in mapping_gfp_mask(inode->i_mapping) 706 * so shmem can relocate pages during swapin if required. 707 */ 708 BUG_ON(mapping_gfp_constraint(mapping, __GFP_DMA32) && 709 (folio_pfn(folio) >= 0x00100000UL)); 710 } 711 712 return pages; 713 714 fail: 715 mapping_clear_unevictable(mapping); 716 folio_batch_init(&fbatch); 717 j = 0; 718 while (j < i) { 719 struct folio *f = page_folio(pages[j]); 720 if (!folio_batch_add(&fbatch, f)) 721 drm_gem_check_release_batch(&fbatch); 722 j += folio_nr_pages(f); 723 } 724 if (fbatch.nr) 725 drm_gem_check_release_batch(&fbatch); 726 727 kvfree(pages); 728 return ERR_CAST(folio); 729 } 730 EXPORT_SYMBOL(drm_gem_get_pages); 731 732 /** 733 * drm_gem_put_pages - helper to free backing pages for a GEM object 734 * @obj: obj in question 735 * @pages: pages to free 736 * @dirty: if true, pages will be marked as dirty 737 * @accessed: if true, the pages will be marked as accessed 738 */ 739 void drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages, 740 bool dirty, bool accessed) 741 { 742 int i, npages; 743 struct address_space *mapping; 744 struct folio_batch fbatch; 745 746 mapping = file_inode(obj->filp)->i_mapping; 747 mapping_clear_unevictable(mapping); 748 749 /* We already BUG_ON() for non-page-aligned sizes in 750 * drm_gem_object_init(), so we should never hit this unless 751 * driver author is doing something really wrong: 752 */ 753 WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0); 754 755 npages = obj->size >> PAGE_SHIFT; 756 757 folio_batch_init(&fbatch); 758 for (i = 0; i < npages; i++) { 759 struct folio *folio; 760 761 if (!pages[i]) 762 continue; 763 folio = page_folio(pages[i]); 764 765 if (dirty) 766 folio_mark_dirty(folio); 767 768 if (accessed) 769 folio_mark_accessed(folio); 770 771 /* Undo the reference we took when populating the table */ 772 if (!folio_batch_add(&fbatch, folio)) 773 drm_gem_check_release_batch(&fbatch); 774 i += folio_nr_pages(folio) - 1; 775 } 776 if (folio_batch_count(&fbatch)) 777 drm_gem_check_release_batch(&fbatch); 778 779 kvfree(pages); 780 } 781 EXPORT_SYMBOL(drm_gem_put_pages); 782 783 static int objects_lookup(struct drm_file *filp, u32 *handle, int count, 784 struct drm_gem_object **objs) 785 { 786 int i, ret = 0; 787 struct drm_gem_object *obj; 788 789 spin_lock(&filp->table_lock); 790 791 for (i = 0; i < count; i++) { 792 /* Check if we currently have a reference on the object */ 793 obj = idr_find(&filp->object_idr, handle[i]); 794 if (!obj) { 795 ret = -ENOENT; 796 break; 797 } 798 drm_gem_object_get(obj); 799 objs[i] = obj; 800 } 801 spin_unlock(&filp->table_lock); 802 803 return ret; 804 } 805 806 /** 807 * drm_gem_objects_lookup - look up GEM objects from an array of handles 808 * @filp: DRM file private date 809 * @bo_handles: user pointer to array of userspace handle 810 * @count: size of handle array 811 * @objs_out: returned pointer to array of drm_gem_object pointers 812 * 813 * Takes an array of userspace handles and returns a newly allocated array of 814 * GEM objects. 815 * 816 * For a single handle lookup, use drm_gem_object_lookup(). 817 * 818 * Returns: 819 * @objs filled in with GEM object pointers. Returned GEM objects need to be 820 * released with drm_gem_object_put(). -ENOENT is returned on a lookup 821 * failure. 0 is returned on success. 822 * 823 */ 824 int drm_gem_objects_lookup(struct drm_file *filp, void __user *bo_handles, 825 int count, struct drm_gem_object ***objs_out) 826 { 827 struct drm_gem_object **objs; 828 u32 *handles; 829 int ret; 830 831 if (!count) 832 return 0; 833 834 objs = kvmalloc_array(count, sizeof(struct drm_gem_object *), 835 GFP_KERNEL | __GFP_ZERO); 836 if (!objs) 837 return -ENOMEM; 838 839 *objs_out = objs; 840 841 handles = vmemdup_array_user(bo_handles, count, sizeof(u32)); 842 if (IS_ERR(handles)) 843 return PTR_ERR(handles); 844 845 ret = objects_lookup(filp, handles, count, objs); 846 kvfree(handles); 847 return ret; 848 849 } 850 EXPORT_SYMBOL(drm_gem_objects_lookup); 851 852 /** 853 * drm_gem_object_lookup - look up a GEM object from its handle 854 * @filp: DRM file private date 855 * @handle: userspace handle 856 * 857 * If looking up an array of handles, use drm_gem_objects_lookup(). 858 * 859 * Returns: 860 * A reference to the object named by the handle if such exists on @filp, NULL 861 * otherwise. 862 */ 863 struct drm_gem_object * 864 drm_gem_object_lookup(struct drm_file *filp, u32 handle) 865 { 866 struct drm_gem_object *obj = NULL; 867 868 objects_lookup(filp, &handle, 1, &obj); 869 return obj; 870 } 871 EXPORT_SYMBOL(drm_gem_object_lookup); 872 873 /** 874 * drm_gem_dma_resv_wait - Wait on GEM object's reservation's objects 875 * shared and/or exclusive fences. 876 * @filep: DRM file private date 877 * @handle: userspace handle 878 * @wait_all: if true, wait on all fences, else wait on just exclusive fence 879 * @timeout: timeout value in jiffies or zero to return immediately 880 * 881 * Returns: 882 * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or 883 * greater than 0 on success. 884 */ 885 long drm_gem_dma_resv_wait(struct drm_file *filep, u32 handle, 886 bool wait_all, unsigned long timeout) 887 { 888 struct drm_device *dev = filep->minor->dev; 889 struct drm_gem_object *obj; 890 long ret; 891 892 obj = drm_gem_object_lookup(filep, handle); 893 if (!obj) { 894 drm_dbg_core(dev, "Failed to look up GEM BO %d\n", handle); 895 return -EINVAL; 896 } 897 898 ret = dma_resv_wait_timeout(obj->resv, dma_resv_usage_rw(wait_all), 899 true, timeout); 900 if (ret == 0) 901 ret = -ETIME; 902 else if (ret > 0) 903 ret = 0; 904 905 drm_gem_object_put(obj); 906 907 return ret; 908 } 909 EXPORT_SYMBOL(drm_gem_dma_resv_wait); 910 911 int 912 drm_gem_close_ioctl(struct drm_device *dev, void *data, 913 struct drm_file *file_priv) 914 { 915 struct drm_gem_close *args = data; 916 int ret; 917 918 if (!drm_core_check_feature(dev, DRIVER_GEM)) 919 return -EOPNOTSUPP; 920 921 ret = drm_gem_handle_delete(file_priv, args->handle); 922 923 return ret; 924 } 925 926 int 927 drm_gem_flink_ioctl(struct drm_device *dev, void *data, 928 struct drm_file *file_priv) 929 { 930 struct drm_gem_flink *args = data; 931 struct drm_gem_object *obj; 932 int ret; 933 934 if (!drm_core_check_feature(dev, DRIVER_GEM)) 935 return -EOPNOTSUPP; 936 937 obj = drm_gem_object_lookup(file_priv, args->handle); 938 if (obj == NULL) 939 return -ENOENT; 940 941 mutex_lock(&dev->object_name_lock); 942 /* prevent races with concurrent gem_close. */ 943 if (obj->handle_count == 0) { 944 ret = -ENOENT; 945 goto err; 946 } 947 948 if (!obj->name) { 949 ret = idr_alloc(&dev->object_name_idr, obj, 1, 0, GFP_KERNEL); 950 if (ret < 0) 951 goto err; 952 953 obj->name = ret; 954 } 955 956 args->name = (uint64_t) obj->name; 957 ret = 0; 958 959 err: 960 mutex_unlock(&dev->object_name_lock); 961 drm_gem_object_put(obj); 962 return ret; 963 } 964 965 int 966 drm_gem_open_ioctl(struct drm_device *dev, void *data, 967 struct drm_file *file_priv) 968 { 969 struct drm_gem_open *args = data; 970 struct drm_gem_object *obj; 971 int ret; 972 u32 handle; 973 974 if (!drm_core_check_feature(dev, DRIVER_GEM)) 975 return -EOPNOTSUPP; 976 977 mutex_lock(&dev->object_name_lock); 978 obj = idr_find(&dev->object_name_idr, (int) args->name); 979 if (obj) { 980 drm_gem_object_get(obj); 981 } else { 982 mutex_unlock(&dev->object_name_lock); 983 return -ENOENT; 984 } 985 986 /* drm_gem_handle_create_tail unlocks dev->object_name_lock. */ 987 ret = drm_gem_handle_create_tail(file_priv, obj, &handle); 988 if (ret) 989 goto err; 990 991 args->handle = handle; 992 args->size = obj->size; 993 994 err: 995 drm_gem_object_put(obj); 996 return ret; 997 } 998 999 int drm_gem_change_handle_ioctl(struct drm_device *dev, void *data, 1000 struct drm_file *file_priv) 1001 { 1002 struct drm_gem_change_handle *args = data; 1003 struct drm_gem_object *obj; 1004 int ret; 1005 1006 if (!drm_core_check_feature(dev, DRIVER_GEM)) 1007 return -EOPNOTSUPP; 1008 1009 obj = drm_gem_object_lookup(file_priv, args->handle); 1010 if (!obj) 1011 return -ENOENT; 1012 1013 if (args->handle == args->new_handle) 1014 return 0; 1015 1016 mutex_lock(&file_priv->prime.lock); 1017 1018 spin_lock(&file_priv->table_lock); 1019 ret = idr_alloc(&file_priv->object_idr, obj, 1020 args->new_handle, args->new_handle + 1, GFP_NOWAIT); 1021 spin_unlock(&file_priv->table_lock); 1022 1023 if (ret < 0) 1024 goto out_unlock; 1025 1026 if (obj->dma_buf) { 1027 ret = drm_prime_add_buf_handle(&file_priv->prime, obj->dma_buf, args->new_handle); 1028 if (ret < 0) { 1029 spin_lock(&file_priv->table_lock); 1030 idr_remove(&file_priv->object_idr, args->new_handle); 1031 spin_unlock(&file_priv->table_lock); 1032 goto out_unlock; 1033 } 1034 1035 drm_prime_remove_buf_handle(&file_priv->prime, args->handle); 1036 } 1037 1038 ret = 0; 1039 1040 spin_lock(&file_priv->table_lock); 1041 idr_remove(&file_priv->object_idr, args->handle); 1042 spin_unlock(&file_priv->table_lock); 1043 1044 out_unlock: 1045 mutex_unlock(&file_priv->prime.lock); 1046 1047 return ret; 1048 } 1049 1050 /** 1051 * drm_gem_open - initializes GEM file-private structures at devnode open time 1052 * @dev: drm_device which is being opened by userspace 1053 * @file_private: drm file-private structure to set up 1054 * 1055 * Called at device open time, sets up the structure for handling refcounting 1056 * of mm objects. 1057 */ 1058 void 1059 drm_gem_open(struct drm_device *dev, struct drm_file *file_private) 1060 { 1061 idr_init_base(&file_private->object_idr, 1); 1062 spin_lock_init(&file_private->table_lock); 1063 } 1064 1065 /** 1066 * drm_gem_release - release file-private GEM resources 1067 * @dev: drm_device which is being closed by userspace 1068 * @file_private: drm file-private structure to clean up 1069 * 1070 * Called at close time when the filp is going away. 1071 * 1072 * Releases any remaining references on objects by this filp. 1073 */ 1074 void 1075 drm_gem_release(struct drm_device *dev, struct drm_file *file_private) 1076 { 1077 idr_for_each(&file_private->object_idr, 1078 &drm_gem_object_release_handle, file_private); 1079 idr_destroy(&file_private->object_idr); 1080 } 1081 1082 /** 1083 * drm_gem_object_release - release GEM buffer object resources 1084 * @obj: GEM buffer object 1085 * 1086 * This releases any structures and resources used by @obj and is the inverse of 1087 * drm_gem_object_init(). 1088 */ 1089 void 1090 drm_gem_object_release(struct drm_gem_object *obj) 1091 { 1092 if (obj->filp) 1093 fput(obj->filp); 1094 1095 drm_gem_private_object_fini(obj); 1096 1097 drm_gem_free_mmap_offset(obj); 1098 drm_gem_lru_remove(obj); 1099 } 1100 EXPORT_SYMBOL(drm_gem_object_release); 1101 1102 /** 1103 * drm_gem_object_free - free a GEM object 1104 * @kref: kref of the object to free 1105 * 1106 * Called after the last reference to the object has been lost. 1107 * 1108 * Frees the object 1109 */ 1110 void 1111 drm_gem_object_free(struct kref *kref) 1112 { 1113 struct drm_gem_object *obj = 1114 container_of(kref, struct drm_gem_object, refcount); 1115 1116 if (WARN_ON(!obj->funcs->free)) 1117 return; 1118 1119 obj->funcs->free(obj); 1120 } 1121 EXPORT_SYMBOL(drm_gem_object_free); 1122 1123 /** 1124 * drm_gem_vm_open - vma->ops->open implementation for GEM 1125 * @vma: VM area structure 1126 * 1127 * This function implements the #vm_operations_struct open() callback for GEM 1128 * drivers. This must be used together with drm_gem_vm_close(). 1129 */ 1130 void drm_gem_vm_open(struct vm_area_struct *vma) 1131 { 1132 struct drm_gem_object *obj = vma->vm_private_data; 1133 1134 drm_gem_object_get(obj); 1135 } 1136 EXPORT_SYMBOL(drm_gem_vm_open); 1137 1138 /** 1139 * drm_gem_vm_close - vma->ops->close implementation for GEM 1140 * @vma: VM area structure 1141 * 1142 * This function implements the #vm_operations_struct close() callback for GEM 1143 * drivers. This must be used together with drm_gem_vm_open(). 1144 */ 1145 void drm_gem_vm_close(struct vm_area_struct *vma) 1146 { 1147 struct drm_gem_object *obj = vma->vm_private_data; 1148 1149 drm_gem_object_put(obj); 1150 } 1151 EXPORT_SYMBOL(drm_gem_vm_close); 1152 1153 /** 1154 * drm_gem_mmap_obj - memory map a GEM object 1155 * @obj: the GEM object to map 1156 * @obj_size: the object size to be mapped, in bytes 1157 * @vma: VMA for the area to be mapped 1158 * 1159 * Set up the VMA to prepare mapping of the GEM object using the GEM object's 1160 * vm_ops. Depending on their requirements, GEM objects can either 1161 * provide a fault handler in their vm_ops (in which case any accesses to 1162 * the object will be trapped, to perform migration, GTT binding, surface 1163 * register allocation, or performance monitoring), or mmap the buffer memory 1164 * synchronously after calling drm_gem_mmap_obj. 1165 * 1166 * This function is mainly intended to implement the DMABUF mmap operation, when 1167 * the GEM object is not looked up based on its fake offset. To implement the 1168 * DRM mmap operation, drivers should use the drm_gem_mmap() function. 1169 * 1170 * drm_gem_mmap_obj() assumes the user is granted access to the buffer while 1171 * drm_gem_mmap() prevents unprivileged users from mapping random objects. So 1172 * callers must verify access restrictions before calling this helper. 1173 * 1174 * Return 0 or success or -EINVAL if the object size is smaller than the VMA 1175 * size, or if no vm_ops are provided. 1176 */ 1177 int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size, 1178 struct vm_area_struct *vma) 1179 { 1180 int ret; 1181 1182 /* Check for valid size. */ 1183 if (obj_size < vma->vm_end - vma->vm_start) 1184 return -EINVAL; 1185 1186 /* Take a ref for this mapping of the object, so that the fault 1187 * handler can dereference the mmap offset's pointer to the object. 1188 * This reference is cleaned up by the corresponding vm_close 1189 * (which should happen whether the vma was created by this call, or 1190 * by a vm_open due to mremap or partial unmap or whatever). 1191 */ 1192 drm_gem_object_get(obj); 1193 1194 vma->vm_private_data = obj; 1195 vma->vm_ops = obj->funcs->vm_ops; 1196 1197 if (obj->funcs->mmap) { 1198 ret = obj->funcs->mmap(obj, vma); 1199 if (ret) 1200 goto err_drm_gem_object_put; 1201 WARN_ON(!(vma->vm_flags & VM_DONTEXPAND)); 1202 } else { 1203 if (!vma->vm_ops) { 1204 ret = -EINVAL; 1205 goto err_drm_gem_object_put; 1206 } 1207 1208 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); 1209 vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); 1210 vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); 1211 } 1212 1213 return 0; 1214 1215 err_drm_gem_object_put: 1216 drm_gem_object_put(obj); 1217 return ret; 1218 } 1219 EXPORT_SYMBOL(drm_gem_mmap_obj); 1220 1221 /* 1222 * Look up a GEM object in offset space based on the exact start address. The 1223 * caller must be granted access to the object. Returns a GEM object on success 1224 * or a negative error code on failure. The returned GEM object needs to be 1225 * released with drm_gem_object_put(). 1226 */ 1227 static struct drm_gem_object * 1228 drm_gem_object_lookup_at_offset(struct file *filp, unsigned long start, 1229 unsigned long pages) 1230 { 1231 struct drm_file *priv = filp->private_data; 1232 struct drm_device *dev = priv->minor->dev; 1233 struct drm_gem_object *obj = NULL; 1234 struct drm_vma_offset_node *node; 1235 1236 if (drm_dev_is_unplugged(dev)) 1237 return ERR_PTR(-ENODEV); 1238 1239 drm_vma_offset_lock_lookup(dev->vma_offset_manager); 1240 node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager, 1241 start, pages); 1242 if (likely(node)) { 1243 obj = container_of(node, struct drm_gem_object, vma_node); 1244 /* 1245 * When the object is being freed, after it hits 0-refcnt it 1246 * proceeds to tear down the object. In the process it will 1247 * attempt to remove the VMA offset and so acquire this 1248 * mgr->vm_lock. Therefore if we find an object with a 0-refcnt 1249 * that matches our range, we know it is in the process of being 1250 * destroyed and will be freed as soon as we release the lock - 1251 * so we have to check for the 0-refcnted object and treat it as 1252 * invalid. 1253 */ 1254 if (!kref_get_unless_zero(&obj->refcount)) 1255 obj = NULL; 1256 } 1257 drm_vma_offset_unlock_lookup(dev->vma_offset_manager); 1258 1259 if (!obj) 1260 return ERR_PTR(-EINVAL); 1261 1262 if (!drm_vma_node_is_allowed(node, priv)) { 1263 drm_gem_object_put(obj); 1264 return ERR_PTR(-EACCES); 1265 } 1266 1267 return obj; 1268 } 1269 1270 #ifdef CONFIG_MMU 1271 /** 1272 * drm_gem_get_unmapped_area - get memory mapping region routine for GEM objects 1273 * @filp: DRM file pointer 1274 * @uaddr: User address hint 1275 * @len: Mapping length 1276 * @pgoff: Offset (in pages) 1277 * @flags: Mapping flags 1278 * 1279 * If a driver supports GEM object mapping, before ending up in drm_gem_mmap(), 1280 * mmap calls on the DRM file descriptor will first try to find a free linear 1281 * address space large enough for a mapping. Since GEM objects are backed by 1282 * shmem buffers, this should preferably be handled by the shmem virtual memory 1283 * filesystem which can appropriately align addresses to huge page sizes when 1284 * needed. 1285 * 1286 * Look up the GEM object based on the offset passed in (vma->vm_pgoff will 1287 * contain the fake offset we created) and call shmem_get_unmapped_area() with 1288 * the right file pointer. 1289 * 1290 * If a GEM object is not available at the given offset or if the caller is not 1291 * granted access to it, fall back to mm_get_unmapped_area(). 1292 */ 1293 unsigned long drm_gem_get_unmapped_area(struct file *filp, unsigned long uaddr, 1294 unsigned long len, unsigned long pgoff, 1295 unsigned long flags) 1296 { 1297 struct drm_gem_object *obj; 1298 unsigned long ret; 1299 1300 obj = drm_gem_object_lookup_at_offset(filp, pgoff, len >> PAGE_SHIFT); 1301 if (IS_ERR(obj)) 1302 obj = NULL; 1303 1304 if (!obj || !obj->filp || !obj->filp->f_op->get_unmapped_area) 1305 ret = mm_get_unmapped_area(filp, uaddr, len, 0, flags); 1306 else 1307 ret = obj->filp->f_op->get_unmapped_area(obj->filp, uaddr, len, 0, flags); 1308 1309 drm_gem_object_put(obj); 1310 1311 return ret; 1312 } 1313 EXPORT_SYMBOL_GPL(drm_gem_get_unmapped_area); 1314 #endif 1315 1316 /** 1317 * drm_gem_mmap - memory map routine for GEM objects 1318 * @filp: DRM file pointer 1319 * @vma: VMA for the area to be mapped 1320 * 1321 * If a driver supports GEM object mapping, mmap calls on the DRM file 1322 * descriptor will end up here. 1323 * 1324 * Look up the GEM object based on the offset passed in (vma->vm_pgoff will 1325 * contain the fake offset we created) and map it with a call to 1326 * drm_gem_mmap_obj(). 1327 * 1328 * If the caller is not granted access to the buffer object, the mmap will fail 1329 * with EACCES. Please see the vma manager for more information. 1330 */ 1331 int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma) 1332 { 1333 struct drm_gem_object *obj; 1334 int ret; 1335 1336 obj = drm_gem_object_lookup_at_offset(filp, vma->vm_pgoff, 1337 vma_pages(vma)); 1338 if (IS_ERR(obj)) 1339 return PTR_ERR(obj); 1340 1341 ret = drm_gem_mmap_obj(obj, 1342 drm_vma_node_size(&obj->vma_node) << PAGE_SHIFT, 1343 vma); 1344 1345 drm_gem_object_put(obj); 1346 1347 return ret; 1348 } 1349 EXPORT_SYMBOL(drm_gem_mmap); 1350 1351 void drm_gem_print_info(struct drm_printer *p, unsigned int indent, 1352 const struct drm_gem_object *obj) 1353 { 1354 drm_printf_indent(p, indent, "name=%d\n", obj->name); 1355 drm_printf_indent(p, indent, "refcount=%u\n", 1356 kref_read(&obj->refcount)); 1357 drm_printf_indent(p, indent, "start=%08lx\n", 1358 drm_vma_node_start(&obj->vma_node)); 1359 drm_printf_indent(p, indent, "size=%zu\n", obj->size); 1360 drm_printf_indent(p, indent, "imported=%s\n", 1361 str_yes_no(drm_gem_is_imported(obj))); 1362 1363 if (obj->funcs->print_info) 1364 obj->funcs->print_info(p, indent, obj); 1365 } 1366 1367 int drm_gem_vmap_locked(struct drm_gem_object *obj, struct iosys_map *map) 1368 { 1369 int ret; 1370 1371 dma_resv_assert_held(obj->resv); 1372 1373 if (!obj->funcs->vmap) 1374 return -EOPNOTSUPP; 1375 1376 ret = obj->funcs->vmap(obj, map); 1377 if (ret) 1378 return ret; 1379 else if (iosys_map_is_null(map)) 1380 return -ENOMEM; 1381 1382 return 0; 1383 } 1384 EXPORT_SYMBOL(drm_gem_vmap_locked); 1385 1386 void drm_gem_vunmap_locked(struct drm_gem_object *obj, struct iosys_map *map) 1387 { 1388 dma_resv_assert_held(obj->resv); 1389 1390 if (iosys_map_is_null(map)) 1391 return; 1392 1393 if (obj->funcs->vunmap) 1394 obj->funcs->vunmap(obj, map); 1395 1396 /* Always set the mapping to NULL. Callers may rely on this. */ 1397 iosys_map_clear(map); 1398 } 1399 EXPORT_SYMBOL(drm_gem_vunmap_locked); 1400 1401 void drm_gem_lock(struct drm_gem_object *obj) 1402 { 1403 dma_resv_lock(obj->resv, NULL); 1404 } 1405 EXPORT_SYMBOL(drm_gem_lock); 1406 1407 void drm_gem_unlock(struct drm_gem_object *obj) 1408 { 1409 dma_resv_unlock(obj->resv); 1410 } 1411 EXPORT_SYMBOL(drm_gem_unlock); 1412 1413 int drm_gem_vmap(struct drm_gem_object *obj, struct iosys_map *map) 1414 { 1415 int ret; 1416 1417 dma_resv_lock(obj->resv, NULL); 1418 ret = drm_gem_vmap_locked(obj, map); 1419 dma_resv_unlock(obj->resv); 1420 1421 return ret; 1422 } 1423 EXPORT_SYMBOL(drm_gem_vmap); 1424 1425 void drm_gem_vunmap(struct drm_gem_object *obj, struct iosys_map *map) 1426 { 1427 dma_resv_lock(obj->resv, NULL); 1428 drm_gem_vunmap_locked(obj, map); 1429 dma_resv_unlock(obj->resv); 1430 } 1431 EXPORT_SYMBOL(drm_gem_vunmap); 1432 1433 /** 1434 * drm_gem_lock_reservations - Sets up the ww context and acquires 1435 * the lock on an array of GEM objects. 1436 * 1437 * Once you've locked your reservations, you'll want to set up space 1438 * for your shared fences (if applicable), submit your job, then 1439 * drm_gem_unlock_reservations(). 1440 * 1441 * @objs: drm_gem_objects to lock 1442 * @count: Number of objects in @objs 1443 * @acquire_ctx: struct ww_acquire_ctx that will be initialized as 1444 * part of tracking this set of locked reservations. 1445 */ 1446 int 1447 drm_gem_lock_reservations(struct drm_gem_object **objs, int count, 1448 struct ww_acquire_ctx *acquire_ctx) 1449 { 1450 int contended = -1; 1451 int i, ret; 1452 1453 ww_acquire_init(acquire_ctx, &reservation_ww_class); 1454 1455 retry: 1456 if (contended != -1) { 1457 struct drm_gem_object *obj = objs[contended]; 1458 1459 ret = dma_resv_lock_slow_interruptible(obj->resv, 1460 acquire_ctx); 1461 if (ret) { 1462 ww_acquire_fini(acquire_ctx); 1463 return ret; 1464 } 1465 } 1466 1467 for (i = 0; i < count; i++) { 1468 if (i == contended) 1469 continue; 1470 1471 ret = dma_resv_lock_interruptible(objs[i]->resv, 1472 acquire_ctx); 1473 if (ret) { 1474 int j; 1475 1476 for (j = 0; j < i; j++) 1477 dma_resv_unlock(objs[j]->resv); 1478 1479 if (contended != -1 && contended >= i) 1480 dma_resv_unlock(objs[contended]->resv); 1481 1482 if (ret == -EDEADLK) { 1483 contended = i; 1484 goto retry; 1485 } 1486 1487 ww_acquire_fini(acquire_ctx); 1488 return ret; 1489 } 1490 } 1491 1492 ww_acquire_done(acquire_ctx); 1493 1494 return 0; 1495 } 1496 EXPORT_SYMBOL(drm_gem_lock_reservations); 1497 1498 void 1499 drm_gem_unlock_reservations(struct drm_gem_object **objs, int count, 1500 struct ww_acquire_ctx *acquire_ctx) 1501 { 1502 int i; 1503 1504 for (i = 0; i < count; i++) 1505 dma_resv_unlock(objs[i]->resv); 1506 1507 ww_acquire_fini(acquire_ctx); 1508 } 1509 EXPORT_SYMBOL(drm_gem_unlock_reservations); 1510 1511 /** 1512 * drm_gem_lru_init - initialize a LRU 1513 * 1514 * @lru: The LRU to initialize 1515 * @lock: The lock protecting the LRU 1516 */ 1517 void 1518 drm_gem_lru_init(struct drm_gem_lru *lru, struct mutex *lock) 1519 { 1520 lru->lock = lock; 1521 lru->count = 0; 1522 INIT_LIST_HEAD(&lru->list); 1523 } 1524 EXPORT_SYMBOL(drm_gem_lru_init); 1525 1526 static void 1527 drm_gem_lru_remove_locked(struct drm_gem_object *obj) 1528 { 1529 obj->lru->count -= obj->size >> PAGE_SHIFT; 1530 WARN_ON(obj->lru->count < 0); 1531 list_del(&obj->lru_node); 1532 obj->lru = NULL; 1533 } 1534 1535 /** 1536 * drm_gem_lru_remove - remove object from whatever LRU it is in 1537 * 1538 * If the object is currently in any LRU, remove it. 1539 * 1540 * @obj: The GEM object to remove from current LRU 1541 */ 1542 void 1543 drm_gem_lru_remove(struct drm_gem_object *obj) 1544 { 1545 struct drm_gem_lru *lru = obj->lru; 1546 1547 if (!lru) 1548 return; 1549 1550 mutex_lock(lru->lock); 1551 drm_gem_lru_remove_locked(obj); 1552 mutex_unlock(lru->lock); 1553 } 1554 EXPORT_SYMBOL(drm_gem_lru_remove); 1555 1556 /** 1557 * drm_gem_lru_move_tail_locked - move the object to the tail of the LRU 1558 * 1559 * Like &drm_gem_lru_move_tail but lru lock must be held 1560 * 1561 * @lru: The LRU to move the object into. 1562 * @obj: The GEM object to move into this LRU 1563 */ 1564 void 1565 drm_gem_lru_move_tail_locked(struct drm_gem_lru *lru, struct drm_gem_object *obj) 1566 { 1567 lockdep_assert_held_once(lru->lock); 1568 1569 if (obj->lru) 1570 drm_gem_lru_remove_locked(obj); 1571 1572 lru->count += obj->size >> PAGE_SHIFT; 1573 list_add_tail(&obj->lru_node, &lru->list); 1574 obj->lru = lru; 1575 } 1576 EXPORT_SYMBOL(drm_gem_lru_move_tail_locked); 1577 1578 /** 1579 * drm_gem_lru_move_tail - move the object to the tail of the LRU 1580 * 1581 * If the object is already in this LRU it will be moved to the 1582 * tail. Otherwise it will be removed from whichever other LRU 1583 * it is in (if any) and moved into this LRU. 1584 * 1585 * @lru: The LRU to move the object into. 1586 * @obj: The GEM object to move into this LRU 1587 */ 1588 void 1589 drm_gem_lru_move_tail(struct drm_gem_lru *lru, struct drm_gem_object *obj) 1590 { 1591 mutex_lock(lru->lock); 1592 drm_gem_lru_move_tail_locked(lru, obj); 1593 mutex_unlock(lru->lock); 1594 } 1595 EXPORT_SYMBOL(drm_gem_lru_move_tail); 1596 1597 /** 1598 * drm_gem_lru_scan - helper to implement shrinker.scan_objects 1599 * 1600 * If the shrink callback succeeds, it is expected that the driver 1601 * move the object out of this LRU. 1602 * 1603 * If the LRU possibly contain active buffers, it is the responsibility 1604 * of the shrink callback to check for this (ie. dma_resv_test_signaled()) 1605 * or if necessary block until the buffer becomes idle. 1606 * 1607 * @lru: The LRU to scan 1608 * @nr_to_scan: The number of pages to try to reclaim 1609 * @remaining: The number of pages left to reclaim, should be initialized by caller 1610 * @shrink: Callback to try to shrink/reclaim the object. 1611 * @ticket: Optional ww_acquire_ctx context to use for locking 1612 */ 1613 unsigned long 1614 drm_gem_lru_scan(struct drm_gem_lru *lru, 1615 unsigned int nr_to_scan, 1616 unsigned long *remaining, 1617 bool (*shrink)(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket), 1618 struct ww_acquire_ctx *ticket) 1619 { 1620 struct drm_gem_lru still_in_lru; 1621 struct drm_gem_object *obj; 1622 unsigned freed = 0; 1623 1624 drm_gem_lru_init(&still_in_lru, lru->lock); 1625 1626 mutex_lock(lru->lock); 1627 1628 while (freed < nr_to_scan) { 1629 obj = list_first_entry_or_null(&lru->list, typeof(*obj), lru_node); 1630 1631 if (!obj) 1632 break; 1633 1634 drm_gem_lru_move_tail_locked(&still_in_lru, obj); 1635 1636 /* 1637 * If it's in the process of being freed, gem_object->free() 1638 * may be blocked on lock waiting to remove it. So just 1639 * skip it. 1640 */ 1641 if (!kref_get_unless_zero(&obj->refcount)) 1642 continue; 1643 1644 /* 1645 * Now that we own a reference, we can drop the lock for the 1646 * rest of the loop body, to reduce contention with other 1647 * code paths that need the LRU lock 1648 */ 1649 mutex_unlock(lru->lock); 1650 1651 if (ticket) 1652 ww_acquire_init(ticket, &reservation_ww_class); 1653 1654 /* 1655 * Note that this still needs to be trylock, since we can 1656 * hit shrinker in response to trying to get backing pages 1657 * for this obj (ie. while it's lock is already held) 1658 */ 1659 if (!ww_mutex_trylock(&obj->resv->lock, ticket)) { 1660 *remaining += obj->size >> PAGE_SHIFT; 1661 goto tail; 1662 } 1663 1664 if (shrink(obj, ticket)) { 1665 freed += obj->size >> PAGE_SHIFT; 1666 1667 /* 1668 * If we succeeded in releasing the object's backing 1669 * pages, we expect the driver to have moved the object 1670 * out of this LRU 1671 */ 1672 WARN_ON(obj->lru == &still_in_lru); 1673 WARN_ON(obj->lru == lru); 1674 } 1675 1676 dma_resv_unlock(obj->resv); 1677 1678 if (ticket) 1679 ww_acquire_fini(ticket); 1680 1681 tail: 1682 drm_gem_object_put(obj); 1683 mutex_lock(lru->lock); 1684 } 1685 1686 /* 1687 * Move objects we've skipped over out of the temporary still_in_lru 1688 * back into this LRU 1689 */ 1690 list_for_each_entry (obj, &still_in_lru.list, lru_node) 1691 obj->lru = lru; 1692 list_splice_tail(&still_in_lru.list, &lru->list); 1693 lru->count += still_in_lru.count; 1694 1695 mutex_unlock(lru->lock); 1696 1697 return freed; 1698 } 1699 EXPORT_SYMBOL(drm_gem_lru_scan); 1700 1701 /** 1702 * drm_gem_evict_locked - helper to evict backing pages for a GEM object 1703 * @obj: obj in question 1704 */ 1705 int drm_gem_evict_locked(struct drm_gem_object *obj) 1706 { 1707 dma_resv_assert_held(obj->resv); 1708 1709 if (!dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ)) 1710 return -EBUSY; 1711 1712 if (obj->funcs->evict) 1713 return obj->funcs->evict(obj); 1714 1715 return 0; 1716 } 1717 EXPORT_SYMBOL(drm_gem_evict_locked); 1718