1 /* 2 * Copyright © 2008 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 */ 27 28 #include <linux/dma-buf.h> 29 #include <linux/export.h> 30 #include <linux/file.h> 31 #include <linux/fs.h> 32 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 33 #include <linux/fs_context.h> 34 #endif 35 #include <linux/iosys-map.h> 36 #include <linux/mem_encrypt.h> 37 #include <linux/mm.h> 38 #include <linux/mman.h> 39 #include <linux/module.h> 40 #include <linux/pagemap.h> 41 #include <linux/folio_batch.h> 42 #include <linux/sched/mm.h> 43 #include <linux/shmem_fs.h> 44 #include <linux/slab.h> 45 #include <linux/string_helpers.h> 46 #include <linux/types.h> 47 #include <linux/uaccess.h> 48 49 #include <drm/drm.h> 50 #include <drm/drm_device.h> 51 #include <drm/drm_drv.h> 52 #include <drm/drm_file.h> 53 #include <drm/drm_gem.h> 54 #include <drm/drm_managed.h> 55 #include <drm/drm_print.h> 56 #include <drm/drm_vma_manager.h> 57 58 #include "drm_internal.h" 59 60 /** @file drm_gem.c 61 * 62 * This file provides some of the base ioctls and library routines for 63 * the graphics memory manager implemented by each device driver. 64 * 65 * Because various devices have different requirements in terms of 66 * synchronization and migration strategies, implementing that is left up to 67 * the driver, and all that the general API provides should be generic -- 68 * allocating objects, reading/writing data with the cpu, freeing objects. 69 * Even there, platform-dependent optimizations for reading/writing data with 70 * the CPU mean we'll likely hook those out to driver-specific calls. However, 71 * the DRI2 implementation wants to have at least allocate/mmap be generic. 72 * 73 * The goal was to have swap-backed object allocation managed through 74 * struct file. However, file descriptors as handles to a struct file have 75 * two major failings: 76 * - Process limits prevent more than 1024 or so being used at a time by 77 * default. 78 * - Inability to allocate high fds will aggravate the X Server's select() 79 * handling, and likely that of many GL client applications as well. 80 * 81 * This led to a plan of using our own integer IDs (called handles, following 82 * DRM terminology) to mimic fds, and implement the fd syscalls we need as 83 * ioctls. The objects themselves will still include the struct file so 84 * that we can transition to fds if the required kernel infrastructure shows 85 * up at a later date, and as our interface with shmfs for memory allocation. 86 */ 87 88 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 89 static void drm_gem_huge_mnt_free(struct drm_device *dev, void *data) 90 { 91 kern_unmount(dev->huge_mnt); 92 } 93 94 /** 95 * drm_gem_huge_mnt_create - Create, mount and use a huge tmpfs mountpoint 96 * @dev: DRM device that will use the huge tmpfs mountpoint 97 * @value: huge tmpfs mount option value 98 * 99 * This function creates and mounts a dedicated huge tmpfs mountpoint for the 100 * lifetime of the DRM device @dev which is used at GEM object initialization 101 * with drm_gem_object_init(). 102 * 103 * The most common option for @value is "within_size" which only allocates huge 104 * pages if the page will be fully within the GEM object size. "always", 105 * "advise" and "never" are supported too but the latter would just create a 106 * mountpoint similar to the default one (`shm_mnt`). See shmemfs and 107 * Transparent Hugepage for more information. 108 * 109 * Returns: 110 * 0 on success or a negative error code on failure. 111 */ 112 int drm_gem_huge_mnt_create(struct drm_device *dev, const char *value) 113 { 114 struct file_system_type *type; 115 struct fs_context *fc; 116 int ret; 117 118 if (unlikely(drm_gem_get_huge_mnt(dev))) 119 return 0; 120 121 type = get_fs_type("tmpfs"); 122 if (unlikely(!type)) 123 return -EOPNOTSUPP; 124 fc = fs_context_for_mount(type, SB_KERNMOUNT); 125 if (IS_ERR(fc)) 126 return PTR_ERR(fc); 127 ret = vfs_parse_fs_string(fc, "source", "tmpfs"); 128 if (unlikely(ret)) 129 return -ENOPARAM; 130 ret = vfs_parse_fs_string(fc, "huge", value); 131 if (unlikely(ret)) 132 return -ENOPARAM; 133 134 dev->huge_mnt = fc_mount_longterm(fc); 135 put_fs_context(fc); 136 137 return drmm_add_action_or_reset(dev, drm_gem_huge_mnt_free, NULL); 138 } 139 EXPORT_SYMBOL_GPL(drm_gem_huge_mnt_create); 140 #endif 141 142 static void 143 drm_gem_init_release(struct drm_device *dev, void *ptr) 144 { 145 drm_vma_offset_manager_destroy(dev->vma_offset_manager); 146 } 147 148 /** 149 * drm_gem_init - Initialize the GEM device fields 150 * @dev: drm_devic structure to initialize 151 */ 152 int 153 drm_gem_init(struct drm_device *dev) 154 { 155 struct drm_vma_offset_manager *vma_offset_manager; 156 157 mutex_init(&dev->object_name_lock); 158 idr_init_base(&dev->object_name_idr, 1); 159 160 vma_offset_manager = drmm_kzalloc(dev, sizeof(*vma_offset_manager), 161 GFP_KERNEL); 162 if (!vma_offset_manager) 163 return -ENOMEM; 164 165 dev->vma_offset_manager = vma_offset_manager; 166 drm_vma_offset_manager_init(vma_offset_manager, 167 DRM_FILE_PAGE_OFFSET_START, 168 DRM_FILE_PAGE_OFFSET_SIZE); 169 170 return drmm_add_action(dev, drm_gem_init_release, NULL); 171 } 172 173 /** 174 * drm_gem_object_init - initialize an allocated shmem-backed GEM object 175 * 176 * @dev: drm_device the object should be initialized for 177 * @obj: drm_gem_object to initialize 178 * @size: object size 179 * 180 * Initialize an already allocated GEM object of the specified size with 181 * shmfs backing store. A huge mountpoint can be used by calling 182 * drm_gem_huge_mnt_create() beforehand. 183 */ 184 int drm_gem_object_init(struct drm_device *dev, struct drm_gem_object *obj, 185 size_t size) 186 { 187 struct vfsmount *huge_mnt; 188 struct file *filp; 189 const vma_flags_t flags = mk_vma_flags(VMA_NORESERVE_BIT); 190 191 drm_gem_private_object_init(dev, obj, size); 192 193 huge_mnt = drm_gem_get_huge_mnt(dev); 194 if (huge_mnt) 195 filp = shmem_file_setup_with_mnt(huge_mnt, "drm mm object", 196 size, flags); 197 else 198 filp = shmem_file_setup("drm mm object", size, flags); 199 200 if (IS_ERR(filp)) 201 return PTR_ERR(filp); 202 203 obj->filp = filp; 204 205 return 0; 206 } 207 EXPORT_SYMBOL(drm_gem_object_init); 208 209 /** 210 * drm_gem_private_object_init - initialize an allocated private GEM object 211 * @dev: drm_device the object should be initialized for 212 * @obj: drm_gem_object to initialize 213 * @size: object size 214 * 215 * Initialize an already allocated GEM object of the specified size with 216 * no GEM provided backing store. Instead the caller is responsible for 217 * backing the object and handling it. 218 */ 219 void drm_gem_private_object_init(struct drm_device *dev, 220 struct drm_gem_object *obj, size_t size) 221 { 222 BUG_ON((size & (PAGE_SIZE - 1)) != 0); 223 224 obj->dev = dev; 225 obj->filp = NULL; 226 227 kref_init(&obj->refcount); 228 obj->handle_count = 0; 229 obj->size = size; 230 mutex_init(&obj->gpuva.lock); 231 dma_resv_init(&obj->_resv); 232 if (!obj->resv) 233 obj->resv = &obj->_resv; 234 235 drm_gem_gpuva_init(obj); 236 237 drm_vma_node_reset(&obj->vma_node); 238 INIT_LIST_HEAD(&obj->lru_node); 239 } 240 EXPORT_SYMBOL(drm_gem_private_object_init); 241 242 /** 243 * drm_gem_private_object_fini - Finalize a failed drm_gem_object 244 * @obj: drm_gem_object 245 * 246 * Uninitialize an already allocated GEM object when it initialized failed 247 */ 248 void drm_gem_private_object_fini(struct drm_gem_object *obj) 249 { 250 WARN_ON(obj->dma_buf); 251 252 dma_resv_fini(&obj->_resv); 253 mutex_destroy(&obj->gpuva.lock); 254 } 255 EXPORT_SYMBOL(drm_gem_private_object_fini); 256 257 static void drm_gem_object_handle_get(struct drm_gem_object *obj) 258 { 259 struct drm_device *dev = obj->dev; 260 261 drm_WARN_ON(dev, !mutex_is_locked(&dev->object_name_lock)); 262 263 if (obj->handle_count++ == 0) 264 drm_gem_object_get(obj); 265 } 266 267 /** 268 * drm_gem_object_handle_get_if_exists_unlocked - acquire reference on user-space handle, if any 269 * @obj: GEM object 270 * 271 * Acquires a reference on the GEM buffer object's handle. Required to keep 272 * the GEM object alive. Call drm_gem_object_handle_put_if_exists_unlocked() 273 * to release the reference. Does nothing if the buffer object has no handle. 274 * 275 * Returns: 276 * True if a handle exists, or false otherwise 277 */ 278 bool drm_gem_object_handle_get_if_exists_unlocked(struct drm_gem_object *obj) 279 { 280 struct drm_device *dev = obj->dev; 281 282 guard(mutex)(&dev->object_name_lock); 283 284 /* 285 * First ref taken during GEM object creation, if any. Some 286 * drivers set up internal framebuffers with GEM objects that 287 * do not have a GEM handle. Hence, this counter can be zero. 288 */ 289 if (!obj->handle_count) 290 return false; 291 292 drm_gem_object_handle_get(obj); 293 294 return true; 295 } 296 297 /** 298 * drm_gem_object_handle_free - release resources bound to userspace handles 299 * @obj: GEM object to clean up. 300 * 301 * Called after the last handle to the object has been closed 302 * 303 * Removes any name for the object. Note that this must be 304 * called before drm_gem_object_free or we'll be touching 305 * freed memory 306 */ 307 static void drm_gem_object_handle_free(struct drm_gem_object *obj) 308 { 309 struct drm_device *dev = obj->dev; 310 311 /* Remove any name for this object */ 312 if (obj->name) { 313 idr_remove(&dev->object_name_idr, obj->name); 314 obj->name = 0; 315 } 316 } 317 318 static void drm_gem_object_exported_dma_buf_free(struct drm_gem_object *obj) 319 { 320 /* Unbreak the reference cycle if we have an exported dma_buf. */ 321 if (obj->dma_buf) { 322 dma_buf_put(obj->dma_buf); 323 obj->dma_buf = NULL; 324 } 325 } 326 327 /** 328 * drm_gem_object_handle_put_unlocked - releases reference on user-space handle 329 * @obj: GEM object 330 * 331 * Releases a reference on the GEM buffer object's handle. Possibly releases 332 * the GEM buffer object and associated dma-buf objects. 333 */ 334 void drm_gem_object_handle_put_unlocked(struct drm_gem_object *obj) 335 { 336 struct drm_device *dev = obj->dev; 337 bool final = false; 338 339 if (drm_WARN_ON(dev, READ_ONCE(obj->handle_count) == 0)) 340 return; 341 342 /* 343 * Must bump handle count first as this may be the last 344 * ref, in which case the object would disappear before 345 * we checked for a name. 346 */ 347 348 mutex_lock(&dev->object_name_lock); 349 if (--obj->handle_count == 0) { 350 drm_gem_object_handle_free(obj); 351 drm_gem_object_exported_dma_buf_free(obj); 352 final = true; 353 } 354 mutex_unlock(&dev->object_name_lock); 355 356 if (final) { 357 if (obj->funcs->handle_free) 358 obj->funcs->handle_free(obj); 359 drm_gem_object_put(obj); 360 } 361 } 362 363 /* 364 * Called at device or object close to release the file's 365 * handle references on objects. 366 */ 367 static int 368 drm_gem_object_release_handle(int id, void *ptr, void *data) 369 { 370 struct drm_file *file_priv = data; 371 struct drm_gem_object *obj = ptr; 372 373 if (drm_WARN_ON(obj->dev, !data)) 374 return 0; 375 376 if (obj->funcs->close) 377 obj->funcs->close(obj, file_priv); 378 379 mutex_lock(&file_priv->prime.lock); 380 381 drm_prime_remove_buf_handle(&file_priv->prime, id); 382 383 mutex_unlock(&file_priv->prime.lock); 384 385 drm_vma_node_revoke(&obj->vma_node, file_priv); 386 387 drm_gem_object_handle_put_unlocked(obj); 388 389 return 0; 390 } 391 392 /** 393 * drm_gem_handle_delete - deletes the given file-private handle 394 * @filp: drm file-private structure to use for the handle look up 395 * @handle: userspace handle to delete 396 * 397 * Removes the GEM handle from the @filp lookup table which has been added with 398 * drm_gem_handle_create(). If this is the last handle also cleans up linked 399 * resources like GEM names. 400 */ 401 int 402 drm_gem_handle_delete(struct drm_file *filp, u32 handle) 403 { 404 struct drm_gem_object *obj; 405 406 spin_lock(&filp->table_lock); 407 408 /* Check if we currently have a reference on the object */ 409 obj = idr_replace(&filp->object_idr, NULL, handle); 410 spin_unlock(&filp->table_lock); 411 if (IS_ERR_OR_NULL(obj)) 412 return -EINVAL; 413 414 /* Release driver's reference and decrement refcount. */ 415 drm_gem_object_release_handle(handle, obj, filp); 416 417 /* And finally make the handle available for future allocations. */ 418 spin_lock(&filp->table_lock); 419 idr_remove(&filp->object_idr, handle); 420 spin_unlock(&filp->table_lock); 421 422 return 0; 423 } 424 EXPORT_SYMBOL(drm_gem_handle_delete); 425 426 /** 427 * drm_gem_dumb_map_offset - return the fake mmap offset for a gem object 428 * @file: drm file-private structure containing the gem object 429 * @dev: corresponding drm_device 430 * @handle: gem object handle 431 * @offset: return location for the fake mmap offset 432 * 433 * This implements the &drm_driver.dumb_map_offset kms driver callback for 434 * drivers which use gem to manage their backing storage. 435 * 436 * Returns: 437 * 0 on success or a negative error code on failure. 438 */ 439 int drm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev, 440 u32 handle, u64 *offset) 441 { 442 struct drm_gem_object *obj; 443 int ret; 444 445 obj = drm_gem_object_lookup(file, handle); 446 if (!obj) 447 return -ENOENT; 448 449 /* Don't allow imported objects to be mapped */ 450 if (drm_gem_is_imported(obj)) { 451 ret = -EINVAL; 452 goto out; 453 } 454 455 ret = drm_gem_create_mmap_offset(obj); 456 if (ret) 457 goto out; 458 459 *offset = drm_vma_node_offset_addr(&obj->vma_node); 460 out: 461 drm_gem_object_put(obj); 462 463 return ret; 464 } 465 EXPORT_SYMBOL_GPL(drm_gem_dumb_map_offset); 466 467 /** 468 * drm_gem_handle_create_tail - internal functions to create a handle 469 * @file_priv: drm file-private structure to register the handle for 470 * @obj: object to register 471 * @handlep: pointer to return the created handle to the caller 472 * 473 * This expects the &drm_device.object_name_lock to be held already and will 474 * drop it before returning. Used to avoid races in establishing new handles 475 * when importing an object from either an flink name or a dma-buf. 476 * 477 * Handles must be release again through drm_gem_handle_delete(). This is done 478 * when userspace closes @file_priv for all attached handles, or through the 479 * GEM_CLOSE ioctl for individual handles. 480 */ 481 int 482 drm_gem_handle_create_tail(struct drm_file *file_priv, 483 struct drm_gem_object *obj, 484 u32 *handlep) 485 { 486 struct drm_device *dev = obj->dev; 487 u32 handle; 488 int ret; 489 490 WARN_ON(!mutex_is_locked(&dev->object_name_lock)); 491 492 drm_gem_object_handle_get(obj); 493 494 /* 495 * Get the user-visible handle using idr. Preload and perform 496 * allocation under our spinlock. 497 */ 498 idr_preload(GFP_KERNEL); 499 spin_lock(&file_priv->table_lock); 500 501 ret = idr_alloc(&file_priv->object_idr, NULL, 1, 0, GFP_NOWAIT); 502 503 spin_unlock(&file_priv->table_lock); 504 idr_preload_end(); 505 506 mutex_unlock(&dev->object_name_lock); 507 if (ret < 0) 508 goto err_unref; 509 510 handle = ret; 511 512 ret = drm_vma_node_allow(&obj->vma_node, file_priv); 513 if (ret) 514 goto err_remove; 515 516 if (obj->funcs->open) { 517 ret = obj->funcs->open(obj, file_priv); 518 if (ret) 519 goto err_revoke; 520 } 521 522 /* mirrors drm_gem_handle_delete to avoid races */ 523 spin_lock(&file_priv->table_lock); 524 obj = idr_replace(&file_priv->object_idr, obj, handle); 525 WARN_ON(obj != NULL); 526 spin_unlock(&file_priv->table_lock); 527 *handlep = handle; 528 return 0; 529 530 err_revoke: 531 drm_vma_node_revoke(&obj->vma_node, file_priv); 532 err_remove: 533 spin_lock(&file_priv->table_lock); 534 idr_remove(&file_priv->object_idr, handle); 535 spin_unlock(&file_priv->table_lock); 536 err_unref: 537 drm_gem_object_handle_put_unlocked(obj); 538 return ret; 539 } 540 541 /** 542 * drm_gem_handle_create - create a gem handle for an object 543 * @file_priv: drm file-private structure to register the handle for 544 * @obj: object to register 545 * @handlep: pointer to return the created handle to the caller 546 * 547 * Create a handle for this object. This adds a handle reference to the object, 548 * which includes a regular reference count. Callers will likely want to 549 * dereference the object afterwards. 550 * 551 * Since this publishes @obj to userspace it must be fully set up by this point, 552 * drivers must call this last in their buffer object creation callbacks. 553 */ 554 int drm_gem_handle_create(struct drm_file *file_priv, 555 struct drm_gem_object *obj, 556 u32 *handlep) 557 { 558 mutex_lock(&obj->dev->object_name_lock); 559 560 return drm_gem_handle_create_tail(file_priv, obj, handlep); 561 } 562 EXPORT_SYMBOL(drm_gem_handle_create); 563 564 565 /** 566 * drm_gem_free_mmap_offset - release a fake mmap offset for an object 567 * @obj: obj in question 568 * 569 * This routine frees fake offsets allocated by drm_gem_create_mmap_offset(). 570 * 571 * Note that drm_gem_object_release() already calls this function, so drivers 572 * don't have to take care of releasing the mmap offset themselves when freeing 573 * the GEM object. 574 */ 575 void 576 drm_gem_free_mmap_offset(struct drm_gem_object *obj) 577 { 578 struct drm_device *dev = obj->dev; 579 580 drm_vma_offset_remove(dev->vma_offset_manager, &obj->vma_node); 581 } 582 EXPORT_SYMBOL(drm_gem_free_mmap_offset); 583 584 /** 585 * drm_gem_create_mmap_offset_size - create a fake mmap offset for an object 586 * @obj: obj in question 587 * @size: the virtual size 588 * 589 * GEM memory mapping works by handing back to userspace a fake mmap offset 590 * it can use in a subsequent mmap(2) call. The DRM core code then looks 591 * up the object based on the offset and sets up the various memory mapping 592 * structures. 593 * 594 * This routine allocates and attaches a fake offset for @obj, in cases where 595 * the virtual size differs from the physical size (ie. &drm_gem_object.size). 596 * Otherwise just use drm_gem_create_mmap_offset(). 597 * 598 * This function is idempotent and handles an already allocated mmap offset 599 * transparently. Drivers do not need to check for this case. 600 */ 601 int 602 drm_gem_create_mmap_offset_size(struct drm_gem_object *obj, size_t size) 603 { 604 struct drm_device *dev = obj->dev; 605 606 return drm_vma_offset_add(dev->vma_offset_manager, &obj->vma_node, 607 size / PAGE_SIZE); 608 } 609 EXPORT_SYMBOL(drm_gem_create_mmap_offset_size); 610 611 /** 612 * drm_gem_create_mmap_offset - create a fake mmap offset for an object 613 * @obj: obj in question 614 * 615 * GEM memory mapping works by handing back to userspace a fake mmap offset 616 * it can use in a subsequent mmap(2) call. The DRM core code then looks 617 * up the object based on the offset and sets up the various memory mapping 618 * structures. 619 * 620 * This routine allocates and attaches a fake offset for @obj. 621 * 622 * Drivers can call drm_gem_free_mmap_offset() before freeing @obj to release 623 * the fake offset again. 624 */ 625 int drm_gem_create_mmap_offset(struct drm_gem_object *obj) 626 { 627 return drm_gem_create_mmap_offset_size(obj, obj->size); 628 } 629 EXPORT_SYMBOL(drm_gem_create_mmap_offset); 630 631 /* 632 * Move folios to appropriate lru and release the folios, decrementing the 633 * ref count of those folios. 634 */ 635 static void drm_gem_check_release_batch(struct folio_batch *fbatch) 636 { 637 check_move_unevictable_folios(fbatch); 638 __folio_batch_release(fbatch); 639 cond_resched(); 640 } 641 642 /** 643 * drm_gem_get_pages - helper to allocate backing pages for a GEM object 644 * from shmem 645 * @obj: obj in question 646 * 647 * This reads the page-array of the shmem-backing storage of the given gem 648 * object. An array of pages is returned. If a page is not allocated or 649 * swapped-out, this will allocate/swap-in the required pages. Note that the 650 * whole object is covered by the page-array and pinned in memory. 651 * 652 * Use drm_gem_put_pages() to release the array and unpin all pages. 653 * 654 * This uses the GFP-mask set on the shmem-mapping (see mapping_set_gfp_mask()). 655 * If you require other GFP-masks, you have to do those allocations yourself. 656 * 657 * Note that you are not allowed to change gfp-zones during runtime. That is, 658 * shmem_read_mapping_page_gfp() must be called with the same gfp_zone(gfp) as 659 * set during initialization. If you have special zone constraints, set them 660 * after drm_gem_object_init() via mapping_set_gfp_mask(). shmem-core takes care 661 * to keep pages in the required zone during swap-in. 662 * 663 * This function is only valid on objects initialized with 664 * drm_gem_object_init(), but not for those initialized with 665 * drm_gem_private_object_init() only. 666 */ 667 struct page **drm_gem_get_pages(struct drm_gem_object *obj) 668 { 669 struct address_space *mapping; 670 struct page **pages; 671 struct folio *folio; 672 struct folio_batch fbatch; 673 unsigned long i, j, npages; 674 675 if (WARN_ON(!obj->filp)) 676 return ERR_PTR(-EINVAL); 677 678 /* This is the shared memory object that backs the GEM resource */ 679 mapping = obj->filp->f_mapping; 680 681 /* We already BUG_ON() for non-page-aligned sizes in 682 * drm_gem_object_init(), so we should never hit this unless 683 * driver author is doing something really wrong: 684 */ 685 WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0); 686 687 npages = obj->size >> PAGE_SHIFT; 688 689 pages = kvmalloc_objs(struct page *, npages); 690 if (pages == NULL) 691 return ERR_PTR(-ENOMEM); 692 693 mapping_set_unevictable(mapping); 694 695 i = 0; 696 while (i < npages) { 697 unsigned long nr; 698 folio = shmem_read_folio_gfp(mapping, i, 699 mapping_gfp_mask(mapping)); 700 if (IS_ERR(folio)) 701 goto fail; 702 nr = min(npages - i, folio_nr_pages(folio)); 703 for (j = 0; j < nr; j++, i++) 704 pages[i] = folio_file_page(folio, i); 705 706 /* Make sure shmem keeps __GFP_DMA32 allocated pages in the 707 * correct region during swapin. Note that this requires 708 * __GFP_DMA32 to be set in mapping_gfp_mask(inode->i_mapping) 709 * so shmem can relocate pages during swapin if required. 710 */ 711 BUG_ON(mapping_gfp_constraint(mapping, __GFP_DMA32) && 712 (folio_pfn(folio) >= 0x00100000UL)); 713 } 714 715 return pages; 716 717 fail: 718 mapping_clear_unevictable(mapping); 719 folio_batch_init(&fbatch); 720 j = 0; 721 while (j < i) { 722 struct folio *f = page_folio(pages[j]); 723 if (!folio_batch_add(&fbatch, f)) 724 drm_gem_check_release_batch(&fbatch); 725 j += folio_nr_pages(f); 726 } 727 if (fbatch.nr) 728 drm_gem_check_release_batch(&fbatch); 729 730 kvfree(pages); 731 return ERR_CAST(folio); 732 } 733 EXPORT_SYMBOL(drm_gem_get_pages); 734 735 /** 736 * drm_gem_put_pages - helper to free backing pages for a GEM object 737 * @obj: obj in question 738 * @pages: pages to free 739 * @dirty: if true, pages will be marked as dirty 740 * @accessed: if true, the pages will be marked as accessed 741 */ 742 void drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages, 743 bool dirty, bool accessed) 744 { 745 int i, npages; 746 struct address_space *mapping; 747 struct folio_batch fbatch; 748 749 mapping = file_inode(obj->filp)->i_mapping; 750 mapping_clear_unevictable(mapping); 751 752 /* We already BUG_ON() for non-page-aligned sizes in 753 * drm_gem_object_init(), so we should never hit this unless 754 * driver author is doing something really wrong: 755 */ 756 WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0); 757 758 npages = obj->size >> PAGE_SHIFT; 759 760 folio_batch_init(&fbatch); 761 for (i = 0; i < npages; i++) { 762 struct folio *folio; 763 764 if (!pages[i]) 765 continue; 766 folio = page_folio(pages[i]); 767 768 if (dirty) 769 folio_mark_dirty(folio); 770 771 if (accessed) 772 folio_mark_accessed(folio); 773 774 /* Undo the reference we took when populating the table */ 775 if (!folio_batch_add(&fbatch, folio)) 776 drm_gem_check_release_batch(&fbatch); 777 i += folio_nr_pages(folio) - 1; 778 } 779 if (folio_batch_count(&fbatch)) 780 drm_gem_check_release_batch(&fbatch); 781 782 kvfree(pages); 783 } 784 EXPORT_SYMBOL(drm_gem_put_pages); 785 786 static int objects_lookup(struct drm_file *filp, u32 *handle, int count, 787 struct drm_gem_object **objs) 788 { 789 int i; 790 struct drm_gem_object *obj; 791 792 spin_lock(&filp->table_lock); 793 794 for (i = 0; i < count; i++) { 795 /* Check if we currently have a reference on the object */ 796 obj = idr_find(&filp->object_idr, handle[i]); 797 if (!obj) 798 goto err; 799 800 drm_gem_object_get(obj); 801 objs[i] = obj; 802 } 803 804 spin_unlock(&filp->table_lock); 805 return 0; 806 807 err: 808 spin_unlock(&filp->table_lock); 809 810 while (i--) 811 drm_gem_object_put(objs[i]); 812 813 return -ENOENT; 814 } 815 816 /** 817 * drm_gem_objects_lookup - look up GEM objects from an array of handles 818 * @filp: DRM file private date 819 * @bo_handles: user pointer to array of userspace handle 820 * @count: size of handle array 821 * @objs_out: returned pointer to array of drm_gem_object pointers 822 * 823 * Takes an array of userspace handles and returns a newly allocated array of 824 * GEM objects. 825 * 826 * After a successful lookup GEM objects need to be released using 827 * drm_gem_object_put() and the array returned in @objs_out must be freed using 828 * kvfree(). 829 * 830 * For a single handle lookup, use drm_gem_object_lookup(). 831 * 832 * Return: 833 * Zero on success or a negative error code. 834 */ 835 int drm_gem_objects_lookup(struct drm_file *filp, void __user *bo_handles, 836 int count, struct drm_gem_object ***objs_out) 837 { 838 struct drm_gem_object **objs; 839 u32 *handles; 840 int ret; 841 842 *objs_out = NULL; 843 844 if (!count) 845 return 0; 846 847 objs = kvmalloc_objs(*objs, count); 848 if (!objs) 849 return -ENOMEM; 850 851 handles = vmemdup_array_user(bo_handles, count, sizeof(u32)); 852 if (IS_ERR(handles)) { 853 ret = PTR_ERR(handles); 854 goto err_free_objs; 855 } 856 857 ret = objects_lookup(filp, handles, count, objs); 858 if (ret) 859 goto err_free_handles; 860 861 kvfree(handles); 862 *objs_out = objs; 863 return 0; 864 865 err_free_handles: 866 kvfree(handles); 867 err_free_objs: 868 kvfree(objs); 869 return ret; 870 } 871 EXPORT_SYMBOL(drm_gem_objects_lookup); 872 873 /** 874 * drm_gem_object_lookup - look up a GEM object from its handle 875 * @filp: DRM file private date 876 * @handle: userspace handle 877 * 878 * If looking up an array of handles, use drm_gem_objects_lookup(). 879 * 880 * Returns: 881 * A reference to the object named by the handle if such exists on @filp, NULL 882 * otherwise. 883 */ 884 struct drm_gem_object * 885 drm_gem_object_lookup(struct drm_file *filp, u32 handle) 886 { 887 struct drm_gem_object *obj = NULL; 888 889 objects_lookup(filp, &handle, 1, &obj); 890 return obj; 891 } 892 EXPORT_SYMBOL(drm_gem_object_lookup); 893 894 /** 895 * drm_gem_dma_resv_wait - Wait on GEM object's reservation's objects 896 * shared and/or exclusive fences. 897 * @filep: DRM file private date 898 * @handle: userspace handle 899 * @wait_all: if true, wait on all fences, else wait on just exclusive fence 900 * @timeout: timeout value in jiffies or zero to return immediately 901 * 902 * Returns: 903 * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or 904 * greater than 0 on success. 905 */ 906 long drm_gem_dma_resv_wait(struct drm_file *filep, u32 handle, 907 bool wait_all, unsigned long timeout) 908 { 909 struct drm_device *dev = filep->minor->dev; 910 struct drm_gem_object *obj; 911 long ret; 912 913 obj = drm_gem_object_lookup(filep, handle); 914 if (!obj) { 915 drm_dbg_core(dev, "Failed to look up GEM BO %d\n", handle); 916 return -EINVAL; 917 } 918 919 ret = dma_resv_wait_timeout(obj->resv, dma_resv_usage_rw(wait_all), 920 true, timeout); 921 if (ret == 0) 922 ret = -ETIME; 923 else if (ret > 0) 924 ret = 0; 925 926 drm_gem_object_put(obj); 927 928 return ret; 929 } 930 EXPORT_SYMBOL(drm_gem_dma_resv_wait); 931 932 int 933 drm_gem_close_ioctl(struct drm_device *dev, void *data, 934 struct drm_file *file_priv) 935 { 936 struct drm_gem_close *args = data; 937 int ret; 938 939 if (!drm_core_check_feature(dev, DRIVER_GEM)) 940 return -EOPNOTSUPP; 941 942 ret = drm_gem_handle_delete(file_priv, args->handle); 943 944 return ret; 945 } 946 947 int 948 drm_gem_flink_ioctl(struct drm_device *dev, void *data, 949 struct drm_file *file_priv) 950 { 951 struct drm_gem_flink *args = data; 952 struct drm_gem_object *obj; 953 int ret; 954 955 if (!drm_core_check_feature(dev, DRIVER_GEM)) 956 return -EOPNOTSUPP; 957 958 obj = drm_gem_object_lookup(file_priv, args->handle); 959 if (obj == NULL) 960 return -ENOENT; 961 962 mutex_lock(&dev->object_name_lock); 963 /* prevent races with concurrent gem_close. */ 964 if (obj->handle_count == 0) { 965 ret = -ENOENT; 966 goto err; 967 } 968 969 if (!obj->name) { 970 ret = idr_alloc(&dev->object_name_idr, obj, 1, 0, GFP_KERNEL); 971 if (ret < 0) 972 goto err; 973 974 obj->name = ret; 975 } 976 977 args->name = (uint64_t) obj->name; 978 ret = 0; 979 980 err: 981 mutex_unlock(&dev->object_name_lock); 982 drm_gem_object_put(obj); 983 return ret; 984 } 985 986 int 987 drm_gem_open_ioctl(struct drm_device *dev, void *data, 988 struct drm_file *file_priv) 989 { 990 struct drm_gem_open *args = data; 991 struct drm_gem_object *obj; 992 int ret; 993 u32 handle; 994 995 if (!drm_core_check_feature(dev, DRIVER_GEM)) 996 return -EOPNOTSUPP; 997 998 mutex_lock(&dev->object_name_lock); 999 obj = idr_find(&dev->object_name_idr, (int) args->name); 1000 if (obj) { 1001 drm_gem_object_get(obj); 1002 } else { 1003 mutex_unlock(&dev->object_name_lock); 1004 return -ENOENT; 1005 } 1006 1007 /* drm_gem_handle_create_tail unlocks dev->object_name_lock. */ 1008 ret = drm_gem_handle_create_tail(file_priv, obj, &handle); 1009 if (ret) 1010 goto err; 1011 1012 args->handle = handle; 1013 args->size = obj->size; 1014 1015 err: 1016 drm_gem_object_put(obj); 1017 return ret; 1018 } 1019 1020 /* 1021 * This ioctl is disabled for security reasons but also it failed 1022 * to follow process in terms of adding testing in igt and verifying 1023 * all the corner cases which made fixing security bugs in it even 1024 * harder than necessary. 1025 * 1026 * To re-enable this ioctl 1027 * 1. land working IGT tests in igt-gpu-tools that cover 1028 * all corner cases and race conditions. 1029 * 2. handle idr_preload 1030 * 3. handle == 0 1031 * 4. handle == new_handle semantics definition. 1032 */ 1033 int drm_gem_change_handle_ioctl(struct drm_device *dev, void *data, 1034 struct drm_file *file_priv) 1035 { 1036 struct drm_gem_change_handle *args = data; 1037 struct drm_gem_object *obj; 1038 int new_handle, ret; 1039 1040 if (!drm_core_check_feature(dev, DRIVER_GEM)) 1041 return -EOPNOTSUPP; 1042 1043 /* idr_alloc() limitation. */ 1044 if (args->new_handle > INT_MAX) 1045 return -EINVAL; 1046 new_handle = args->new_handle; 1047 1048 if (args->handle == new_handle) 1049 return 0; 1050 1051 mutex_lock(&file_priv->prime.lock); 1052 spin_lock(&file_priv->table_lock); 1053 ret = idr_alloc(&file_priv->object_idr, NULL, new_handle, new_handle + 1, 1054 GFP_NOWAIT); 1055 1056 if (ret < 0) { 1057 spin_unlock(&file_priv->table_lock); 1058 goto out_unlock; 1059 } 1060 1061 obj = idr_replace(&file_priv->object_idr, NULL, args->handle); 1062 if (IS_ERR_OR_NULL(obj)) { 1063 idr_remove(&file_priv->object_idr, new_handle); 1064 spin_unlock(&file_priv->table_lock); 1065 ret = -ENOENT; 1066 goto out_unlock; 1067 } 1068 spin_unlock(&file_priv->table_lock); 1069 1070 if (obj->dma_buf) { 1071 ret = drm_prime_add_buf_handle(&file_priv->prime, obj->dma_buf, 1072 new_handle); 1073 if (ret < 0) { 1074 spin_lock(&file_priv->table_lock); 1075 idr_remove(&file_priv->object_idr, new_handle); 1076 idr_replace(&file_priv->object_idr, obj, args->handle); 1077 spin_unlock(&file_priv->table_lock); 1078 goto out_unlock; 1079 } 1080 1081 drm_prime_remove_buf_handle(&file_priv->prime, args->handle); 1082 } 1083 1084 ret = 0; 1085 1086 spin_lock(&file_priv->table_lock); 1087 idr_remove(&file_priv->object_idr, args->handle); 1088 obj = idr_replace(&file_priv->object_idr, obj, new_handle); 1089 spin_unlock(&file_priv->table_lock); 1090 WARN_ON(obj != NULL); 1091 1092 out_unlock: 1093 mutex_unlock(&file_priv->prime.lock); 1094 1095 return ret; 1096 } 1097 1098 /** 1099 * drm_gem_open - initializes GEM file-private structures at devnode open time 1100 * @dev: drm_device which is being opened by userspace 1101 * @file_private: drm file-private structure to set up 1102 * 1103 * Called at device open time, sets up the structure for handling refcounting 1104 * of mm objects. 1105 */ 1106 void 1107 drm_gem_open(struct drm_device *dev, struct drm_file *file_private) 1108 { 1109 idr_init_base(&file_private->object_idr, 1); 1110 spin_lock_init(&file_private->table_lock); 1111 } 1112 1113 /** 1114 * drm_gem_release - release file-private GEM resources 1115 * @dev: drm_device which is being closed by userspace 1116 * @file_private: drm file-private structure to clean up 1117 * 1118 * Called at close time when the filp is going away. 1119 * 1120 * Releases any remaining references on objects by this filp. 1121 */ 1122 void 1123 drm_gem_release(struct drm_device *dev, struct drm_file *file_private) 1124 { 1125 idr_for_each(&file_private->object_idr, 1126 &drm_gem_object_release_handle, file_private); 1127 idr_destroy(&file_private->object_idr); 1128 } 1129 1130 /** 1131 * drm_gem_object_release - release GEM buffer object resources 1132 * @obj: GEM buffer object 1133 * 1134 * This releases any structures and resources used by @obj and is the inverse of 1135 * drm_gem_object_init(). 1136 */ 1137 void 1138 drm_gem_object_release(struct drm_gem_object *obj) 1139 { 1140 if (obj->filp) 1141 fput(obj->filp); 1142 1143 drm_gem_private_object_fini(obj); 1144 1145 drm_gem_free_mmap_offset(obj); 1146 drm_gem_lru_remove(obj); 1147 } 1148 EXPORT_SYMBOL(drm_gem_object_release); 1149 1150 /** 1151 * drm_gem_object_free - free a GEM object 1152 * @kref: kref of the object to free 1153 * 1154 * Called after the last reference to the object has been lost. 1155 * 1156 * Frees the object 1157 */ 1158 void 1159 drm_gem_object_free(struct kref *kref) 1160 { 1161 struct drm_gem_object *obj = 1162 container_of(kref, struct drm_gem_object, refcount); 1163 1164 if (WARN_ON(!obj->funcs->free)) 1165 return; 1166 1167 obj->funcs->free(obj); 1168 } 1169 EXPORT_SYMBOL(drm_gem_object_free); 1170 1171 /** 1172 * drm_gem_vm_open - vma->ops->open implementation for GEM 1173 * @vma: VM area structure 1174 * 1175 * This function implements the #vm_operations_struct open() callback for GEM 1176 * drivers. This must be used together with drm_gem_vm_close(). 1177 */ 1178 void drm_gem_vm_open(struct vm_area_struct *vma) 1179 { 1180 struct drm_gem_object *obj = vma->vm_private_data; 1181 1182 drm_gem_object_get(obj); 1183 } 1184 EXPORT_SYMBOL(drm_gem_vm_open); 1185 1186 /** 1187 * drm_gem_vm_close - vma->ops->close implementation for GEM 1188 * @vma: VM area structure 1189 * 1190 * This function implements the #vm_operations_struct close() callback for GEM 1191 * drivers. This must be used together with drm_gem_vm_open(). 1192 */ 1193 void drm_gem_vm_close(struct vm_area_struct *vma) 1194 { 1195 struct drm_gem_object *obj = vma->vm_private_data; 1196 1197 drm_gem_object_put(obj); 1198 } 1199 EXPORT_SYMBOL(drm_gem_vm_close); 1200 1201 /** 1202 * drm_gem_mmap_obj - memory map a GEM object 1203 * @obj: the GEM object to map 1204 * @obj_size: the object size to be mapped, in bytes 1205 * @vma: VMA for the area to be mapped 1206 * 1207 * Set up the VMA to prepare mapping of the GEM object using the GEM object's 1208 * vm_ops. Depending on their requirements, GEM objects can either 1209 * provide a fault handler in their vm_ops (in which case any accesses to 1210 * the object will be trapped, to perform migration, GTT binding, surface 1211 * register allocation, or performance monitoring), or mmap the buffer memory 1212 * synchronously after calling drm_gem_mmap_obj. 1213 * 1214 * This function is mainly intended to implement the DMABUF mmap operation, when 1215 * the GEM object is not looked up based on its fake offset. To implement the 1216 * DRM mmap operation, drivers should use the drm_gem_mmap() function. 1217 * 1218 * drm_gem_mmap_obj() assumes the user is granted access to the buffer while 1219 * drm_gem_mmap() prevents unprivileged users from mapping random objects. So 1220 * callers must verify access restrictions before calling this helper. 1221 * 1222 * Return 0 or success or -EINVAL if the object size is smaller than the VMA 1223 * size, or if no vm_ops are provided. 1224 */ 1225 int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size, 1226 struct vm_area_struct *vma) 1227 { 1228 int ret; 1229 1230 /* Check for valid size. */ 1231 if (obj_size < vma->vm_end - vma->vm_start) 1232 return -EINVAL; 1233 1234 /* Take a ref for this mapping of the object, so that the fault 1235 * handler can dereference the mmap offset's pointer to the object. 1236 * This reference is cleaned up by the corresponding vm_close 1237 * (which should happen whether the vma was created by this call, or 1238 * by a vm_open due to mremap or partial unmap or whatever). 1239 */ 1240 drm_gem_object_get(obj); 1241 1242 vma->vm_private_data = obj; 1243 vma->vm_ops = obj->funcs->vm_ops; 1244 1245 if (obj->funcs->mmap) { 1246 ret = obj->funcs->mmap(obj, vma); 1247 if (ret) 1248 goto err_drm_gem_object_put; 1249 WARN_ON(!(vma->vm_flags & VM_DONTEXPAND)); 1250 } else { 1251 if (!vma->vm_ops) { 1252 ret = -EINVAL; 1253 goto err_drm_gem_object_put; 1254 } 1255 1256 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); 1257 vma->vm_page_prot = pgprot_writecombine(vma_get_page_prot(vma)); 1258 vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); 1259 } 1260 1261 return 0; 1262 1263 err_drm_gem_object_put: 1264 drm_gem_object_put(obj); 1265 return ret; 1266 } 1267 EXPORT_SYMBOL(drm_gem_mmap_obj); 1268 1269 /* 1270 * Look up a GEM object in offset space based on the exact start address. The 1271 * caller must be granted access to the object. Returns a GEM object on success 1272 * or a negative error code on failure. The returned GEM object needs to be 1273 * released with drm_gem_object_put(). 1274 */ 1275 static struct drm_gem_object * 1276 drm_gem_object_lookup_at_offset(struct file *filp, unsigned long start, 1277 unsigned long pages) 1278 { 1279 struct drm_file *priv = filp->private_data; 1280 struct drm_device *dev = priv->minor->dev; 1281 struct drm_gem_object *obj = NULL; 1282 struct drm_vma_offset_node *node; 1283 1284 if (drm_dev_is_unplugged(dev)) 1285 return ERR_PTR(-ENODEV); 1286 1287 drm_vma_offset_lock_lookup(dev->vma_offset_manager); 1288 node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager, 1289 start, pages); 1290 if (likely(node)) { 1291 obj = container_of(node, struct drm_gem_object, vma_node); 1292 /* 1293 * When the object is being freed, after it hits 0-refcnt it 1294 * proceeds to tear down the object. In the process it will 1295 * attempt to remove the VMA offset and so acquire this 1296 * mgr->vm_lock. Therefore if we find an object with a 0-refcnt 1297 * that matches our range, we know it is in the process of being 1298 * destroyed and will be freed as soon as we release the lock - 1299 * so we have to check for the 0-refcnted object and treat it as 1300 * invalid. 1301 */ 1302 if (!kref_get_unless_zero(&obj->refcount)) 1303 obj = NULL; 1304 } 1305 drm_vma_offset_unlock_lookup(dev->vma_offset_manager); 1306 1307 if (!obj) 1308 return ERR_PTR(-EINVAL); 1309 1310 if (!drm_vma_node_is_allowed(node, priv)) { 1311 drm_gem_object_put(obj); 1312 return ERR_PTR(-EACCES); 1313 } 1314 1315 return obj; 1316 } 1317 1318 #ifdef CONFIG_MMU 1319 /** 1320 * drm_gem_get_unmapped_area - get memory mapping region routine for GEM objects 1321 * @filp: DRM file pointer 1322 * @uaddr: User address hint 1323 * @len: Mapping length 1324 * @pgoff: Offset (in pages) 1325 * @flags: Mapping flags 1326 * 1327 * If a driver supports GEM object mapping, before ending up in drm_gem_mmap(), 1328 * mmap calls on the DRM file descriptor will first try to find a free linear 1329 * address space large enough for a mapping. Since GEM objects are backed by 1330 * shmem buffers, this should preferably be handled by the shmem virtual memory 1331 * filesystem which can appropriately align addresses to huge page sizes when 1332 * needed. 1333 * 1334 * Look up the GEM object based on the offset passed in (vma->vm_pgoff will 1335 * contain the fake offset we created) and call shmem_get_unmapped_area() with 1336 * the right file pointer. 1337 * 1338 * If a GEM object is not available at the given offset or if the caller is not 1339 * granted access to it, fall back to mm_get_unmapped_area(). 1340 */ 1341 unsigned long drm_gem_get_unmapped_area(struct file *filp, unsigned long uaddr, 1342 unsigned long len, unsigned long pgoff, 1343 unsigned long flags) 1344 { 1345 struct drm_gem_object *obj; 1346 unsigned long ret; 1347 1348 obj = drm_gem_object_lookup_at_offset(filp, pgoff, len >> PAGE_SHIFT); 1349 if (IS_ERR(obj)) 1350 obj = NULL; 1351 1352 if (!obj || !obj->filp || !obj->filp->f_op->get_unmapped_area) 1353 ret = mm_get_unmapped_area(filp, uaddr, len, 0, flags); 1354 else 1355 ret = obj->filp->f_op->get_unmapped_area(obj->filp, uaddr, len, 0, flags); 1356 1357 drm_gem_object_put(obj); 1358 1359 return ret; 1360 } 1361 EXPORT_SYMBOL_GPL(drm_gem_get_unmapped_area); 1362 #endif 1363 1364 /** 1365 * drm_gem_mmap - memory map routine for GEM objects 1366 * @filp: DRM file pointer 1367 * @vma: VMA for the area to be mapped 1368 * 1369 * If a driver supports GEM object mapping, mmap calls on the DRM file 1370 * descriptor will end up here. 1371 * 1372 * Look up the GEM object based on the offset passed in (vma->vm_pgoff will 1373 * contain the fake offset we created) and map it with a call to 1374 * drm_gem_mmap_obj(). 1375 * 1376 * If the caller is not granted access to the buffer object, the mmap will fail 1377 * with EACCES. Please see the vma manager for more information. 1378 */ 1379 int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma) 1380 { 1381 struct drm_gem_object *obj; 1382 int ret; 1383 1384 obj = drm_gem_object_lookup_at_offset(filp, vma->vm_pgoff, 1385 vma_pages(vma)); 1386 if (IS_ERR(obj)) 1387 return PTR_ERR(obj); 1388 1389 ret = drm_gem_mmap_obj(obj, 1390 drm_vma_node_size(&obj->vma_node) << PAGE_SHIFT, 1391 vma); 1392 1393 drm_gem_object_put(obj); 1394 1395 return ret; 1396 } 1397 EXPORT_SYMBOL(drm_gem_mmap); 1398 1399 void drm_gem_print_info(struct drm_printer *p, unsigned int indent, 1400 const struct drm_gem_object *obj) 1401 { 1402 drm_printf_indent(p, indent, "name=%d\n", obj->name); 1403 drm_printf_indent(p, indent, "refcount=%u\n", 1404 kref_read(&obj->refcount)); 1405 drm_printf_indent(p, indent, "start=%08lx\n", 1406 drm_vma_node_start(&obj->vma_node)); 1407 drm_printf_indent(p, indent, "size=%zu\n", obj->size); 1408 drm_printf_indent(p, indent, "imported=%s\n", 1409 str_yes_no(drm_gem_is_imported(obj))); 1410 1411 if (obj->funcs->print_info) 1412 obj->funcs->print_info(p, indent, obj); 1413 } 1414 1415 int drm_gem_vmap_locked(struct drm_gem_object *obj, struct iosys_map *map) 1416 { 1417 int ret; 1418 1419 dma_resv_assert_held(obj->resv); 1420 1421 if (!obj->funcs->vmap) 1422 return -EOPNOTSUPP; 1423 1424 ret = obj->funcs->vmap(obj, map); 1425 if (ret) 1426 return ret; 1427 else if (iosys_map_is_null(map)) 1428 return -ENOMEM; 1429 1430 return 0; 1431 } 1432 EXPORT_SYMBOL(drm_gem_vmap_locked); 1433 1434 void drm_gem_vunmap_locked(struct drm_gem_object *obj, struct iosys_map *map) 1435 { 1436 dma_resv_assert_held(obj->resv); 1437 1438 if (iosys_map_is_null(map)) 1439 return; 1440 1441 if (obj->funcs->vunmap) 1442 obj->funcs->vunmap(obj, map); 1443 1444 /* Always set the mapping to NULL. Callers may rely on this. */ 1445 iosys_map_clear(map); 1446 } 1447 EXPORT_SYMBOL(drm_gem_vunmap_locked); 1448 1449 void drm_gem_lock(struct drm_gem_object *obj) 1450 { 1451 dma_resv_lock(obj->resv, NULL); 1452 } 1453 EXPORT_SYMBOL(drm_gem_lock); 1454 1455 void drm_gem_unlock(struct drm_gem_object *obj) 1456 { 1457 dma_resv_unlock(obj->resv); 1458 } 1459 EXPORT_SYMBOL(drm_gem_unlock); 1460 1461 int drm_gem_vmap(struct drm_gem_object *obj, struct iosys_map *map) 1462 { 1463 int ret; 1464 1465 dma_resv_lock(obj->resv, NULL); 1466 ret = drm_gem_vmap_locked(obj, map); 1467 dma_resv_unlock(obj->resv); 1468 1469 return ret; 1470 } 1471 EXPORT_SYMBOL(drm_gem_vmap); 1472 1473 void drm_gem_vunmap(struct drm_gem_object *obj, struct iosys_map *map) 1474 { 1475 dma_resv_lock(obj->resv, NULL); 1476 drm_gem_vunmap_locked(obj, map); 1477 dma_resv_unlock(obj->resv); 1478 } 1479 EXPORT_SYMBOL(drm_gem_vunmap); 1480 1481 /** 1482 * drm_gem_lock_reservations - Sets up the ww context and acquires 1483 * the lock on an array of GEM objects. 1484 * 1485 * Once you've locked your reservations, you'll want to set up space 1486 * for your shared fences (if applicable), submit your job, then 1487 * drm_gem_unlock_reservations(). 1488 * 1489 * @objs: drm_gem_objects to lock 1490 * @count: Number of objects in @objs 1491 * @acquire_ctx: struct ww_acquire_ctx that will be initialized as 1492 * part of tracking this set of locked reservations. 1493 */ 1494 int 1495 drm_gem_lock_reservations(struct drm_gem_object **objs, int count, 1496 struct ww_acquire_ctx *acquire_ctx) 1497 { 1498 int contended = -1; 1499 int i, ret; 1500 1501 ww_acquire_init(acquire_ctx, &reservation_ww_class); 1502 1503 retry: 1504 if (contended != -1) { 1505 struct drm_gem_object *obj = objs[contended]; 1506 1507 ret = dma_resv_lock_slow_interruptible(obj->resv, 1508 acquire_ctx); 1509 if (ret) { 1510 ww_acquire_fini(acquire_ctx); 1511 return ret; 1512 } 1513 } 1514 1515 for (i = 0; i < count; i++) { 1516 if (i == contended) 1517 continue; 1518 1519 ret = dma_resv_lock_interruptible(objs[i]->resv, 1520 acquire_ctx); 1521 if (ret) { 1522 int j; 1523 1524 for (j = 0; j < i; j++) 1525 dma_resv_unlock(objs[j]->resv); 1526 1527 if (contended != -1 && contended >= i) 1528 dma_resv_unlock(objs[contended]->resv); 1529 1530 if (ret == -EDEADLK) { 1531 contended = i; 1532 goto retry; 1533 } 1534 1535 ww_acquire_fini(acquire_ctx); 1536 return ret; 1537 } 1538 } 1539 1540 ww_acquire_done(acquire_ctx); 1541 1542 return 0; 1543 } 1544 EXPORT_SYMBOL(drm_gem_lock_reservations); 1545 1546 void 1547 drm_gem_unlock_reservations(struct drm_gem_object **objs, int count, 1548 struct ww_acquire_ctx *acquire_ctx) 1549 { 1550 int i; 1551 1552 for (i = 0; i < count; i++) 1553 dma_resv_unlock(objs[i]->resv); 1554 1555 ww_acquire_fini(acquire_ctx); 1556 } 1557 EXPORT_SYMBOL(drm_gem_unlock_reservations); 1558 1559 /** 1560 * drm_gem_lru_init - initialize a LRU 1561 * 1562 * @lru: The LRU to initialize 1563 */ 1564 void 1565 drm_gem_lru_init(struct drm_gem_lru *lru) 1566 { 1567 lru->count = 0; 1568 INIT_LIST_HEAD(&lru->list); 1569 } 1570 EXPORT_SYMBOL(drm_gem_lru_init); 1571 1572 static void 1573 drm_gem_lru_remove_locked(struct drm_gem_object *obj) 1574 { 1575 obj->lru->count -= obj->size >> PAGE_SHIFT; 1576 WARN_ON(obj->lru->count < 0); 1577 list_del(&obj->lru_node); 1578 obj->lru = NULL; 1579 } 1580 1581 /** 1582 * drm_gem_lru_remove - remove object from whatever LRU it is in 1583 * 1584 * If the object is currently in any LRU, remove it. 1585 * 1586 * @obj: The GEM object to remove from current LRU 1587 */ 1588 void 1589 drm_gem_lru_remove(struct drm_gem_object *obj) 1590 { 1591 mutex_lock(&obj->dev->gem_lru_mutex); 1592 if (obj->lru) 1593 drm_gem_lru_remove_locked(obj); 1594 mutex_unlock(&obj->dev->gem_lru_mutex); 1595 } 1596 EXPORT_SYMBOL(drm_gem_lru_remove); 1597 1598 /** 1599 * drm_gem_lru_move_tail_locked - move the object to the tail of the LRU 1600 * 1601 * Like &drm_gem_lru_move_tail but lru lock must be held 1602 * 1603 * @lru: The LRU to move the object into. 1604 * @obj: The GEM object to move into this LRU 1605 */ 1606 void 1607 drm_gem_lru_move_tail_locked(struct drm_gem_lru *lru, struct drm_gem_object *obj) 1608 { 1609 lockdep_assert_held_once(&obj->dev->gem_lru_mutex); 1610 1611 if (obj->lru) 1612 drm_gem_lru_remove_locked(obj); 1613 1614 lru->count += obj->size >> PAGE_SHIFT; 1615 list_add_tail(&obj->lru_node, &lru->list); 1616 obj->lru = lru; 1617 } 1618 EXPORT_SYMBOL(drm_gem_lru_move_tail_locked); 1619 1620 /** 1621 * drm_gem_lru_move_tail - move the object to the tail of the LRU 1622 * 1623 * If the object is already in this LRU it will be moved to the 1624 * tail. Otherwise it will be removed from whichever other LRU 1625 * it is in (if any) and moved into this LRU. 1626 * 1627 * @lru: The LRU to move the object into. 1628 * @obj: The GEM object to move into this LRU 1629 */ 1630 void 1631 drm_gem_lru_move_tail(struct drm_gem_lru *lru, struct drm_gem_object *obj) 1632 { 1633 mutex_lock(&obj->dev->gem_lru_mutex); 1634 drm_gem_lru_move_tail_locked(lru, obj); 1635 mutex_unlock(&obj->dev->gem_lru_mutex); 1636 } 1637 EXPORT_SYMBOL(drm_gem_lru_move_tail); 1638 1639 /** 1640 * drm_gem_lru_scan - helper to implement shrinker.scan_objects 1641 * 1642 * If the shrink callback succeeds, it is expected that the driver 1643 * move the object out of this LRU. 1644 * 1645 * If the LRU possibly contain active buffers, it is the responsibility 1646 * of the shrink callback to check for this (ie. dma_resv_test_signaled()) 1647 * or if necessary block until the buffer becomes idle. 1648 * 1649 * @dev: DRM device the LRU belongs to 1650 * @lru: The LRU to scan 1651 * @nr_to_scan: The number of pages to try to reclaim 1652 * @remaining: The number of pages left to reclaim, should be initialized by caller 1653 * @shrink: Callback to try to shrink/reclaim the object. 1654 */ 1655 unsigned long 1656 drm_gem_lru_scan(struct drm_device *dev, 1657 struct drm_gem_lru *lru, 1658 unsigned int nr_to_scan, 1659 unsigned long *remaining, 1660 bool (*shrink)(struct drm_gem_object *obj)) 1661 { 1662 struct drm_gem_lru still_in_lru; 1663 struct drm_gem_object *obj; 1664 unsigned freed = 0; 1665 1666 drm_gem_lru_init(&still_in_lru); 1667 1668 mutex_lock(&dev->gem_lru_mutex); 1669 1670 while (freed < nr_to_scan) { 1671 obj = list_first_entry_or_null(&lru->list, typeof(*obj), lru_node); 1672 1673 if (!obj) 1674 break; 1675 1676 drm_gem_lru_move_tail_locked(&still_in_lru, obj); 1677 1678 /* 1679 * If it's in the process of being freed, gem_object->free() 1680 * may be blocked on lock waiting to remove it. So just 1681 * skip it. 1682 */ 1683 if (!kref_get_unless_zero(&obj->refcount)) 1684 continue; 1685 1686 /* 1687 * Now that we own a reference, we can drop the lock for the 1688 * rest of the loop body, to reduce contention with other 1689 * code paths that need the LRU lock 1690 */ 1691 mutex_unlock(&dev->gem_lru_mutex); 1692 1693 /* 1694 * Note that this still needs to be trylock, since we can 1695 * hit shrinker in response to trying to get backing pages 1696 * for this obj (ie. while it's lock is already held) 1697 */ 1698 if (!ww_mutex_trylock(&obj->resv->lock, NULL)) { 1699 *remaining += obj->size >> PAGE_SHIFT; 1700 goto tail; 1701 } 1702 1703 if (shrink(obj)) { 1704 freed += obj->size >> PAGE_SHIFT; 1705 1706 /* 1707 * If we succeeded in releasing the object's backing 1708 * pages, we expect the driver to have moved the object 1709 * out of this LRU 1710 */ 1711 WARN_ON(obj->lru == &still_in_lru); 1712 WARN_ON(obj->lru == lru); 1713 } else if (obj->lru == &still_in_lru) { 1714 /* 1715 * If the object wasn't moved and wasn't shrunk either, 1716 * it's still remaining as reclaimable. Note that 1717 * obj->lru is supposed to be checked with the LRU lock 1718 * held for an accurate result, but we don't care about 1719 * accuracy here. Worst thing that could happen is an 1720 * extra scan. 1721 */ 1722 *remaining += obj->size >> PAGE_SHIFT; 1723 } 1724 1725 dma_resv_unlock(obj->resv); 1726 1727 tail: 1728 drm_gem_object_put(obj); 1729 mutex_lock(&dev->gem_lru_mutex); 1730 } 1731 1732 /* 1733 * Move objects we've skipped over out of the temporary still_in_lru 1734 * back into this LRU 1735 */ 1736 list_for_each_entry (obj, &still_in_lru.list, lru_node) 1737 obj->lru = lru; 1738 list_splice_tail(&still_in_lru.list, &lru->list); 1739 lru->count += still_in_lru.count; 1740 1741 mutex_unlock(&dev->gem_lru_mutex); 1742 1743 return freed; 1744 } 1745 EXPORT_SYMBOL(drm_gem_lru_scan); 1746 1747 /** 1748 * drm_gem_evict_locked - helper to evict backing pages for a GEM object 1749 * @obj: obj in question 1750 */ 1751 int drm_gem_evict_locked(struct drm_gem_object *obj) 1752 { 1753 dma_resv_assert_held(obj->resv); 1754 1755 if (!dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ)) 1756 return -EBUSY; 1757 1758 if (obj->funcs->evict) 1759 return obj->funcs->evict(obj); 1760 1761 return 0; 1762 } 1763 EXPORT_SYMBOL(drm_gem_evict_locked); 1764