1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2016 Intel Corporation 5 */ 6 7 #ifndef __I915_GEM_OBJECT_H__ 8 #define __I915_GEM_OBJECT_H__ 9 10 #include <drm/drm_gem.h> 11 #include <drm/drm_file.h> 12 #include <drm/drm_device.h> 13 14 #include "intel_memory_region.h" 15 #include "i915_gem_object_types.h" 16 #include "i915_gem_gtt.h" 17 #include "i915_gem_ww.h" 18 #include "i915_vma_types.h" 19 20 enum intel_region_id; 21 22 #define obj_to_i915(obj__) to_i915((obj__)->base.dev) 23 24 static inline bool i915_gem_object_size_2big(u64 size) 25 { 26 struct drm_i915_gem_object *obj; 27 28 if (overflows_type(size, obj->base.size)) 29 return true; 30 31 return false; 32 } 33 34 unsigned int i915_gem_get_pat_index(struct drm_i915_private *i915, 35 enum i915_cache_level level); 36 bool i915_gem_object_has_cache_level(const struct drm_i915_gem_object *obj, 37 enum i915_cache_level lvl); 38 void i915_gem_init__objects(struct drm_i915_private *i915); 39 40 void i915_objects_module_exit(void); 41 int i915_objects_module_init(void); 42 43 struct drm_i915_gem_object *i915_gem_object_alloc(void); 44 void i915_gem_object_free(struct drm_i915_gem_object *obj); 45 46 void i915_gem_object_init(struct drm_i915_gem_object *obj, 47 const struct drm_i915_gem_object_ops *ops, 48 struct lock_class_key *key, 49 unsigned alloc_flags); 50 51 void __i915_gem_object_fini(struct drm_i915_gem_object *obj); 52 53 struct drm_i915_gem_object * 54 i915_gem_object_create_shmem(struct drm_i915_private *i915, 55 resource_size_t size); 56 struct drm_i915_gem_object * 57 i915_gem_object_create_shmem_from_data(struct drm_i915_private *i915, 58 const void *data, resource_size_t size); 59 struct drm_i915_gem_object * 60 __i915_gem_object_create_user(struct drm_i915_private *i915, u64 size, 61 struct intel_memory_region **placements, 62 unsigned int n_placements); 63 64 extern const struct drm_i915_gem_object_ops i915_gem_shmem_ops; 65 66 void __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, 67 struct sg_table *pages, 68 bool needs_clflush); 69 70 int i915_gem_object_pwrite_phys(struct drm_i915_gem_object *obj, 71 const struct drm_i915_gem_pwrite *args); 72 int i915_gem_object_pread_phys(struct drm_i915_gem_object *obj, 73 const struct drm_i915_gem_pread *args); 74 75 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align); 76 void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, 77 struct sg_table *pages); 78 void i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, 79 struct sg_table *pages); 80 81 void i915_gem_flush_free_objects(struct drm_i915_private *i915); 82 83 struct sg_table * 84 __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj); 85 86 /** 87 * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle 88 * @file: DRM file private date 89 * @handle: userspace handle 90 * 91 * Returns: 92 * A pointer to the object named by the handle if such exists on @filp, NULL 93 * otherwise. This object is only valid whilst under the RCU read lock, and 94 * note carefully the object may be in the process of being destroyed. 95 */ 96 static inline struct drm_i915_gem_object * 97 i915_gem_object_lookup_rcu(struct drm_file *file, u32 handle) 98 { 99 #ifdef CONFIG_LOCKDEP 100 WARN_ON(debug_locks && !lock_is_held(&rcu_lock_map)); 101 #endif 102 return idr_find(&file->object_idr, handle); 103 } 104 105 static inline struct drm_i915_gem_object * 106 i915_gem_object_get_rcu(struct drm_i915_gem_object *obj) 107 { 108 if (obj && !kref_get_unless_zero(&obj->base.refcount)) 109 obj = NULL; 110 111 return obj; 112 } 113 114 static inline struct drm_i915_gem_object * 115 i915_gem_object_lookup(struct drm_file *file, u32 handle) 116 { 117 struct drm_i915_gem_object *obj; 118 119 rcu_read_lock(); 120 obj = i915_gem_object_lookup_rcu(file, handle); 121 obj = i915_gem_object_get_rcu(obj); 122 rcu_read_unlock(); 123 124 return obj; 125 } 126 127 __deprecated 128 struct drm_gem_object * 129 drm_gem_object_lookup(struct drm_file *file, u32 handle); 130 131 __attribute__((nonnull)) 132 static inline struct drm_i915_gem_object * 133 i915_gem_object_get(struct drm_i915_gem_object *obj) 134 { 135 drm_gem_object_get(&obj->base); 136 return obj; 137 } 138 139 __attribute__((nonnull)) 140 static inline void 141 i915_gem_object_put(struct drm_i915_gem_object *obj) 142 { 143 __drm_gem_object_put(&obj->base); 144 } 145 146 #define assert_object_held(obj) dma_resv_assert_held((obj)->base.resv) 147 148 /* 149 * If more than one potential simultaneous locker, assert held. 150 */ 151 static inline void assert_object_held_shared(const struct drm_i915_gem_object *obj) 152 { 153 /* 154 * Note mm list lookup is protected by 155 * kref_get_unless_zero(). 156 */ 157 if (IS_ENABLED(CONFIG_LOCKDEP) && 158 kref_read(&obj->base.refcount) > 0) 159 assert_object_held(obj); 160 } 161 162 static inline int __i915_gem_object_lock(struct drm_i915_gem_object *obj, 163 struct i915_gem_ww_ctx *ww, 164 bool intr) 165 { 166 int ret; 167 168 if (intr) 169 ret = dma_resv_lock_interruptible(obj->base.resv, ww ? &ww->ctx : NULL); 170 else 171 ret = dma_resv_lock(obj->base.resv, ww ? &ww->ctx : NULL); 172 173 if (!ret && ww) { 174 i915_gem_object_get(obj); 175 list_add_tail(&obj->obj_link, &ww->obj_list); 176 } 177 if (ret == -EALREADY) 178 ret = 0; 179 180 if (ret == -EDEADLK) { 181 i915_gem_object_get(obj); 182 ww->contended = obj; 183 } 184 185 return ret; 186 } 187 188 static inline int i915_gem_object_lock(struct drm_i915_gem_object *obj, 189 struct i915_gem_ww_ctx *ww) 190 { 191 return __i915_gem_object_lock(obj, ww, ww && ww->intr); 192 } 193 194 static inline int i915_gem_object_lock_interruptible(struct drm_i915_gem_object *obj, 195 struct i915_gem_ww_ctx *ww) 196 { 197 WARN_ON(ww && !ww->intr); 198 return __i915_gem_object_lock(obj, ww, true); 199 } 200 201 static inline bool i915_gem_object_trylock(struct drm_i915_gem_object *obj, 202 struct i915_gem_ww_ctx *ww) 203 { 204 if (!ww) 205 return dma_resv_trylock(obj->base.resv); 206 else 207 return ww_mutex_trylock(&obj->base.resv->lock, &ww->ctx); 208 } 209 210 static inline void i915_gem_object_unlock(struct drm_i915_gem_object *obj) 211 { 212 if (obj->ops->adjust_lru) 213 obj->ops->adjust_lru(obj); 214 215 dma_resv_unlock(obj->base.resv); 216 } 217 218 static inline void 219 i915_gem_object_set_readonly(struct drm_i915_gem_object *obj) 220 { 221 obj->flags |= I915_BO_READONLY; 222 } 223 224 static inline bool 225 i915_gem_object_is_readonly(const struct drm_i915_gem_object *obj) 226 { 227 return obj->flags & I915_BO_READONLY; 228 } 229 230 static inline bool 231 i915_gem_object_is_contiguous(const struct drm_i915_gem_object *obj) 232 { 233 return obj->flags & I915_BO_ALLOC_CONTIGUOUS; 234 } 235 236 static inline bool 237 i915_gem_object_is_volatile(const struct drm_i915_gem_object *obj) 238 { 239 return obj->flags & I915_BO_ALLOC_VOLATILE; 240 } 241 242 static inline void 243 i915_gem_object_set_volatile(struct drm_i915_gem_object *obj) 244 { 245 obj->flags |= I915_BO_ALLOC_VOLATILE; 246 } 247 248 static inline bool 249 i915_gem_object_has_tiling_quirk(struct drm_i915_gem_object *obj) 250 { 251 return test_bit(I915_TILING_QUIRK_BIT, &obj->flags); 252 } 253 254 static inline void 255 i915_gem_object_set_tiling_quirk(struct drm_i915_gem_object *obj) 256 { 257 set_bit(I915_TILING_QUIRK_BIT, &obj->flags); 258 } 259 260 static inline void 261 i915_gem_object_clear_tiling_quirk(struct drm_i915_gem_object *obj) 262 { 263 clear_bit(I915_TILING_QUIRK_BIT, &obj->flags); 264 } 265 266 static inline bool 267 i915_gem_object_is_protected(const struct drm_i915_gem_object *obj) 268 { 269 return obj->flags & I915_BO_PROTECTED; 270 } 271 272 static inline bool 273 i915_gem_object_type_has(const struct drm_i915_gem_object *obj, 274 unsigned long flags) 275 { 276 return obj->ops->flags & flags; 277 } 278 279 bool i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj); 280 281 bool i915_gem_object_has_iomem(const struct drm_i915_gem_object *obj); 282 283 static inline bool 284 i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj) 285 { 286 /* TODO: make DPT shrinkable when it has no bound vmas */ 287 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_IS_SHRINKABLE) && 288 !obj->is_dpt; 289 } 290 291 static inline bool 292 i915_gem_object_has_self_managed_shrink_list(const struct drm_i915_gem_object *obj) 293 { 294 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_SELF_MANAGED_SHRINK_LIST); 295 } 296 297 static inline bool 298 i915_gem_object_is_proxy(const struct drm_i915_gem_object *obj) 299 { 300 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_IS_PROXY); 301 } 302 303 static inline bool 304 i915_gem_object_never_mmap(const struct drm_i915_gem_object *obj) 305 { 306 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_NO_MMAP); 307 } 308 309 static inline bool 310 i915_gem_object_is_framebuffer(const struct drm_i915_gem_object *obj) 311 { 312 return READ_ONCE(obj->frontbuffer) || obj->is_dpt; 313 } 314 315 static inline unsigned int 316 i915_gem_object_get_tiling(const struct drm_i915_gem_object *obj) 317 { 318 return obj->tiling_and_stride & TILING_MASK; 319 } 320 321 static inline bool 322 i915_gem_object_is_tiled(const struct drm_i915_gem_object *obj) 323 { 324 return i915_gem_object_get_tiling(obj) != I915_TILING_NONE; 325 } 326 327 static inline unsigned int 328 i915_gem_object_get_stride(const struct drm_i915_gem_object *obj) 329 { 330 return obj->tiling_and_stride & STRIDE_MASK; 331 } 332 333 static inline unsigned int 334 i915_gem_tile_height(unsigned int tiling) 335 { 336 GEM_BUG_ON(!tiling); 337 return tiling == I915_TILING_Y ? 32 : 8; 338 } 339 340 static inline unsigned int 341 i915_gem_object_get_tile_height(const struct drm_i915_gem_object *obj) 342 { 343 return i915_gem_tile_height(i915_gem_object_get_tiling(obj)); 344 } 345 346 static inline unsigned int 347 i915_gem_object_get_tile_row_size(const struct drm_i915_gem_object *obj) 348 { 349 return (i915_gem_object_get_stride(obj) * 350 i915_gem_object_get_tile_height(obj)); 351 } 352 353 int i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, 354 unsigned int tiling, unsigned int stride); 355 356 /** 357 * __i915_gem_object_page_iter_get_sg - helper to find the target scatterlist 358 * pointer and the target page position using pgoff_t n input argument and 359 * i915_gem_object_page_iter 360 * @obj: i915 GEM buffer object 361 * @iter: i915 GEM buffer object page iterator 362 * @n: page offset 363 * @offset: searched physical offset, 364 * it will be used for returning physical page offset value 365 * 366 * Context: Takes and releases the mutex lock of the i915_gem_object_page_iter. 367 * Takes and releases the RCU lock to search the radix_tree of 368 * i915_gem_object_page_iter. 369 * 370 * Returns: 371 * The target scatterlist pointer and the target page position. 372 * 373 * Recommended to use wrapper macro: i915_gem_object_page_iter_get_sg() 374 */ 375 struct scatterlist * 376 __i915_gem_object_page_iter_get_sg(struct drm_i915_gem_object *obj, 377 struct i915_gem_object_page_iter *iter, 378 pgoff_t n, 379 unsigned int *offset); 380 381 /** 382 * i915_gem_object_page_iter_get_sg - wrapper macro for 383 * __i915_gem_object_page_iter_get_sg() 384 * @obj: i915 GEM buffer object 385 * @it: i915 GEM buffer object page iterator 386 * @n: page offset 387 * @offset: searched physical offset, 388 * it will be used for returning physical page offset value 389 * 390 * Context: Takes and releases the mutex lock of the i915_gem_object_page_iter. 391 * Takes and releases the RCU lock to search the radix_tree of 392 * i915_gem_object_page_iter. 393 * 394 * Returns: 395 * The target scatterlist pointer and the target page position. 396 * 397 * In order to avoid the truncation of the input parameter, it checks the page 398 * offset n's type from the input parameter before calling 399 * __i915_gem_object_page_iter_get_sg(). 400 */ 401 #define i915_gem_object_page_iter_get_sg(obj, it, n, offset) ({ \ 402 static_assert(castable_to_type(n, pgoff_t)); \ 403 __i915_gem_object_page_iter_get_sg(obj, it, n, offset); \ 404 }) 405 406 /** 407 * __i915_gem_object_get_sg - helper to find the target scatterlist 408 * pointer and the target page position using pgoff_t n input argument and 409 * drm_i915_gem_object. It uses an internal shmem scatterlist lookup function. 410 * @obj: i915 GEM buffer object 411 * @n: page offset 412 * @offset: searched physical offset, 413 * it will be used for returning physical page offset value 414 * 415 * It uses drm_i915_gem_object's internal shmem scatterlist lookup function as 416 * i915_gem_object_page_iter and calls __i915_gem_object_page_iter_get_sg(). 417 * 418 * Returns: 419 * The target scatterlist pointer and the target page position. 420 * 421 * Recommended to use wrapper macro: i915_gem_object_get_sg() 422 * See also __i915_gem_object_page_iter_get_sg() 423 */ 424 static inline struct scatterlist * 425 __i915_gem_object_get_sg(struct drm_i915_gem_object *obj, pgoff_t n, 426 unsigned int *offset) 427 { 428 return __i915_gem_object_page_iter_get_sg(obj, &obj->mm.get_page, n, offset); 429 } 430 431 /** 432 * i915_gem_object_get_sg - wrapper macro for __i915_gem_object_get_sg() 433 * @obj: i915 GEM buffer object 434 * @n: page offset 435 * @offset: searched physical offset, 436 * it will be used for returning physical page offset value 437 * 438 * Returns: 439 * The target scatterlist pointer and the target page position. 440 * 441 * In order to avoid the truncation of the input parameter, it checks the page 442 * offset n's type from the input parameter before calling 443 * __i915_gem_object_get_sg(). 444 * See also __i915_gem_object_page_iter_get_sg() 445 */ 446 #define i915_gem_object_get_sg(obj, n, offset) ({ \ 447 static_assert(castable_to_type(n, pgoff_t)); \ 448 __i915_gem_object_get_sg(obj, n, offset); \ 449 }) 450 451 /** 452 * __i915_gem_object_get_sg_dma - helper to find the target scatterlist 453 * pointer and the target page position using pgoff_t n input argument and 454 * drm_i915_gem_object. It uses an internal DMA mapped scatterlist lookup function 455 * @obj: i915 GEM buffer object 456 * @n: page offset 457 * @offset: searched physical offset, 458 * it will be used for returning physical page offset value 459 * 460 * It uses drm_i915_gem_object's internal DMA mapped scatterlist lookup function 461 * as i915_gem_object_page_iter and calls __i915_gem_object_page_iter_get_sg(). 462 * 463 * Returns: 464 * The target scatterlist pointer and the target page position. 465 * 466 * Recommended to use wrapper macro: i915_gem_object_get_sg_dma() 467 * See also __i915_gem_object_page_iter_get_sg() 468 */ 469 static inline struct scatterlist * 470 __i915_gem_object_get_sg_dma(struct drm_i915_gem_object *obj, pgoff_t n, 471 unsigned int *offset) 472 { 473 return __i915_gem_object_page_iter_get_sg(obj, &obj->mm.get_dma_page, n, offset); 474 } 475 476 /** 477 * i915_gem_object_get_sg_dma - wrapper macro for __i915_gem_object_get_sg_dma() 478 * @obj: i915 GEM buffer object 479 * @n: page offset 480 * @offset: searched physical offset, 481 * it will be used for returning physical page offset value 482 * 483 * Returns: 484 * The target scatterlist pointer and the target page position. 485 * 486 * In order to avoid the truncation of the input parameter, it checks the page 487 * offset n's type from the input parameter before calling 488 * __i915_gem_object_get_sg_dma(). 489 * See also __i915_gem_object_page_iter_get_sg() 490 */ 491 #define i915_gem_object_get_sg_dma(obj, n, offset) ({ \ 492 static_assert(castable_to_type(n, pgoff_t)); \ 493 __i915_gem_object_get_sg_dma(obj, n, offset); \ 494 }) 495 496 /** 497 * __i915_gem_object_get_page - helper to find the target page with a page offset 498 * @obj: i915 GEM buffer object 499 * @n: page offset 500 * 501 * It uses drm_i915_gem_object's internal shmem scatterlist lookup function as 502 * i915_gem_object_page_iter and calls __i915_gem_object_page_iter_get_sg() 503 * internally. 504 * 505 * Returns: 506 * The target page pointer. 507 * 508 * Recommended to use wrapper macro: i915_gem_object_get_page() 509 * See also __i915_gem_object_page_iter_get_sg() 510 */ 511 struct page * 512 __i915_gem_object_get_page(struct drm_i915_gem_object *obj, pgoff_t n); 513 514 /** 515 * i915_gem_object_get_page - wrapper macro for __i915_gem_object_get_page 516 * @obj: i915 GEM buffer object 517 * @n: page offset 518 * 519 * Returns: 520 * The target page pointer. 521 * 522 * In order to avoid the truncation of the input parameter, it checks the page 523 * offset n's type from the input parameter before calling 524 * __i915_gem_object_get_page(). 525 * See also __i915_gem_object_page_iter_get_sg() 526 */ 527 #define i915_gem_object_get_page(obj, n) ({ \ 528 static_assert(castable_to_type(n, pgoff_t)); \ 529 __i915_gem_object_get_page(obj, n); \ 530 }) 531 532 /** 533 * __i915_gem_object_get_dirty_page - helper to find the target page with a page 534 * offset 535 * @obj: i915 GEM buffer object 536 * @n: page offset 537 * 538 * It works like i915_gem_object_get_page(), but it marks the returned page dirty. 539 * 540 * Returns: 541 * The target page pointer. 542 * 543 * Recommended to use wrapper macro: i915_gem_object_get_dirty_page() 544 * See also __i915_gem_object_page_iter_get_sg() and __i915_gem_object_get_page() 545 */ 546 struct page * 547 __i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, pgoff_t n); 548 549 /** 550 * i915_gem_object_get_dirty_page - wrapper macro for __i915_gem_object_get_dirty_page 551 * @obj: i915 GEM buffer object 552 * @n: page offset 553 * 554 * Returns: 555 * The target page pointer. 556 * 557 * In order to avoid the truncation of the input parameter, it checks the page 558 * offset n's type from the input parameter before calling 559 * __i915_gem_object_get_dirty_page(). 560 * See also __i915_gem_object_page_iter_get_sg() and __i915_gem_object_get_page() 561 */ 562 #define i915_gem_object_get_dirty_page(obj, n) ({ \ 563 static_assert(castable_to_type(n, pgoff_t)); \ 564 __i915_gem_object_get_dirty_page(obj, n); \ 565 }) 566 567 /** 568 * __i915_gem_object_get_dma_address_len - helper to get bus addresses of 569 * targeted DMA mapped scatterlist from i915 GEM buffer object and it's length 570 * @obj: i915 GEM buffer object 571 * @n: page offset 572 * @len: DMA mapped scatterlist's DMA bus addresses length to return 573 * 574 * Returns: 575 * Bus addresses of targeted DMA mapped scatterlist 576 * 577 * Recommended to use wrapper macro: i915_gem_object_get_dma_address_len() 578 * See also __i915_gem_object_page_iter_get_sg() and __i915_gem_object_get_sg_dma() 579 */ 580 dma_addr_t 581 __i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, pgoff_t n, 582 unsigned int *len); 583 584 /** 585 * i915_gem_object_get_dma_address_len - wrapper macro for 586 * __i915_gem_object_get_dma_address_len 587 * @obj: i915 GEM buffer object 588 * @n: page offset 589 * @len: DMA mapped scatterlist's DMA bus addresses length to return 590 * 591 * Returns: 592 * Bus addresses of targeted DMA mapped scatterlist 593 * 594 * In order to avoid the truncation of the input parameter, it checks the page 595 * offset n's type from the input parameter before calling 596 * __i915_gem_object_get_dma_address_len(). 597 * See also __i915_gem_object_page_iter_get_sg() and 598 * __i915_gem_object_get_dma_address_len() 599 */ 600 #define i915_gem_object_get_dma_address_len(obj, n, len) ({ \ 601 static_assert(castable_to_type(n, pgoff_t)); \ 602 __i915_gem_object_get_dma_address_len(obj, n, len); \ 603 }) 604 605 /** 606 * __i915_gem_object_get_dma_address - helper to get bus addresses of 607 * targeted DMA mapped scatterlist from i915 GEM buffer object 608 * @obj: i915 GEM buffer object 609 * @n: page offset 610 * 611 * Returns: 612 * Bus addresses of targeted DMA mapped scatterlis 613 * 614 * Recommended to use wrapper macro: i915_gem_object_get_dma_address() 615 * See also __i915_gem_object_page_iter_get_sg() and __i915_gem_object_get_sg_dma() 616 */ 617 dma_addr_t 618 __i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, pgoff_t n); 619 620 /** 621 * i915_gem_object_get_dma_address - wrapper macro for 622 * __i915_gem_object_get_dma_address 623 * @obj: i915 GEM buffer object 624 * @n: page offset 625 * 626 * Returns: 627 * Bus addresses of targeted DMA mapped scatterlist 628 * 629 * In order to avoid the truncation of the input parameter, it checks the page 630 * offset n's type from the input parameter before calling 631 * __i915_gem_object_get_dma_address(). 632 * See also __i915_gem_object_page_iter_get_sg() and 633 * __i915_gem_object_get_dma_address() 634 */ 635 #define i915_gem_object_get_dma_address(obj, n) ({ \ 636 static_assert(castable_to_type(n, pgoff_t)); \ 637 __i915_gem_object_get_dma_address(obj, n); \ 638 }) 639 640 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, 641 struct sg_table *pages); 642 643 int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj); 644 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj); 645 646 static inline int __must_check 647 i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) 648 { 649 assert_object_held(obj); 650 651 if (atomic_inc_not_zero(&obj->mm.pages_pin_count)) 652 return 0; 653 654 return __i915_gem_object_get_pages(obj); 655 } 656 657 int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj); 658 659 static inline bool 660 i915_gem_object_has_pages(struct drm_i915_gem_object *obj) 661 { 662 return !IS_ERR_OR_NULL(READ_ONCE(obj->mm.pages)); 663 } 664 665 static inline void 666 __i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) 667 { 668 GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 669 670 atomic_inc(&obj->mm.pages_pin_count); 671 } 672 673 static inline bool 674 i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj) 675 { 676 return atomic_read(&obj->mm.pages_pin_count); 677 } 678 679 static inline void 680 __i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) 681 { 682 GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 683 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 684 685 atomic_dec(&obj->mm.pages_pin_count); 686 } 687 688 static inline void 689 i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) 690 { 691 __i915_gem_object_unpin_pages(obj); 692 } 693 694 int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj); 695 int i915_gem_object_truncate(struct drm_i915_gem_object *obj); 696 697 /** 698 * i915_gem_object_pin_map - return a contiguous mapping of the entire object 699 * @obj: the object to map into kernel address space 700 * @type: the type of mapping, used to select pgprot_t 701 * 702 * Calls i915_gem_object_pin_pages() to prevent reaping of the object's 703 * pages and then returns a contiguous mapping of the backing storage into 704 * the kernel address space. Based on the @type of mapping, the PTE will be 705 * set to either WriteBack or WriteCombine (via pgprot_t). 706 * 707 * The caller is responsible for calling i915_gem_object_unpin_map() when the 708 * mapping is no longer required. 709 * 710 * Returns the pointer through which to access the mapped object, or an 711 * ERR_PTR() on error. 712 */ 713 void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj, 714 enum i915_map_type type); 715 716 void *__must_check i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj, 717 enum i915_map_type type); 718 719 void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj, 720 unsigned long offset, 721 unsigned long size); 722 static inline void i915_gem_object_flush_map(struct drm_i915_gem_object *obj) 723 { 724 __i915_gem_object_flush_map(obj, 0, obj->base.size); 725 } 726 727 /** 728 * i915_gem_object_unpin_map - releases an earlier mapping 729 * @obj: the object to unmap 730 * 731 * After pinning the object and mapping its pages, once you are finished 732 * with your access, call i915_gem_object_unpin_map() to release the pin 733 * upon the mapping. Once the pin count reaches zero, that mapping may be 734 * removed. 735 */ 736 static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj) 737 { 738 i915_gem_object_unpin_pages(obj); 739 } 740 741 void __i915_gem_object_release_map(struct drm_i915_gem_object *obj); 742 743 int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj, 744 unsigned int *needs_clflush); 745 int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj, 746 unsigned int *needs_clflush); 747 #define CLFLUSH_BEFORE BIT(0) 748 #define CLFLUSH_AFTER BIT(1) 749 #define CLFLUSH_FLAGS (CLFLUSH_BEFORE | CLFLUSH_AFTER) 750 751 static inline void 752 i915_gem_object_finish_access(struct drm_i915_gem_object *obj) 753 { 754 i915_gem_object_unpin_pages(obj); 755 } 756 757 int i915_gem_object_get_moving_fence(struct drm_i915_gem_object *obj, 758 struct dma_fence **fence); 759 int i915_gem_object_wait_moving_fence(struct drm_i915_gem_object *obj, 760 bool intr); 761 bool i915_gem_object_has_unknown_state(struct drm_i915_gem_object *obj); 762 763 void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj, 764 unsigned int cache_level); 765 void i915_gem_object_set_pat_index(struct drm_i915_gem_object *obj, 766 unsigned int pat_index); 767 bool i915_gem_object_can_bypass_llc(struct drm_i915_gem_object *obj); 768 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj); 769 void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj); 770 bool i915_gem_cpu_write_needs_clflush(struct drm_i915_gem_object *obj); 771 772 int __must_check 773 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write); 774 int __must_check 775 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write); 776 int __must_check 777 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write); 778 struct i915_vma * __must_check 779 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 780 struct i915_gem_ww_ctx *ww, 781 u32 alignment, 782 const struct i915_gtt_view *view, 783 unsigned int flags); 784 785 void i915_gem_object_make_unshrinkable(struct drm_i915_gem_object *obj); 786 void i915_gem_object_make_shrinkable(struct drm_i915_gem_object *obj); 787 void __i915_gem_object_make_shrinkable(struct drm_i915_gem_object *obj); 788 void __i915_gem_object_make_purgeable(struct drm_i915_gem_object *obj); 789 void i915_gem_object_make_purgeable(struct drm_i915_gem_object *obj); 790 791 static inline void __start_cpu_write(struct drm_i915_gem_object *obj) 792 { 793 obj->read_domains = I915_GEM_DOMAIN_CPU; 794 obj->write_domain = I915_GEM_DOMAIN_CPU; 795 if (i915_gem_cpu_write_needs_clflush(obj)) 796 obj->cache_dirty = true; 797 } 798 799 void i915_gem_fence_wait_priority(struct dma_fence *fence, 800 const struct i915_sched_attr *attr); 801 802 int i915_gem_object_wait(struct drm_i915_gem_object *obj, 803 unsigned int flags, 804 long timeout); 805 int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, 806 unsigned int flags, 807 const struct i915_sched_attr *attr); 808 809 int i915_gem_object_read_from_page(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size); 810 811 bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj); 812 813 void __i915_gem_free_object_rcu(struct rcu_head *head); 814 815 void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj); 816 817 void __i915_gem_free_object(struct drm_i915_gem_object *obj); 818 819 bool i915_gem_object_evictable(struct drm_i915_gem_object *obj); 820 821 bool i915_gem_object_migratable(struct drm_i915_gem_object *obj); 822 823 int i915_gem_object_migrate(struct drm_i915_gem_object *obj, 824 struct i915_gem_ww_ctx *ww, 825 enum intel_region_id id); 826 int __i915_gem_object_migrate(struct drm_i915_gem_object *obj, 827 struct i915_gem_ww_ctx *ww, 828 enum intel_region_id id, 829 unsigned int flags); 830 831 bool i915_gem_object_can_migrate(struct drm_i915_gem_object *obj, 832 enum intel_region_id id); 833 834 int i915_gem_object_wait_migration(struct drm_i915_gem_object *obj, 835 unsigned int flags); 836 837 bool i915_gem_object_placement_possible(struct drm_i915_gem_object *obj, 838 enum intel_memory_type type); 839 840 bool i915_gem_object_needs_ccs_pages(struct drm_i915_gem_object *obj); 841 842 int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st, 843 size_t size, struct intel_memory_region *mr, 844 struct address_space *mapping, 845 unsigned int max_segment); 846 void shmem_sg_free_table(struct sg_table *st, struct address_space *mapping, 847 bool dirty, bool backup); 848 void __shmem_writeback(size_t size, struct address_space *mapping); 849 850 #ifdef CONFIG_MMU_NOTIFIER 851 static inline bool 852 i915_gem_object_is_userptr(struct drm_i915_gem_object *obj) 853 { 854 return obj->userptr.notifier.mm; 855 } 856 857 int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj); 858 int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj); 859 int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj); 860 #else 861 static inline bool i915_gem_object_is_userptr(struct drm_i915_gem_object *obj) { return false; } 862 863 static inline int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } 864 static inline int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } 865 static inline int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } 866 867 #endif 868 869 #endif 870