1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #ifndef _XE_BO_H_ 7 #define _XE_BO_H_ 8 9 #include <drm/ttm/ttm_tt.h> 10 11 #include "xe_bo_types.h" 12 #include "xe_macros.h" 13 #include "xe_vm_types.h" 14 #include "xe_vm.h" 15 #include "xe_vram_types.h" 16 17 #define XE_DEFAULT_GTT_SIZE_MB 3072ULL /* 3GB by default */ 18 19 #define XE_BO_FLAG_USER BIT(0) 20 /* The bits below need to be contiguous, or things break */ 21 #define XE_BO_FLAG_SYSTEM BIT(1) 22 #define XE_BO_FLAG_VRAM0 BIT(2) 23 #define XE_BO_FLAG_VRAM1 BIT(3) 24 #define XE_BO_FLAG_VRAM_MASK (XE_BO_FLAG_VRAM0 | XE_BO_FLAG_VRAM1) 25 /* -- */ 26 #define XE_BO_FLAG_STOLEN BIT(4) 27 #define XE_BO_FLAG_VRAM(vram) (XE_BO_FLAG_VRAM0 << ((vram)->id)) 28 #define XE_BO_FLAG_VRAM_IF_DGFX(tile) (IS_DGFX(tile_to_xe(tile)) ? \ 29 XE_BO_FLAG_VRAM((tile)->mem.vram) : \ 30 XE_BO_FLAG_SYSTEM) 31 #define XE_BO_FLAG_GGTT BIT(5) 32 #define XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE BIT(6) 33 #define XE_BO_FLAG_PINNED BIT(7) 34 #define XE_BO_FLAG_NO_RESV_EVICT BIT(8) 35 #define XE_BO_FLAG_DEFER_BACKING BIT(9) 36 #define XE_BO_FLAG_SCANOUT BIT(10) 37 #define XE_BO_FLAG_FIXED_PLACEMENT BIT(11) 38 #define XE_BO_FLAG_PAGETABLE BIT(12) 39 #define XE_BO_FLAG_NEEDS_CPU_ACCESS BIT(13) 40 #define XE_BO_FLAG_NEEDS_UC BIT(14) 41 #define XE_BO_FLAG_NEEDS_64K BIT(15) 42 #define XE_BO_FLAG_NEEDS_2M BIT(16) 43 #define XE_BO_FLAG_GGTT_INVALIDATE BIT(17) 44 #define XE_BO_FLAG_PINNED_NORESTORE BIT(18) 45 #define XE_BO_FLAG_PINNED_LATE_RESTORE BIT(19) 46 #define XE_BO_FLAG_GGTT0 BIT(20) 47 #define XE_BO_FLAG_GGTT1 BIT(21) 48 #define XE_BO_FLAG_GGTT2 BIT(22) 49 #define XE_BO_FLAG_GGTT3 BIT(23) 50 #define XE_BO_FLAG_CPU_ADDR_MIRROR BIT(24) 51 52 /* this one is trigger internally only */ 53 #define XE_BO_FLAG_INTERNAL_TEST BIT(30) 54 #define XE_BO_FLAG_INTERNAL_64K BIT(31) 55 56 #define XE_BO_FLAG_GGTT_ALL (XE_BO_FLAG_GGTT0 | \ 57 XE_BO_FLAG_GGTT1 | \ 58 XE_BO_FLAG_GGTT2 | \ 59 XE_BO_FLAG_GGTT3) 60 61 #define XE_BO_FLAG_GGTTx(tile) \ 62 (XE_BO_FLAG_GGTT0 << (tile)->id) 63 64 #define XE_PTE_SHIFT 12 65 #define XE_PAGE_SIZE (1 << XE_PTE_SHIFT) 66 #define XE_PTE_MASK (XE_PAGE_SIZE - 1) 67 #define XE_PDE_SHIFT (XE_PTE_SHIFT - 3) 68 #define XE_PDES (1 << XE_PDE_SHIFT) 69 #define XE_PDE_MASK (XE_PDES - 1) 70 71 #define XE_64K_PTE_SHIFT 16 72 #define XE_64K_PAGE_SIZE (1 << XE_64K_PTE_SHIFT) 73 #define XE_64K_PTE_MASK (XE_64K_PAGE_SIZE - 1) 74 #define XE_64K_PDE_MASK (XE_PDE_MASK >> 4) 75 76 #define XE_PL_SYSTEM TTM_PL_SYSTEM 77 #define XE_PL_TT TTM_PL_TT 78 #define XE_PL_VRAM0 TTM_PL_VRAM 79 #define XE_PL_VRAM1 (XE_PL_VRAM0 + 1) 80 #define XE_PL_STOLEN (TTM_NUM_MEM_TYPES - 1) 81 82 #define XE_BO_PROPS_INVALID (-1) 83 84 #define XE_PCI_BARRIER_MMAP_OFFSET (0x50 << XE_PTE_SHIFT) 85 86 struct sg_table; 87 88 struct xe_bo *xe_bo_alloc(void); 89 void xe_bo_free(struct xe_bo *bo); 90 91 struct xe_bo *___xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, 92 struct xe_tile *tile, struct dma_resv *resv, 93 struct ttm_lru_bulk_move *bulk, size_t size, 94 u16 cpu_caching, enum ttm_bo_type type, 95 u32 flags); 96 struct xe_bo * 97 xe_bo_create_locked_range(struct xe_device *xe, 98 struct xe_tile *tile, struct xe_vm *vm, 99 size_t size, u64 start, u64 end, 100 enum ttm_bo_type type, u32 flags, u64 alignment); 101 struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile, 102 struct xe_vm *vm, size_t size, 103 enum ttm_bo_type type, u32 flags); 104 struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_tile *tile, 105 struct xe_vm *vm, size_t size, 106 enum ttm_bo_type type, u32 flags); 107 struct xe_bo *xe_bo_create_user(struct xe_device *xe, struct xe_tile *tile, 108 struct xe_vm *vm, size_t size, 109 u16 cpu_caching, 110 u32 flags); 111 struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile, 112 struct xe_vm *vm, size_t size, 113 enum ttm_bo_type type, u32 flags); 114 struct xe_bo *xe_bo_create_pin_map_at(struct xe_device *xe, struct xe_tile *tile, 115 struct xe_vm *vm, size_t size, u64 offset, 116 enum ttm_bo_type type, u32 flags); 117 struct xe_bo *xe_bo_create_pin_map_at_aligned(struct xe_device *xe, 118 struct xe_tile *tile, 119 struct xe_vm *vm, 120 size_t size, u64 offset, 121 enum ttm_bo_type type, u32 flags, 122 u64 alignment); 123 struct xe_bo *xe_managed_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile, 124 size_t size, u32 flags); 125 struct xe_bo *xe_managed_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile, 126 const void *data, size_t size, u32 flags); 127 int xe_managed_bo_reinit_in_vram(struct xe_device *xe, struct xe_tile *tile, struct xe_bo **src); 128 129 int xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo, 130 u32 bo_flags); 131 132 static inline struct xe_bo *ttm_to_xe_bo(const struct ttm_buffer_object *bo) 133 { 134 return container_of(bo, struct xe_bo, ttm); 135 } 136 137 static inline struct xe_bo *gem_to_xe_bo(const struct drm_gem_object *obj) 138 { 139 return container_of(obj, struct xe_bo, ttm.base); 140 } 141 142 #define xe_bo_device(bo) ttm_to_xe_device((bo)->ttm.bdev) 143 144 static inline struct xe_bo *xe_bo_get(struct xe_bo *bo) 145 { 146 if (bo) 147 drm_gem_object_get(&bo->ttm.base); 148 149 return bo; 150 } 151 152 void xe_bo_put(struct xe_bo *bo); 153 154 /* 155 * xe_bo_get_unless_zero() - Conditionally obtain a GEM object refcount on an 156 * xe bo 157 * @bo: The bo for which we want to obtain a refcount. 158 * 159 * There is a short window between where the bo's GEM object refcount reaches 160 * zero and where we put the final ttm_bo reference. Code in the eviction- and 161 * shrinking path should therefore attempt to grab a gem object reference before 162 * trying to use members outside of the base class ttm object. This function is 163 * intended for that purpose. On successful return, this function must be paired 164 * with an xe_bo_put(). 165 * 166 * Return: @bo on success, NULL on failure. 167 */ 168 static inline __must_check struct xe_bo *xe_bo_get_unless_zero(struct xe_bo *bo) 169 { 170 if (!bo || !kref_get_unless_zero(&bo->ttm.base.refcount)) 171 return NULL; 172 173 return bo; 174 } 175 176 static inline void __xe_bo_unset_bulk_move(struct xe_bo *bo) 177 { 178 if (bo) 179 ttm_bo_set_bulk_move(&bo->ttm, NULL); 180 } 181 182 static inline void xe_bo_assert_held(struct xe_bo *bo) 183 { 184 if (bo) 185 dma_resv_assert_held((bo)->ttm.base.resv); 186 } 187 188 int xe_bo_lock(struct xe_bo *bo, bool intr); 189 190 void xe_bo_unlock(struct xe_bo *bo); 191 192 static inline void xe_bo_unlock_vm_held(struct xe_bo *bo) 193 { 194 if (bo) { 195 XE_WARN_ON(bo->vm && bo->ttm.base.resv != xe_vm_resv(bo->vm)); 196 if (bo->vm) 197 xe_vm_assert_held(bo->vm); 198 else 199 dma_resv_unlock(bo->ttm.base.resv); 200 } 201 } 202 203 int xe_bo_pin_external(struct xe_bo *bo); 204 int xe_bo_pin(struct xe_bo *bo); 205 void xe_bo_unpin_external(struct xe_bo *bo); 206 void xe_bo_unpin(struct xe_bo *bo); 207 int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict); 208 209 static inline bool xe_bo_is_pinned(struct xe_bo *bo) 210 { 211 return bo->ttm.pin_count; 212 } 213 214 static inline bool xe_bo_is_protected(const struct xe_bo *bo) 215 { 216 return bo->pxp_key_instance; 217 } 218 219 static inline void xe_bo_unpin_map_no_vm(struct xe_bo *bo) 220 { 221 if (likely(bo)) { 222 xe_bo_lock(bo, false); 223 xe_bo_unpin(bo); 224 xe_bo_unlock(bo); 225 226 xe_bo_put(bo); 227 } 228 } 229 230 bool xe_bo_is_xe_bo(struct ttm_buffer_object *bo); 231 dma_addr_t __xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size); 232 dma_addr_t xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size); 233 234 static inline dma_addr_t 235 xe_bo_main_addr(struct xe_bo *bo, size_t page_size) 236 { 237 return xe_bo_addr(bo, 0, page_size); 238 } 239 240 /** 241 * xe_bo_size() - Xe BO size 242 * @bo: The bo object. 243 * 244 * Simple helper to return Xe BO's size. 245 * 246 * Return: Xe BO's size 247 */ 248 static inline size_t xe_bo_size(struct xe_bo *bo) 249 { 250 return bo->ttm.base.size; 251 } 252 253 static inline u32 254 __xe_bo_ggtt_addr(struct xe_bo *bo, u8 tile_id) 255 { 256 struct xe_ggtt_node *ggtt_node = bo->ggtt_node[tile_id]; 257 258 if (XE_WARN_ON(!ggtt_node)) 259 return 0; 260 261 XE_WARN_ON(ggtt_node->base.size > xe_bo_size(bo)); 262 XE_WARN_ON(ggtt_node->base.start + ggtt_node->base.size > (1ull << 32)); 263 return ggtt_node->base.start; 264 } 265 266 static inline u32 267 xe_bo_ggtt_addr(struct xe_bo *bo) 268 { 269 xe_assert(xe_bo_device(bo), bo->tile); 270 271 return __xe_bo_ggtt_addr(bo, bo->tile->id); 272 } 273 274 int xe_bo_vmap(struct xe_bo *bo); 275 void xe_bo_vunmap(struct xe_bo *bo); 276 int xe_bo_read(struct xe_bo *bo, u64 offset, void *dst, int size); 277 278 bool mem_type_is_vram(u32 mem_type); 279 bool xe_bo_is_vram(struct xe_bo *bo); 280 bool xe_bo_is_stolen(struct xe_bo *bo); 281 bool xe_bo_is_stolen_devmem(struct xe_bo *bo); 282 bool xe_bo_is_vm_bound(struct xe_bo *bo); 283 bool xe_bo_has_single_placement(struct xe_bo *bo); 284 uint64_t vram_region_gpu_offset(struct ttm_resource *res); 285 286 bool xe_bo_can_migrate(struct xe_bo *bo, u32 mem_type); 287 288 int xe_bo_migrate(struct xe_bo *bo, u32 mem_type); 289 int xe_bo_evict(struct xe_bo *bo); 290 291 int xe_bo_evict_pinned(struct xe_bo *bo); 292 int xe_bo_notifier_prepare_pinned(struct xe_bo *bo); 293 int xe_bo_notifier_unprepare_pinned(struct xe_bo *bo); 294 int xe_bo_restore_pinned(struct xe_bo *bo); 295 296 int xe_bo_dma_unmap_pinned(struct xe_bo *bo); 297 298 extern const struct ttm_device_funcs xe_ttm_funcs; 299 extern const char *const xe_mem_type_to_name[]; 300 301 int xe_gem_create_ioctl(struct drm_device *dev, void *data, 302 struct drm_file *file); 303 int xe_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, 304 struct drm_file *file); 305 void xe_bo_runtime_pm_release_mmap_offset(struct xe_bo *bo); 306 307 int xe_bo_dumb_create(struct drm_file *file_priv, 308 struct drm_device *dev, 309 struct drm_mode_create_dumb *args); 310 311 bool xe_bo_needs_ccs_pages(struct xe_bo *bo); 312 313 static inline size_t xe_bo_ccs_pages_start(struct xe_bo *bo) 314 { 315 return PAGE_ALIGN(xe_bo_size(bo)); 316 } 317 318 static inline bool xe_bo_has_pages(struct xe_bo *bo) 319 { 320 if ((bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm)) || 321 xe_bo_is_vram(bo)) 322 return true; 323 324 return false; 325 } 326 327 void __xe_bo_release_dummy(struct kref *kref); 328 329 /** 330 * xe_bo_put_deferred() - Put a buffer object with delayed final freeing 331 * @bo: The bo to put. 332 * @deferred: List to which to add the buffer object if we cannot put, or 333 * NULL if the function is to put unconditionally. 334 * 335 * Since the final freeing of an object includes both sleeping and (!) 336 * memory allocation in the dma_resv individualization, it's not ok 337 * to put an object from atomic context nor from within a held lock 338 * tainted by reclaim. In such situations we want to defer the final 339 * freeing until we've exited the restricting context, or in the worst 340 * case to a workqueue. 341 * This function either puts the object if possible without the refcount 342 * reaching zero, or adds it to the @deferred list if that was not possible. 343 * The caller needs to follow up with a call to xe_bo_put_commit() to actually 344 * put the bo iff this function returns true. It's safe to always 345 * follow up with a call to xe_bo_put_commit(). 346 * TODO: It's TTM that is the villain here. Perhaps TTM should add an 347 * interface like this. 348 * 349 * Return: true if @bo was the first object put on the @freed list, 350 * false otherwise. 351 */ 352 static inline bool 353 xe_bo_put_deferred(struct xe_bo *bo, struct llist_head *deferred) 354 { 355 if (!deferred) { 356 xe_bo_put(bo); 357 return false; 358 } 359 360 if (!kref_put(&bo->ttm.base.refcount, __xe_bo_release_dummy)) 361 return false; 362 363 return llist_add(&bo->freed, deferred); 364 } 365 366 void xe_bo_put_commit(struct llist_head *deferred); 367 368 /** 369 * xe_bo_put_async() - Put BO async 370 * @bo: The bo to put. 371 * 372 * Put BO async, the final put is deferred to a worker to exit an IRQ context. 373 */ 374 static inline void 375 xe_bo_put_async(struct xe_bo *bo) 376 { 377 struct xe_bo_dev *bo_device = &xe_bo_device(bo)->bo_device; 378 379 if (xe_bo_put_deferred(bo, &bo_device->async_list)) 380 schedule_work(&bo_device->async_free); 381 } 382 383 void xe_bo_dev_init(struct xe_bo_dev *bo_device); 384 385 void xe_bo_dev_fini(struct xe_bo_dev *bo_device); 386 387 struct sg_table *xe_bo_sg(struct xe_bo *bo); 388 389 /* 390 * xe_sg_segment_size() - Provides upper limit for sg segment size. 391 * @dev: device pointer 392 * 393 * Returns the maximum segment size for the 'struct scatterlist' 394 * elements. 395 */ 396 static inline unsigned int xe_sg_segment_size(struct device *dev) 397 { 398 struct scatterlist __maybe_unused sg; 399 size_t max = BIT_ULL(sizeof(sg.length) * 8) - 1; 400 401 max = min_t(size_t, max, dma_max_mapping_size(dev)); 402 403 /* 404 * The iommu_dma_map_sg() function ensures iova allocation doesn't 405 * cross dma segment boundary. It does so by padding some sg elements. 406 * This can cause overflow, ending up with sg->length being set to 0. 407 * Avoid this by ensuring maximum segment size is half of 'max' 408 * rounded down to PAGE_SIZE. 409 */ 410 return round_down(max / 2, PAGE_SIZE); 411 } 412 413 /** 414 * struct xe_bo_shrink_flags - flags governing the shrink behaviour. 415 * @purge: Only purging allowed. Don't shrink if bo not purgeable. 416 * @writeback: Attempt to immediately move content to swap. 417 */ 418 struct xe_bo_shrink_flags { 419 u32 purge : 1; 420 u32 writeback : 1; 421 }; 422 423 long xe_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo, 424 const struct xe_bo_shrink_flags flags, 425 unsigned long *scanned); 426 427 /** 428 * xe_bo_is_mem_type - Whether the bo currently resides in the given 429 * TTM memory type 430 * @bo: The bo to check. 431 * @mem_type: The TTM memory type. 432 * 433 * Return: true iff the bo resides in @mem_type, false otherwise. 434 */ 435 static inline bool xe_bo_is_mem_type(struct xe_bo *bo, u32 mem_type) 436 { 437 xe_bo_assert_held(bo); 438 return bo->ttm.resource->mem_type == mem_type; 439 } 440 #endif 441