1 /* 2 * Copyright 2020 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Christian König 23 */ 24 25 #ifndef _TTM_RESOURCE_H_ 26 #define _TTM_RESOURCE_H_ 27 28 #include <linux/types.h> 29 #include <linux/list.h> 30 #include <linux/mutex.h> 31 #include <linux/iosys-map.h> 32 #include <linux/dma-fence.h> 33 34 #include <drm/ttm/ttm_caching.h> 35 #include <drm/ttm/ttm_kmap_iter.h> 36 37 #define TTM_MAX_BO_PRIORITY 4U 38 #define TTM_NUM_MEM_TYPES 9 39 40 struct dentry; 41 struct dmem_cgroup_device; 42 struct dmem_cgroup_region; 43 struct drm_printer; 44 struct ttm_device; 45 struct ttm_resource_manager; 46 struct ttm_resource; 47 struct ttm_place; 48 struct ttm_buffer_object; 49 struct ttm_placement; 50 struct iosys_map; 51 struct io_mapping; 52 struct sg_table; 53 struct scatterlist; 54 55 /** 56 * define TTM_NUM_MOVE_FENCES - How many entities can be used for evictions 57 * 58 * Pipelined evictions can be spread on multiple entities. This 59 * is the max number of entities that can be used by the driver 60 * for that purpose. 61 */ 62 #define TTM_NUM_MOVE_FENCES 8 63 64 /** 65 * enum ttm_lru_item_type - enumerate ttm_lru_item subclasses 66 */ 67 enum ttm_lru_item_type { 68 /** @TTM_LRU_RESOURCE: The resource subclass */ 69 TTM_LRU_RESOURCE, 70 /** @TTM_LRU_HITCH: The iterator hitch subclass */ 71 TTM_LRU_HITCH 72 }; 73 74 /** 75 * struct ttm_lru_item - The TTM lru list node base class 76 * @link: The list link 77 * @type: The subclass type 78 */ 79 struct ttm_lru_item { 80 struct list_head link; 81 enum ttm_lru_item_type type; 82 }; 83 84 /** 85 * ttm_lru_item_init() - initialize a struct ttm_lru_item 86 * @item: The item to initialize 87 * @type: The subclass type 88 */ 89 static inline void ttm_lru_item_init(struct ttm_lru_item *item, 90 enum ttm_lru_item_type type) 91 { 92 item->type = type; 93 INIT_LIST_HEAD(&item->link); 94 } 95 96 static inline bool ttm_lru_item_is_res(const struct ttm_lru_item *item) 97 { 98 return item->type == TTM_LRU_RESOURCE; 99 } 100 101 struct ttm_resource_manager_func { 102 /** 103 * struct ttm_resource_manager_func member alloc 104 * 105 * @man: Pointer to a memory type manager. 106 * @bo: Pointer to the buffer object we're allocating space for. 107 * @place: Placement details. 108 * @res: Resulting pointer to the ttm_resource. 109 * 110 * This function should allocate space in the memory type managed 111 * by @man. Placement details if applicable are given by @place. If 112 * successful, a filled in ttm_resource object should be returned in 113 * @res. @res::start should be set to a value identifying the beginning 114 * of the range allocated, and the function should return zero. 115 * If the manager can't fulfill the request -ENOSPC should be returned. 116 * If a system error occurred, preventing the request to be fulfilled, 117 * the function should return a negative error code. 118 * 119 * This function may not be called from within atomic context and needs 120 * to take care of its own locking to protect any data structures 121 * managing the space. 122 */ 123 int (*alloc)(struct ttm_resource_manager *man, 124 struct ttm_buffer_object *bo, 125 const struct ttm_place *place, 126 struct ttm_resource **res); 127 128 /** 129 * struct ttm_resource_manager_func member free 130 * 131 * @man: Pointer to a memory type manager. 132 * @res: Pointer to a struct ttm_resource to be freed. 133 * 134 * This function frees memory type resources previously allocated. 135 * May not be called from within atomic context. 136 */ 137 void (*free)(struct ttm_resource_manager *man, 138 struct ttm_resource *res); 139 140 /** 141 * struct ttm_resource_manager_func member intersects 142 * 143 * @man: Pointer to a memory type manager. 144 * @res: Pointer to a struct ttm_resource to be checked. 145 * @place: Placement to check against. 146 * @size: Size of the check. 147 * 148 * Test if @res intersects with @place + @size. Used to judge if 149 * evictions are valueable or not. 150 */ 151 bool (*intersects)(struct ttm_resource_manager *man, 152 struct ttm_resource *res, 153 const struct ttm_place *place, 154 size_t size); 155 156 /** 157 * struct ttm_resource_manager_func member compatible 158 * 159 * @man: Pointer to a memory type manager. 160 * @res: Pointer to a struct ttm_resource to be checked. 161 * @place: Placement to check against. 162 * @size: Size of the check. 163 * 164 * Test if @res compatible with @place + @size. Used to check of 165 * the need to move the backing store or not. 166 */ 167 bool (*compatible)(struct ttm_resource_manager *man, 168 struct ttm_resource *res, 169 const struct ttm_place *place, 170 size_t size); 171 172 /** 173 * struct ttm_resource_manager_func member debug 174 * 175 * @man: Pointer to a memory type manager. 176 * @printer: Prefix to be used in printout to identify the caller. 177 * 178 * This function is called to print out the state of the memory 179 * type manager to aid debugging of out-of-memory conditions. 180 * It may not be called from within atomic context. 181 */ 182 void (*debug)(struct ttm_resource_manager *man, 183 struct drm_printer *printer); 184 }; 185 186 /** 187 * struct ttm_resource_manager 188 * 189 * @use_type: The memory type is enabled. 190 * @use_tt: If a TT object should be used for the backing store. 191 * @size: Size of the managed region. 192 * @bdev: ttm device this manager belongs to 193 * @func: structure pointer implementing the range manager. See above 194 * @eviction_lock: lock for eviction fences 195 * @eviction_fences: The fences of the last pipelined move operation. 196 * @lru: The lru list for this memory type. 197 * 198 * This structure is used to identify and manage memory types for a device. 199 */ 200 struct ttm_resource_manager { 201 /* 202 * No protection. Constant from start. 203 */ 204 bool use_type; 205 bool use_tt; 206 struct ttm_device *bdev; 207 uint64_t size; 208 const struct ttm_resource_manager_func *func; 209 210 /* This is very similar to a dma_resv object, but locking rules make 211 * it difficult to use one in this context. 212 */ 213 spinlock_t eviction_lock; 214 struct dma_fence *eviction_fences[TTM_NUM_MOVE_FENCES]; 215 216 /* 217 * Protected by the bdev->lru_lock. 218 */ 219 struct list_head lru[TTM_MAX_BO_PRIORITY]; 220 221 /** 222 * @usage: How much of the resources are used, protected by the 223 * bdev->lru_lock. 224 */ 225 uint64_t usage; 226 227 /** 228 * @cg: &dmem_cgroup_region used for memory accounting, if not NULL. 229 */ 230 struct dmem_cgroup_region *cg; 231 }; 232 233 /** 234 * struct ttm_bus_placement 235 * 236 * @addr: mapped virtual address 237 * @offset: physical addr 238 * @is_iomem: is this io memory ? 239 * @caching: See enum ttm_caching 240 * 241 * Structure indicating the bus placement of an object. 242 */ 243 struct ttm_bus_placement { 244 void *addr; 245 phys_addr_t offset; 246 bool is_iomem; 247 enum ttm_caching caching; 248 }; 249 250 /** 251 * struct ttm_resource 252 * 253 * @start: Start of the allocation. 254 * @size: Actual size of resource in bytes. 255 * @mem_type: Resource type of the allocation. 256 * @placement: Placement flags. 257 * @bus: Placement on io bus accessible to the CPU 258 * @bo: weak reference to the BO, protected by ttm_device::lru_lock 259 * @css: cgroup state this resource is charged to 260 * 261 * Structure indicating the placement and space resources used by a 262 * buffer object. 263 */ 264 struct ttm_resource { 265 unsigned long start; 266 size_t size; 267 uint32_t mem_type; 268 uint32_t placement; 269 struct ttm_bus_placement bus; 270 struct ttm_buffer_object *bo; 271 272 struct dmem_cgroup_pool_state *css; 273 274 /** 275 * @lru: Least recently used list, see &ttm_resource_manager.lru 276 */ 277 struct ttm_lru_item lru; 278 }; 279 280 /** 281 * ttm_lru_item_to_res() - Downcast a struct ttm_lru_item to a struct ttm_resource 282 * @item: The struct ttm_lru_item to downcast 283 * 284 * Return: Pointer to the embedding struct ttm_resource 285 */ 286 static inline struct ttm_resource * 287 ttm_lru_item_to_res(struct ttm_lru_item *item) 288 { 289 return container_of(item, struct ttm_resource, lru); 290 } 291 292 /** 293 * struct ttm_lru_bulk_move_pos 294 * 295 * @first: first res in the bulk move range 296 * @last: last res in the bulk move range 297 * 298 * Range of resources for a lru bulk move. 299 */ 300 struct ttm_lru_bulk_move_pos { 301 struct ttm_resource *first; 302 struct ttm_resource *last; 303 }; 304 305 /** 306 * struct ttm_lru_bulk_move 307 * @pos: first/last lru entry for resources in the each domain/priority 308 * @cursor_list: The list of cursors currently traversing any of 309 * the sublists of @pos. Protected by the ttm device's lru_lock. 310 * 311 * Container for the current bulk move state. Should be used with 312 * ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move(). 313 * All BOs in a bulk_move structure need to share the same reservation object to 314 * ensure that the bulk as a whole is locked for eviction even if only one BO of 315 * the bulk is evicted. 316 */ 317 struct ttm_lru_bulk_move { 318 struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY]; 319 struct list_head cursor_list; 320 }; 321 322 /** 323 * struct ttm_resource_cursor 324 * @man: The resource manager currently being iterated over 325 * @hitch: A hitch list node inserted before the next resource 326 * to iterate over. 327 * @bulk_link: A list link for the list of cursors traversing the 328 * bulk sublist of @bulk. Protected by the ttm device's lru_lock. 329 * @bulk: Pointer to struct ttm_lru_bulk_move whose subrange @hitch is 330 * inserted to. NULL if none. Never dereference this pointer since 331 * the struct ttm_lru_bulk_move object pointed to might have been 332 * freed. The pointer is only for comparison. 333 * @mem_type: The memory type of the LRU list being traversed. 334 * This field is valid iff @bulk != NULL. 335 * @priority: the current priority 336 * 337 * Cursor to iterate over the resources in a manager. 338 */ 339 struct ttm_resource_cursor { 340 struct ttm_resource_manager *man; 341 struct ttm_lru_item hitch; 342 struct list_head bulk_link; 343 struct ttm_lru_bulk_move *bulk; 344 unsigned int mem_type; 345 unsigned int priority; 346 }; 347 348 void ttm_resource_cursor_init(struct ttm_resource_cursor *cursor, 349 struct ttm_resource_manager *man); 350 351 void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor); 352 353 /** 354 * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping + 355 * struct sg_table backed struct ttm_resource. 356 * @base: Embedded struct ttm_kmap_iter providing the usage interface. 357 * @iomap: struct io_mapping representing the underlying linear io_memory. 358 * @st: sg_table into @iomap, representing the memory of the struct ttm_resource. 359 * @start: Offset that needs to be subtracted from @st to make 360 * sg_dma_address(st->sgl) - @start == 0 for @iomap start. 361 * @cache: Scatterlist traversal cache for fast lookups. 362 * @cache.sg: Pointer to the currently cached scatterlist segment. 363 * @cache.i: First index of @sg. PAGE_SIZE granularity. 364 * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity. 365 * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity. 366 */ 367 struct ttm_kmap_iter_iomap { 368 struct ttm_kmap_iter base; 369 struct io_mapping *iomap; 370 struct sg_table *st; 371 resource_size_t start; 372 struct { 373 struct scatterlist *sg; 374 pgoff_t i; 375 pgoff_t end; 376 pgoff_t offs; 377 } cache; 378 }; 379 380 /** 381 * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io 382 * @base: The base iterator 383 * @dmap: Points to the starting address of the region 384 * @needs_unmap: Whether we need to unmap on fini 385 */ 386 struct ttm_kmap_iter_linear_io { 387 struct ttm_kmap_iter base; 388 struct iosys_map dmap; 389 bool needs_unmap; 390 }; 391 392 /** 393 * ttm_resource_manager_set_used 394 * 395 * @man: A memory manager object. 396 * @used: usage state to set. 397 * 398 * Set the manager in use flag. If disabled the manager is no longer 399 * used for object placement. 400 */ 401 static inline void 402 ttm_resource_manager_set_used(struct ttm_resource_manager *man, bool used) 403 { 404 int i; 405 406 for (i = 0; i < TTM_MAX_BO_PRIORITY; i++) 407 WARN_ON(!list_empty(&man->lru[i])); 408 man->use_type = used; 409 } 410 411 /** 412 * ttm_resource_manager_used 413 * 414 * @man: Manager to get used state for 415 * 416 * Get the in use flag for a manager. 417 * Returns: 418 * true is used, false if not. 419 */ 420 static inline bool ttm_resource_manager_used(struct ttm_resource_manager *man) 421 { 422 return man->use_type; 423 } 424 425 /** 426 * ttm_resource_manager_cleanup 427 * 428 * @man: A memory manager object. 429 * 430 * Cleanup the move fences from the memory manager object. 431 */ 432 static inline void 433 ttm_resource_manager_cleanup(struct ttm_resource_manager *man) 434 { 435 int i; 436 437 for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) { 438 dma_fence_put(man->eviction_fences[i]); 439 man->eviction_fences[i] = NULL; 440 } 441 } 442 443 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk); 444 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk); 445 void ttm_lru_bulk_move_fini(struct ttm_device *bdev, 446 struct ttm_lru_bulk_move *bulk); 447 448 void ttm_resource_add_bulk_move(struct ttm_resource *res, 449 struct ttm_buffer_object *bo); 450 void ttm_resource_del_bulk_move(struct ttm_resource *res, 451 struct ttm_buffer_object *bo); 452 void ttm_resource_del_bulk_move_unevictable(struct ttm_resource *res, 453 struct ttm_buffer_object *bo); 454 void ttm_resource_move_to_lru_tail(struct ttm_resource *res); 455 456 void ttm_resource_init(struct ttm_buffer_object *bo, 457 const struct ttm_place *place, 458 struct ttm_resource *res); 459 void ttm_resource_fini(struct ttm_resource_manager *man, 460 struct ttm_resource *res); 461 462 int ttm_resource_try_charge(struct ttm_buffer_object *bo, 463 const struct ttm_place *place, 464 struct dmem_cgroup_pool_state **ret_pool, 465 struct dmem_cgroup_pool_state **ret_limit_pool); 466 int ttm_resource_alloc(struct ttm_buffer_object *bo, 467 const struct ttm_place *place, 468 struct ttm_resource **res, 469 struct dmem_cgroup_pool_state *charge_pool); 470 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res); 471 bool ttm_resource_intersects(struct ttm_device *bdev, 472 struct ttm_resource *res, 473 const struct ttm_place *place, 474 size_t size); 475 bool ttm_resource_compatible(struct ttm_resource *res, 476 struct ttm_placement *placement, 477 bool evicting); 478 void ttm_resource_set_bo(struct ttm_resource *res, 479 struct ttm_buffer_object *bo); 480 481 void ttm_resource_manager_init(struct ttm_resource_manager *man, 482 struct ttm_device *bdev, 483 uint64_t size); 484 485 void ttm_resource_manager_set_dmem_region(struct ttm_resource_manager *man, 486 struct dmem_cgroup_region *region); 487 488 int ttm_resource_manager_dmem_reclaim(struct dmem_cgroup_pool_state *pool, 489 u64 target_bytes, void *priv); 490 491 int ttm_resource_manager_evict_all(struct ttm_device *bdev, 492 struct ttm_resource_manager *man); 493 494 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man); 495 void ttm_resource_manager_debug(struct ttm_resource_manager *man, 496 struct drm_printer *p); 497 498 struct ttm_resource * 499 ttm_resource_manager_first(struct ttm_resource_cursor *cursor); 500 struct ttm_resource * 501 ttm_resource_manager_next(struct ttm_resource_cursor *cursor); 502 503 struct ttm_resource * 504 ttm_lru_first_res_or_null(struct list_head *head); 505 506 /** 507 * ttm_resource_manager_for_each_res - iterate over all resources 508 * @cursor: struct ttm_resource_cursor for the current position 509 * @res: the current resource 510 * 511 * Iterate over all the evictable resources in a resource manager. 512 */ 513 #define ttm_resource_manager_for_each_res(cursor, res) \ 514 for (res = ttm_resource_manager_first(cursor); res; \ 515 res = ttm_resource_manager_next(cursor)) 516 517 struct ttm_kmap_iter * 518 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io, 519 struct io_mapping *iomap, 520 struct sg_table *st, 521 resource_size_t start); 522 523 struct ttm_kmap_iter_linear_io; 524 525 struct ttm_kmap_iter * 526 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, 527 struct ttm_device *bdev, 528 struct ttm_resource *mem); 529 530 void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io, 531 struct ttm_device *bdev, 532 struct ttm_resource *mem); 533 534 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man, 535 struct dentry * parent, 536 const char *name); 537 #endif 538