1 /* 2 * Copyright 2020 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Christian König 23 */ 24 25 #ifndef _TTM_RESOURCE_H_ 26 #define _TTM_RESOURCE_H_ 27 28 #include <linux/types.h> 29 #include <linux/list.h> 30 #include <linux/mutex.h> 31 #include <linux/iosys-map.h> 32 #include <linux/dma-fence.h> 33 34 #include <drm/ttm/ttm_caching.h> 35 #include <drm/ttm/ttm_kmap_iter.h> 36 37 #define TTM_MAX_BO_PRIORITY 4U 38 #define TTM_NUM_MEM_TYPES 9 39 40 struct dentry; 41 struct dmem_cgroup_device; 42 struct drm_printer; 43 struct ttm_device; 44 struct ttm_resource_manager; 45 struct ttm_resource; 46 struct ttm_place; 47 struct ttm_buffer_object; 48 struct ttm_placement; 49 struct iosys_map; 50 struct io_mapping; 51 struct sg_table; 52 struct scatterlist; 53 54 /** 55 * define TTM_NUM_MOVE_FENCES - How many entities can be used for evictions 56 * 57 * Pipelined evictions can be spread on multiple entities. This 58 * is the max number of entities that can be used by the driver 59 * for that purpose. 60 */ 61 #define TTM_NUM_MOVE_FENCES 8 62 63 /** 64 * enum ttm_lru_item_type - enumerate ttm_lru_item subclasses 65 */ 66 enum ttm_lru_item_type { 67 /** @TTM_LRU_RESOURCE: The resource subclass */ 68 TTM_LRU_RESOURCE, 69 /** @TTM_LRU_HITCH: The iterator hitch subclass */ 70 TTM_LRU_HITCH 71 }; 72 73 /** 74 * struct ttm_lru_item - The TTM lru list node base class 75 * @link: The list link 76 * @type: The subclass type 77 */ 78 struct ttm_lru_item { 79 struct list_head link; 80 enum ttm_lru_item_type type; 81 }; 82 83 /** 84 * ttm_lru_item_init() - initialize a struct ttm_lru_item 85 * @item: The item to initialize 86 * @type: The subclass type 87 */ 88 static inline void ttm_lru_item_init(struct ttm_lru_item *item, 89 enum ttm_lru_item_type type) 90 { 91 item->type = type; 92 INIT_LIST_HEAD(&item->link); 93 } 94 95 static inline bool ttm_lru_item_is_res(const struct ttm_lru_item *item) 96 { 97 return item->type == TTM_LRU_RESOURCE; 98 } 99 100 struct ttm_resource_manager_func { 101 /** 102 * struct ttm_resource_manager_func member alloc 103 * 104 * @man: Pointer to a memory type manager. 105 * @bo: Pointer to the buffer object we're allocating space for. 106 * @place: Placement details. 107 * @res: Resulting pointer to the ttm_resource. 108 * 109 * This function should allocate space in the memory type managed 110 * by @man. Placement details if applicable are given by @place. If 111 * successful, a filled in ttm_resource object should be returned in 112 * @res. @res::start should be set to a value identifying the beginning 113 * of the range allocated, and the function should return zero. 114 * If the manager can't fulfill the request -ENOSPC should be returned. 115 * If a system error occurred, preventing the request to be fulfilled, 116 * the function should return a negative error code. 117 * 118 * This function may not be called from within atomic context and needs 119 * to take care of its own locking to protect any data structures 120 * managing the space. 121 */ 122 int (*alloc)(struct ttm_resource_manager *man, 123 struct ttm_buffer_object *bo, 124 const struct ttm_place *place, 125 struct ttm_resource **res); 126 127 /** 128 * struct ttm_resource_manager_func member free 129 * 130 * @man: Pointer to a memory type manager. 131 * @res: Pointer to a struct ttm_resource to be freed. 132 * 133 * This function frees memory type resources previously allocated. 134 * May not be called from within atomic context. 135 */ 136 void (*free)(struct ttm_resource_manager *man, 137 struct ttm_resource *res); 138 139 /** 140 * struct ttm_resource_manager_func member intersects 141 * 142 * @man: Pointer to a memory type manager. 143 * @res: Pointer to a struct ttm_resource to be checked. 144 * @place: Placement to check against. 145 * @size: Size of the check. 146 * 147 * Test if @res intersects with @place + @size. Used to judge if 148 * evictions are valueable or not. 149 */ 150 bool (*intersects)(struct ttm_resource_manager *man, 151 struct ttm_resource *res, 152 const struct ttm_place *place, 153 size_t size); 154 155 /** 156 * struct ttm_resource_manager_func member compatible 157 * 158 * @man: Pointer to a memory type manager. 159 * @res: Pointer to a struct ttm_resource to be checked. 160 * @place: Placement to check against. 161 * @size: Size of the check. 162 * 163 * Test if @res compatible with @place + @size. Used to check of 164 * the need to move the backing store or not. 165 */ 166 bool (*compatible)(struct ttm_resource_manager *man, 167 struct ttm_resource *res, 168 const struct ttm_place *place, 169 size_t size); 170 171 /** 172 * struct ttm_resource_manager_func member debug 173 * 174 * @man: Pointer to a memory type manager. 175 * @printer: Prefix to be used in printout to identify the caller. 176 * 177 * This function is called to print out the state of the memory 178 * type manager to aid debugging of out-of-memory conditions. 179 * It may not be called from within atomic context. 180 */ 181 void (*debug)(struct ttm_resource_manager *man, 182 struct drm_printer *printer); 183 }; 184 185 /** 186 * struct ttm_resource_manager 187 * 188 * @use_type: The memory type is enabled. 189 * @use_tt: If a TT object should be used for the backing store. 190 * @size: Size of the managed region. 191 * @bdev: ttm device this manager belongs to 192 * @func: structure pointer implementing the range manager. See above 193 * @eviction_lock: lock for eviction fences 194 * @eviction_fences: The fences of the last pipelined move operation. 195 * @lru: The lru list for this memory type. 196 * 197 * This structure is used to identify and manage memory types for a device. 198 */ 199 struct ttm_resource_manager { 200 /* 201 * No protection. Constant from start. 202 */ 203 bool use_type; 204 bool use_tt; 205 struct ttm_device *bdev; 206 uint64_t size; 207 const struct ttm_resource_manager_func *func; 208 209 /* This is very similar to a dma_resv object, but locking rules make 210 * it difficult to use one in this context. 211 */ 212 spinlock_t eviction_lock; 213 struct dma_fence *eviction_fences[TTM_NUM_MOVE_FENCES]; 214 215 /* 216 * Protected by the bdev->lru_lock. 217 */ 218 struct list_head lru[TTM_MAX_BO_PRIORITY]; 219 220 /** 221 * @usage: How much of the resources are used, protected by the 222 * bdev->lru_lock. 223 */ 224 uint64_t usage; 225 226 /** 227 * @cg: &dmem_cgroup_region used for memory accounting, if not NULL. 228 */ 229 struct dmem_cgroup_region *cg; 230 }; 231 232 /** 233 * struct ttm_bus_placement 234 * 235 * @addr: mapped virtual address 236 * @offset: physical addr 237 * @is_iomem: is this io memory ? 238 * @caching: See enum ttm_caching 239 * 240 * Structure indicating the bus placement of an object. 241 */ 242 struct ttm_bus_placement { 243 void *addr; 244 phys_addr_t offset; 245 bool is_iomem; 246 enum ttm_caching caching; 247 }; 248 249 /** 250 * struct ttm_resource 251 * 252 * @start: Start of the allocation. 253 * @size: Actual size of resource in bytes. 254 * @mem_type: Resource type of the allocation. 255 * @placement: Placement flags. 256 * @bus: Placement on io bus accessible to the CPU 257 * @bo: weak reference to the BO, protected by ttm_device::lru_lock 258 * @css: cgroup state this resource is charged to 259 * 260 * Structure indicating the placement and space resources used by a 261 * buffer object. 262 */ 263 struct ttm_resource { 264 unsigned long start; 265 size_t size; 266 uint32_t mem_type; 267 uint32_t placement; 268 struct ttm_bus_placement bus; 269 struct ttm_buffer_object *bo; 270 271 struct dmem_cgroup_pool_state *css; 272 273 /** 274 * @lru: Least recently used list, see &ttm_resource_manager.lru 275 */ 276 struct ttm_lru_item lru; 277 }; 278 279 /** 280 * ttm_lru_item_to_res() - Downcast a struct ttm_lru_item to a struct ttm_resource 281 * @item: The struct ttm_lru_item to downcast 282 * 283 * Return: Pointer to the embedding struct ttm_resource 284 */ 285 static inline struct ttm_resource * 286 ttm_lru_item_to_res(struct ttm_lru_item *item) 287 { 288 return container_of(item, struct ttm_resource, lru); 289 } 290 291 /** 292 * struct ttm_lru_bulk_move_pos 293 * 294 * @first: first res in the bulk move range 295 * @last: last res in the bulk move range 296 * 297 * Range of resources for a lru bulk move. 298 */ 299 struct ttm_lru_bulk_move_pos { 300 struct ttm_resource *first; 301 struct ttm_resource *last; 302 }; 303 304 /** 305 * struct ttm_lru_bulk_move 306 * @pos: first/last lru entry for resources in the each domain/priority 307 * @cursor_list: The list of cursors currently traversing any of 308 * the sublists of @pos. Protected by the ttm device's lru_lock. 309 * 310 * Container for the current bulk move state. Should be used with 311 * ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move(). 312 * All BOs in a bulk_move structure need to share the same reservation object to 313 * ensure that the bulk as a whole is locked for eviction even if only one BO of 314 * the bulk is evicted. 315 */ 316 struct ttm_lru_bulk_move { 317 struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY]; 318 struct list_head cursor_list; 319 }; 320 321 /** 322 * struct ttm_resource_cursor 323 * @man: The resource manager currently being iterated over 324 * @hitch: A hitch list node inserted before the next resource 325 * to iterate over. 326 * @bulk_link: A list link for the list of cursors traversing the 327 * bulk sublist of @bulk. Protected by the ttm device's lru_lock. 328 * @bulk: Pointer to struct ttm_lru_bulk_move whose subrange @hitch is 329 * inserted to. NULL if none. Never dereference this pointer since 330 * the struct ttm_lru_bulk_move object pointed to might have been 331 * freed. The pointer is only for comparison. 332 * @mem_type: The memory type of the LRU list being traversed. 333 * This field is valid iff @bulk != NULL. 334 * @priority: the current priority 335 * 336 * Cursor to iterate over the resources in a manager. 337 */ 338 struct ttm_resource_cursor { 339 struct ttm_resource_manager *man; 340 struct ttm_lru_item hitch; 341 struct list_head bulk_link; 342 struct ttm_lru_bulk_move *bulk; 343 unsigned int mem_type; 344 unsigned int priority; 345 }; 346 347 void ttm_resource_cursor_init(struct ttm_resource_cursor *cursor, 348 struct ttm_resource_manager *man); 349 350 void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor); 351 352 /** 353 * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping + 354 * struct sg_table backed struct ttm_resource. 355 * @base: Embedded struct ttm_kmap_iter providing the usage interface. 356 * @iomap: struct io_mapping representing the underlying linear io_memory. 357 * @st: sg_table into @iomap, representing the memory of the struct ttm_resource. 358 * @start: Offset that needs to be subtracted from @st to make 359 * sg_dma_address(st->sgl) - @start == 0 for @iomap start. 360 * @cache: Scatterlist traversal cache for fast lookups. 361 * @cache.sg: Pointer to the currently cached scatterlist segment. 362 * @cache.i: First index of @sg. PAGE_SIZE granularity. 363 * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity. 364 * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity. 365 */ 366 struct ttm_kmap_iter_iomap { 367 struct ttm_kmap_iter base; 368 struct io_mapping *iomap; 369 struct sg_table *st; 370 resource_size_t start; 371 struct { 372 struct scatterlist *sg; 373 pgoff_t i; 374 pgoff_t end; 375 pgoff_t offs; 376 } cache; 377 }; 378 379 /** 380 * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io 381 * @base: The base iterator 382 * @dmap: Points to the starting address of the region 383 * @needs_unmap: Whether we need to unmap on fini 384 */ 385 struct ttm_kmap_iter_linear_io { 386 struct ttm_kmap_iter base; 387 struct iosys_map dmap; 388 bool needs_unmap; 389 }; 390 391 /** 392 * ttm_resource_manager_set_used 393 * 394 * @man: A memory manager object. 395 * @used: usage state to set. 396 * 397 * Set the manager in use flag. If disabled the manager is no longer 398 * used for object placement. 399 */ 400 static inline void 401 ttm_resource_manager_set_used(struct ttm_resource_manager *man, bool used) 402 { 403 int i; 404 405 for (i = 0; i < TTM_MAX_BO_PRIORITY; i++) 406 WARN_ON(!list_empty(&man->lru[i])); 407 man->use_type = used; 408 } 409 410 /** 411 * ttm_resource_manager_used 412 * 413 * @man: Manager to get used state for 414 * 415 * Get the in use flag for a manager. 416 * Returns: 417 * true is used, false if not. 418 */ 419 static inline bool ttm_resource_manager_used(struct ttm_resource_manager *man) 420 { 421 return man->use_type; 422 } 423 424 /** 425 * ttm_resource_manager_cleanup 426 * 427 * @man: A memory manager object. 428 * 429 * Cleanup the move fences from the memory manager object. 430 */ 431 static inline void 432 ttm_resource_manager_cleanup(struct ttm_resource_manager *man) 433 { 434 int i; 435 436 for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) { 437 dma_fence_put(man->eviction_fences[i]); 438 man->eviction_fences[i] = NULL; 439 } 440 } 441 442 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk); 443 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk); 444 void ttm_lru_bulk_move_fini(struct ttm_device *bdev, 445 struct ttm_lru_bulk_move *bulk); 446 447 void ttm_resource_add_bulk_move(struct ttm_resource *res, 448 struct ttm_buffer_object *bo); 449 void ttm_resource_del_bulk_move(struct ttm_resource *res, 450 struct ttm_buffer_object *bo); 451 void ttm_resource_move_to_lru_tail(struct ttm_resource *res); 452 453 void ttm_resource_init(struct ttm_buffer_object *bo, 454 const struct ttm_place *place, 455 struct ttm_resource *res); 456 void ttm_resource_fini(struct ttm_resource_manager *man, 457 struct ttm_resource *res); 458 459 int ttm_resource_alloc(struct ttm_buffer_object *bo, 460 const struct ttm_place *place, 461 struct ttm_resource **res, 462 struct dmem_cgroup_pool_state **ret_limit_pool); 463 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res); 464 bool ttm_resource_intersects(struct ttm_device *bdev, 465 struct ttm_resource *res, 466 const struct ttm_place *place, 467 size_t size); 468 bool ttm_resource_compatible(struct ttm_resource *res, 469 struct ttm_placement *placement, 470 bool evicting); 471 void ttm_resource_set_bo(struct ttm_resource *res, 472 struct ttm_buffer_object *bo); 473 474 void ttm_resource_manager_init(struct ttm_resource_manager *man, 475 struct ttm_device *bdev, 476 uint64_t size); 477 478 int ttm_resource_manager_evict_all(struct ttm_device *bdev, 479 struct ttm_resource_manager *man); 480 481 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man); 482 void ttm_resource_manager_debug(struct ttm_resource_manager *man, 483 struct drm_printer *p); 484 485 struct ttm_resource * 486 ttm_resource_manager_first(struct ttm_resource_cursor *cursor); 487 struct ttm_resource * 488 ttm_resource_manager_next(struct ttm_resource_cursor *cursor); 489 490 struct ttm_resource * 491 ttm_lru_first_res_or_null(struct list_head *head); 492 493 /** 494 * ttm_resource_manager_for_each_res - iterate over all resources 495 * @cursor: struct ttm_resource_cursor for the current position 496 * @res: the current resource 497 * 498 * Iterate over all the evictable resources in a resource manager. 499 */ 500 #define ttm_resource_manager_for_each_res(cursor, res) \ 501 for (res = ttm_resource_manager_first(cursor); res; \ 502 res = ttm_resource_manager_next(cursor)) 503 504 struct ttm_kmap_iter * 505 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io, 506 struct io_mapping *iomap, 507 struct sg_table *st, 508 resource_size_t start); 509 510 struct ttm_kmap_iter_linear_io; 511 512 struct ttm_kmap_iter * 513 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, 514 struct ttm_device *bdev, 515 struct ttm_resource *mem); 516 517 void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io, 518 struct ttm_device *bdev, 519 struct ttm_resource *mem); 520 521 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man, 522 struct dentry * parent, 523 const char *name); 524 #endif 525