1 /* 2 * Copyright 2020 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Christian König 23 */ 24 25 #ifndef _TTM_RESOURCE_H_ 26 #define _TTM_RESOURCE_H_ 27 28 #include <linux/types.h> 29 #include <linux/list.h> 30 #include <linux/mutex.h> 31 #include <linux/iosys-map.h> 32 #include <linux/dma-fence.h> 33 34 #include <drm/ttm/ttm_caching.h> 35 #include <drm/ttm/ttm_kmap_iter.h> 36 37 #define TTM_MAX_BO_PRIORITY 4U 38 #define TTM_NUM_MEM_TYPES 9 39 40 struct dentry; 41 struct dmem_cgroup_device; 42 struct drm_printer; 43 struct ttm_device; 44 struct ttm_resource_manager; 45 struct ttm_resource; 46 struct ttm_place; 47 struct ttm_buffer_object; 48 struct ttm_placement; 49 struct iosys_map; 50 struct io_mapping; 51 struct sg_table; 52 struct scatterlist; 53 54 /** 55 * enum ttm_lru_item_type - enumerate ttm_lru_item subclasses 56 */ 57 enum ttm_lru_item_type { 58 /** @TTM_LRU_RESOURCE: The resource subclass */ 59 TTM_LRU_RESOURCE, 60 /** @TTM_LRU_HITCH: The iterator hitch subclass */ 61 TTM_LRU_HITCH 62 }; 63 64 /** 65 * struct ttm_lru_item - The TTM lru list node base class 66 * @link: The list link 67 * @type: The subclass type 68 */ 69 struct ttm_lru_item { 70 struct list_head link; 71 enum ttm_lru_item_type type; 72 }; 73 74 /** 75 * ttm_lru_item_init() - initialize a struct ttm_lru_item 76 * @item: The item to initialize 77 * @type: The subclass type 78 */ 79 static inline void ttm_lru_item_init(struct ttm_lru_item *item, 80 enum ttm_lru_item_type type) 81 { 82 item->type = type; 83 INIT_LIST_HEAD(&item->link); 84 } 85 86 static inline bool ttm_lru_item_is_res(const struct ttm_lru_item *item) 87 { 88 return item->type == TTM_LRU_RESOURCE; 89 } 90 91 struct ttm_resource_manager_func { 92 /** 93 * struct ttm_resource_manager_func member alloc 94 * 95 * @man: Pointer to a memory type manager. 96 * @bo: Pointer to the buffer object we're allocating space for. 97 * @place: Placement details. 98 * @res: Resulting pointer to the ttm_resource. 99 * 100 * This function should allocate space in the memory type managed 101 * by @man. Placement details if applicable are given by @place. If 102 * successful, a filled in ttm_resource object should be returned in 103 * @res. @res::start should be set to a value identifying the beginning 104 * of the range allocated, and the function should return zero. 105 * If the manager can't fulfill the request -ENOSPC should be returned. 106 * If a system error occurred, preventing the request to be fulfilled, 107 * the function should return a negative error code. 108 * 109 * This function may not be called from within atomic context and needs 110 * to take care of its own locking to protect any data structures 111 * managing the space. 112 */ 113 int (*alloc)(struct ttm_resource_manager *man, 114 struct ttm_buffer_object *bo, 115 const struct ttm_place *place, 116 struct ttm_resource **res); 117 118 /** 119 * struct ttm_resource_manager_func member free 120 * 121 * @man: Pointer to a memory type manager. 122 * @res: Pointer to a struct ttm_resource to be freed. 123 * 124 * This function frees memory type resources previously allocated. 125 * May not be called from within atomic context. 126 */ 127 void (*free)(struct ttm_resource_manager *man, 128 struct ttm_resource *res); 129 130 /** 131 * struct ttm_resource_manager_func member intersects 132 * 133 * @man: Pointer to a memory type manager. 134 * @res: Pointer to a struct ttm_resource to be checked. 135 * @place: Placement to check against. 136 * @size: Size of the check. 137 * 138 * Test if @res intersects with @place + @size. Used to judge if 139 * evictions are valueable or not. 140 */ 141 bool (*intersects)(struct ttm_resource_manager *man, 142 struct ttm_resource *res, 143 const struct ttm_place *place, 144 size_t size); 145 146 /** 147 * struct ttm_resource_manager_func member compatible 148 * 149 * @man: Pointer to a memory type manager. 150 * @res: Pointer to a struct ttm_resource to be checked. 151 * @place: Placement to check against. 152 * @size: Size of the check. 153 * 154 * Test if @res compatible with @place + @size. Used to check of 155 * the need to move the backing store or not. 156 */ 157 bool (*compatible)(struct ttm_resource_manager *man, 158 struct ttm_resource *res, 159 const struct ttm_place *place, 160 size_t size); 161 162 /** 163 * struct ttm_resource_manager_func member debug 164 * 165 * @man: Pointer to a memory type manager. 166 * @printer: Prefix to be used in printout to identify the caller. 167 * 168 * This function is called to print out the state of the memory 169 * type manager to aid debugging of out-of-memory conditions. 170 * It may not be called from within atomic context. 171 */ 172 void (*debug)(struct ttm_resource_manager *man, 173 struct drm_printer *printer); 174 }; 175 176 /** 177 * struct ttm_resource_manager 178 * 179 * @use_type: The memory type is enabled. 180 * @use_tt: If a TT object should be used for the backing store. 181 * @size: Size of the managed region. 182 * @bdev: ttm device this manager belongs to 183 * @func: structure pointer implementing the range manager. See above 184 * @move_lock: lock for move fence 185 * @move: The fence of the last pipelined move operation. 186 * @lru: The lru list for this memory type. 187 * 188 * This structure is used to identify and manage memory types for a device. 189 */ 190 struct ttm_resource_manager { 191 /* 192 * No protection. Constant from start. 193 */ 194 bool use_type; 195 bool use_tt; 196 struct ttm_device *bdev; 197 uint64_t size; 198 const struct ttm_resource_manager_func *func; 199 spinlock_t move_lock; 200 201 /* 202 * Protected by @move_lock. 203 */ 204 struct dma_fence *move; 205 206 /* 207 * Protected by the bdev->lru_lock. 208 */ 209 struct list_head lru[TTM_MAX_BO_PRIORITY]; 210 211 /** 212 * @usage: How much of the resources are used, protected by the 213 * bdev->lru_lock. 214 */ 215 uint64_t usage; 216 217 /** 218 * @cg: &dmem_cgroup_region used for memory accounting, if not NULL. 219 */ 220 struct dmem_cgroup_region *cg; 221 }; 222 223 /** 224 * struct ttm_bus_placement 225 * 226 * @addr: mapped virtual address 227 * @offset: physical addr 228 * @is_iomem: is this io memory ? 229 * @caching: See enum ttm_caching 230 * 231 * Structure indicating the bus placement of an object. 232 */ 233 struct ttm_bus_placement { 234 void *addr; 235 phys_addr_t offset; 236 bool is_iomem; 237 enum ttm_caching caching; 238 }; 239 240 /** 241 * struct ttm_resource 242 * 243 * @start: Start of the allocation. 244 * @size: Actual size of resource in bytes. 245 * @mem_type: Resource type of the allocation. 246 * @placement: Placement flags. 247 * @bus: Placement on io bus accessible to the CPU 248 * @bo: weak reference to the BO, protected by ttm_device::lru_lock 249 * @css: cgroup state this resource is charged to 250 * 251 * Structure indicating the placement and space resources used by a 252 * buffer object. 253 */ 254 struct ttm_resource { 255 unsigned long start; 256 size_t size; 257 uint32_t mem_type; 258 uint32_t placement; 259 struct ttm_bus_placement bus; 260 struct ttm_buffer_object *bo; 261 262 struct dmem_cgroup_pool_state *css; 263 264 /** 265 * @lru: Least recently used list, see &ttm_resource_manager.lru 266 */ 267 struct ttm_lru_item lru; 268 }; 269 270 /** 271 * ttm_lru_item_to_res() - Downcast a struct ttm_lru_item to a struct ttm_resource 272 * @item: The struct ttm_lru_item to downcast 273 * 274 * Return: Pointer to the embedding struct ttm_resource 275 */ 276 static inline struct ttm_resource * 277 ttm_lru_item_to_res(struct ttm_lru_item *item) 278 { 279 return container_of(item, struct ttm_resource, lru); 280 } 281 282 /** 283 * struct ttm_lru_bulk_move_pos 284 * 285 * @first: first res in the bulk move range 286 * @last: last res in the bulk move range 287 * 288 * Range of resources for a lru bulk move. 289 */ 290 struct ttm_lru_bulk_move_pos { 291 struct ttm_resource *first; 292 struct ttm_resource *last; 293 }; 294 295 /** 296 * struct ttm_lru_bulk_move 297 * @pos: first/last lru entry for resources in the each domain/priority 298 * @cursor_list: The list of cursors currently traversing any of 299 * the sublists of @pos. Protected by the ttm device's lru_lock. 300 * 301 * Container for the current bulk move state. Should be used with 302 * ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move(). 303 * All BOs in a bulk_move structure need to share the same reservation object to 304 * ensure that the bulk as a whole is locked for eviction even if only one BO of 305 * the bulk is evicted. 306 */ 307 struct ttm_lru_bulk_move { 308 struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY]; 309 struct list_head cursor_list; 310 }; 311 312 /** 313 * struct ttm_resource_cursor 314 * @man: The resource manager currently being iterated over 315 * @hitch: A hitch list node inserted before the next resource 316 * to iterate over. 317 * @bulk_link: A list link for the list of cursors traversing the 318 * bulk sublist of @bulk. Protected by the ttm device's lru_lock. 319 * @bulk: Pointer to struct ttm_lru_bulk_move whose subrange @hitch is 320 * inserted to. NULL if none. Never dereference this pointer since 321 * the struct ttm_lru_bulk_move object pointed to might have been 322 * freed. The pointer is only for comparison. 323 * @mem_type: The memory type of the LRU list being traversed. 324 * This field is valid iff @bulk != NULL. 325 * @priority: the current priority 326 * 327 * Cursor to iterate over the resources in a manager. 328 */ 329 struct ttm_resource_cursor { 330 struct ttm_resource_manager *man; 331 struct ttm_lru_item hitch; 332 struct list_head bulk_link; 333 struct ttm_lru_bulk_move *bulk; 334 unsigned int mem_type; 335 unsigned int priority; 336 }; 337 338 void ttm_resource_cursor_init(struct ttm_resource_cursor *cursor, 339 struct ttm_resource_manager *man); 340 341 void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor); 342 343 /** 344 * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping + 345 * struct sg_table backed struct ttm_resource. 346 * @base: Embedded struct ttm_kmap_iter providing the usage interface. 347 * @iomap: struct io_mapping representing the underlying linear io_memory. 348 * @st: sg_table into @iomap, representing the memory of the struct ttm_resource. 349 * @start: Offset that needs to be subtracted from @st to make 350 * sg_dma_address(st->sgl) - @start == 0 for @iomap start. 351 * @cache: Scatterlist traversal cache for fast lookups. 352 * @cache.sg: Pointer to the currently cached scatterlist segment. 353 * @cache.i: First index of @sg. PAGE_SIZE granularity. 354 * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity. 355 * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity. 356 */ 357 struct ttm_kmap_iter_iomap { 358 struct ttm_kmap_iter base; 359 struct io_mapping *iomap; 360 struct sg_table *st; 361 resource_size_t start; 362 struct { 363 struct scatterlist *sg; 364 pgoff_t i; 365 pgoff_t end; 366 pgoff_t offs; 367 } cache; 368 }; 369 370 /** 371 * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io 372 * @base: The base iterator 373 * @dmap: Points to the starting address of the region 374 * @needs_unmap: Whether we need to unmap on fini 375 */ 376 struct ttm_kmap_iter_linear_io { 377 struct ttm_kmap_iter base; 378 struct iosys_map dmap; 379 bool needs_unmap; 380 }; 381 382 /** 383 * ttm_resource_manager_set_used 384 * 385 * @man: A memory manager object. 386 * @used: usage state to set. 387 * 388 * Set the manager in use flag. If disabled the manager is no longer 389 * used for object placement. 390 */ 391 static inline void 392 ttm_resource_manager_set_used(struct ttm_resource_manager *man, bool used) 393 { 394 int i; 395 396 for (i = 0; i < TTM_MAX_BO_PRIORITY; i++) 397 WARN_ON(!list_empty(&man->lru[i])); 398 man->use_type = used; 399 } 400 401 /** 402 * ttm_resource_manager_used 403 * 404 * @man: Manager to get used state for 405 * 406 * Get the in use flag for a manager. 407 * Returns: 408 * true is used, false if not. 409 */ 410 static inline bool ttm_resource_manager_used(struct ttm_resource_manager *man) 411 { 412 return man->use_type; 413 } 414 415 /** 416 * ttm_resource_manager_cleanup 417 * 418 * @man: A memory manager object. 419 * 420 * Cleanup the move fences from the memory manager object. 421 */ 422 static inline void 423 ttm_resource_manager_cleanup(struct ttm_resource_manager *man) 424 { 425 dma_fence_put(man->move); 426 man->move = NULL; 427 } 428 429 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk); 430 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk); 431 void ttm_lru_bulk_move_fini(struct ttm_device *bdev, 432 struct ttm_lru_bulk_move *bulk); 433 434 void ttm_resource_add_bulk_move(struct ttm_resource *res, 435 struct ttm_buffer_object *bo); 436 void ttm_resource_del_bulk_move(struct ttm_resource *res, 437 struct ttm_buffer_object *bo); 438 void ttm_resource_move_to_lru_tail(struct ttm_resource *res); 439 440 void ttm_resource_init(struct ttm_buffer_object *bo, 441 const struct ttm_place *place, 442 struct ttm_resource *res); 443 void ttm_resource_fini(struct ttm_resource_manager *man, 444 struct ttm_resource *res); 445 446 int ttm_resource_alloc(struct ttm_buffer_object *bo, 447 const struct ttm_place *place, 448 struct ttm_resource **res, 449 struct dmem_cgroup_pool_state **ret_limit_pool); 450 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res); 451 bool ttm_resource_intersects(struct ttm_device *bdev, 452 struct ttm_resource *res, 453 const struct ttm_place *place, 454 size_t size); 455 bool ttm_resource_compatible(struct ttm_resource *res, 456 struct ttm_placement *placement, 457 bool evicting); 458 void ttm_resource_set_bo(struct ttm_resource *res, 459 struct ttm_buffer_object *bo); 460 461 void ttm_resource_manager_init(struct ttm_resource_manager *man, 462 struct ttm_device *bdev, 463 uint64_t size); 464 465 int ttm_resource_manager_evict_all(struct ttm_device *bdev, 466 struct ttm_resource_manager *man); 467 468 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man); 469 void ttm_resource_manager_debug(struct ttm_resource_manager *man, 470 struct drm_printer *p); 471 472 struct ttm_resource * 473 ttm_resource_manager_first(struct ttm_resource_cursor *cursor); 474 struct ttm_resource * 475 ttm_resource_manager_next(struct ttm_resource_cursor *cursor); 476 477 struct ttm_resource * 478 ttm_lru_first_res_or_null(struct list_head *head); 479 480 /** 481 * ttm_resource_manager_for_each_res - iterate over all resources 482 * @cursor: struct ttm_resource_cursor for the current position 483 * @res: the current resource 484 * 485 * Iterate over all the evictable resources in a resource manager. 486 */ 487 #define ttm_resource_manager_for_each_res(cursor, res) \ 488 for (res = ttm_resource_manager_first(cursor); res; \ 489 res = ttm_resource_manager_next(cursor)) 490 491 struct ttm_kmap_iter * 492 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io, 493 struct io_mapping *iomap, 494 struct sg_table *st, 495 resource_size_t start); 496 497 struct ttm_kmap_iter_linear_io; 498 499 struct ttm_kmap_iter * 500 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, 501 struct ttm_device *bdev, 502 struct ttm_resource *mem); 503 504 void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io, 505 struct ttm_device *bdev, 506 struct ttm_resource *mem); 507 508 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man, 509 struct dentry * parent, 510 const char *name); 511 #endif 512