1 /*
2 * Copyright 2020 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christian König
23 */
24
25 #ifndef _TTM_RESOURCE_H_
26 #define _TTM_RESOURCE_H_
27
28 #include <linux/types.h>
29 #include <linux/list.h>
30 #include <linux/mutex.h>
31 #include <linux/iosys-map.h>
32 #include <linux/dma-fence.h>
33
34 #include <drm/drm_print.h>
35 #include <drm/ttm/ttm_caching.h>
36 #include <drm/ttm/ttm_kmap_iter.h>
37
38 #define TTM_MAX_BO_PRIORITY 4U
39 #define TTM_NUM_MEM_TYPES 8
40
41 struct dmem_cgroup_device;
42 struct ttm_device;
43 struct ttm_resource_manager;
44 struct ttm_resource;
45 struct ttm_place;
46 struct ttm_buffer_object;
47 struct ttm_placement;
48 struct iosys_map;
49 struct io_mapping;
50 struct sg_table;
51 struct scatterlist;
52
53 /**
54 * enum ttm_lru_item_type - enumerate ttm_lru_item subclasses
55 */
56 enum ttm_lru_item_type {
57 /** @TTM_LRU_RESOURCE: The resource subclass */
58 TTM_LRU_RESOURCE,
59 /** @TTM_LRU_HITCH: The iterator hitch subclass */
60 TTM_LRU_HITCH
61 };
62
63 /**
64 * struct ttm_lru_item - The TTM lru list node base class
65 * @link: The list link
66 * @type: The subclass type
67 */
68 struct ttm_lru_item {
69 struct list_head link;
70 enum ttm_lru_item_type type;
71 };
72
73 /**
74 * ttm_lru_item_init() - initialize a struct ttm_lru_item
75 * @item: The item to initialize
76 * @type: The subclass type
77 */
ttm_lru_item_init(struct ttm_lru_item * item,enum ttm_lru_item_type type)78 static inline void ttm_lru_item_init(struct ttm_lru_item *item,
79 enum ttm_lru_item_type type)
80 {
81 item->type = type;
82 INIT_LIST_HEAD(&item->link);
83 }
84
ttm_lru_item_is_res(const struct ttm_lru_item * item)85 static inline bool ttm_lru_item_is_res(const struct ttm_lru_item *item)
86 {
87 return item->type == TTM_LRU_RESOURCE;
88 }
89
90 struct ttm_resource_manager_func {
91 /**
92 * struct ttm_resource_manager_func member alloc
93 *
94 * @man: Pointer to a memory type manager.
95 * @bo: Pointer to the buffer object we're allocating space for.
96 * @place: Placement details.
97 * @res: Resulting pointer to the ttm_resource.
98 *
99 * This function should allocate space in the memory type managed
100 * by @man. Placement details if applicable are given by @place. If
101 * successful, a filled in ttm_resource object should be returned in
102 * @res. @res::start should be set to a value identifying the beginning
103 * of the range allocated, and the function should return zero.
104 * If the manager can't fulfill the request -ENOSPC should be returned.
105 * If a system error occurred, preventing the request to be fulfilled,
106 * the function should return a negative error code.
107 *
108 * This function may not be called from within atomic context and needs
109 * to take care of its own locking to protect any data structures
110 * managing the space.
111 */
112 int (*alloc)(struct ttm_resource_manager *man,
113 struct ttm_buffer_object *bo,
114 const struct ttm_place *place,
115 struct ttm_resource **res);
116
117 /**
118 * struct ttm_resource_manager_func member free
119 *
120 * @man: Pointer to a memory type manager.
121 * @res: Pointer to a struct ttm_resource to be freed.
122 *
123 * This function frees memory type resources previously allocated.
124 * May not be called from within atomic context.
125 */
126 void (*free)(struct ttm_resource_manager *man,
127 struct ttm_resource *res);
128
129 /**
130 * struct ttm_resource_manager_func member intersects
131 *
132 * @man: Pointer to a memory type manager.
133 * @res: Pointer to a struct ttm_resource to be checked.
134 * @place: Placement to check against.
135 * @size: Size of the check.
136 *
137 * Test if @res intersects with @place + @size. Used to judge if
138 * evictions are valueable or not.
139 */
140 bool (*intersects)(struct ttm_resource_manager *man,
141 struct ttm_resource *res,
142 const struct ttm_place *place,
143 size_t size);
144
145 /**
146 * struct ttm_resource_manager_func member compatible
147 *
148 * @man: Pointer to a memory type manager.
149 * @res: Pointer to a struct ttm_resource to be checked.
150 * @place: Placement to check against.
151 * @size: Size of the check.
152 *
153 * Test if @res compatible with @place + @size. Used to check of
154 * the need to move the backing store or not.
155 */
156 bool (*compatible)(struct ttm_resource_manager *man,
157 struct ttm_resource *res,
158 const struct ttm_place *place,
159 size_t size);
160
161 /**
162 * struct ttm_resource_manager_func member debug
163 *
164 * @man: Pointer to a memory type manager.
165 * @printer: Prefix to be used in printout to identify the caller.
166 *
167 * This function is called to print out the state of the memory
168 * type manager to aid debugging of out-of-memory conditions.
169 * It may not be called from within atomic context.
170 */
171 void (*debug)(struct ttm_resource_manager *man,
172 struct drm_printer *printer);
173 };
174
175 /**
176 * struct ttm_resource_manager
177 *
178 * @use_type: The memory type is enabled.
179 * @use_tt: If a TT object should be used for the backing store.
180 * @size: Size of the managed region.
181 * @bdev: ttm device this manager belongs to
182 * @func: structure pointer implementing the range manager. See above
183 * @move_lock: lock for move fence
184 * @move: The fence of the last pipelined move operation.
185 * @lru: The lru list for this memory type.
186 *
187 * This structure is used to identify and manage memory types for a device.
188 */
189 struct ttm_resource_manager {
190 /*
191 * No protection. Constant from start.
192 */
193 bool use_type;
194 bool use_tt;
195 struct ttm_device *bdev;
196 uint64_t size;
197 const struct ttm_resource_manager_func *func;
198 spinlock_t move_lock;
199
200 /*
201 * Protected by @move_lock.
202 */
203 struct dma_fence *move;
204
205 /*
206 * Protected by the bdev->lru_lock.
207 */
208 struct list_head lru[TTM_MAX_BO_PRIORITY];
209
210 /**
211 * @usage: How much of the resources are used, protected by the
212 * bdev->lru_lock.
213 */
214 uint64_t usage;
215
216 /**
217 * @cg: &dmem_cgroup_region used for memory accounting, if not NULL.
218 */
219 struct dmem_cgroup_region *cg;
220 };
221
222 /**
223 * struct ttm_bus_placement
224 *
225 * @addr: mapped virtual address
226 * @offset: physical addr
227 * @is_iomem: is this io memory ?
228 * @caching: See enum ttm_caching
229 *
230 * Structure indicating the bus placement of an object.
231 */
232 struct ttm_bus_placement {
233 void *addr;
234 phys_addr_t offset;
235 bool is_iomem;
236 enum ttm_caching caching;
237 };
238
239 /**
240 * struct ttm_resource
241 *
242 * @start: Start of the allocation.
243 * @size: Actual size of resource in bytes.
244 * @mem_type: Resource type of the allocation.
245 * @placement: Placement flags.
246 * @bus: Placement on io bus accessible to the CPU
247 * @bo: weak reference to the BO, protected by ttm_device::lru_lock
248 * @css: cgroup state this resource is charged to
249 *
250 * Structure indicating the placement and space resources used by a
251 * buffer object.
252 */
253 struct ttm_resource {
254 unsigned long start;
255 size_t size;
256 uint32_t mem_type;
257 uint32_t placement;
258 struct ttm_bus_placement bus;
259 struct ttm_buffer_object *bo;
260
261 struct dmem_cgroup_pool_state *css;
262
263 /**
264 * @lru: Least recently used list, see &ttm_resource_manager.lru
265 */
266 struct ttm_lru_item lru;
267 };
268
269 /**
270 * ttm_lru_item_to_res() - Downcast a struct ttm_lru_item to a struct ttm_resource
271 * @item: The struct ttm_lru_item to downcast
272 *
273 * Return: Pointer to the embedding struct ttm_resource
274 */
275 static inline struct ttm_resource *
ttm_lru_item_to_res(struct ttm_lru_item * item)276 ttm_lru_item_to_res(struct ttm_lru_item *item)
277 {
278 return container_of(item, struct ttm_resource, lru);
279 }
280
281 /**
282 * struct ttm_lru_bulk_move_pos
283 *
284 * @first: first res in the bulk move range
285 * @last: last res in the bulk move range
286 *
287 * Range of resources for a lru bulk move.
288 */
289 struct ttm_lru_bulk_move_pos {
290 struct ttm_resource *first;
291 struct ttm_resource *last;
292 };
293
294 /**
295 * struct ttm_lru_bulk_move
296 * @pos: first/last lru entry for resources in the each domain/priority
297 * @cursor_list: The list of cursors currently traversing any of
298 * the sublists of @pos. Protected by the ttm device's lru_lock.
299 *
300 * Container for the current bulk move state. Should be used with
301 * ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move().
302 * All BOs in a bulk_move structure need to share the same reservation object to
303 * ensure that the bulk as a whole is locked for eviction even if only one BO of
304 * the bulk is evicted.
305 */
306 struct ttm_lru_bulk_move {
307 struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY];
308 struct list_head cursor_list;
309 };
310
311 /**
312 * struct ttm_resource_cursor
313 * @man: The resource manager currently being iterated over
314 * @hitch: A hitch list node inserted before the next resource
315 * to iterate over.
316 * @bulk_link: A list link for the list of cursors traversing the
317 * bulk sublist of @bulk. Protected by the ttm device's lru_lock.
318 * @bulk: Pointer to struct ttm_lru_bulk_move whose subrange @hitch is
319 * inserted to. NULL if none. Never dereference this pointer since
320 * the struct ttm_lru_bulk_move object pointed to might have been
321 * freed. The pointer is only for comparison.
322 * @mem_type: The memory type of the LRU list being traversed.
323 * This field is valid iff @bulk != NULL.
324 * @priority: the current priority
325 *
326 * Cursor to iterate over the resources in a manager.
327 */
328 struct ttm_resource_cursor {
329 struct ttm_resource_manager *man;
330 struct ttm_lru_item hitch;
331 struct list_head bulk_link;
332 struct ttm_lru_bulk_move *bulk;
333 unsigned int mem_type;
334 unsigned int priority;
335 };
336
337 void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor);
338
339 /**
340 * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping +
341 * struct sg_table backed struct ttm_resource.
342 * @base: Embedded struct ttm_kmap_iter providing the usage interface.
343 * @iomap: struct io_mapping representing the underlying linear io_memory.
344 * @st: sg_table into @iomap, representing the memory of the struct ttm_resource.
345 * @start: Offset that needs to be subtracted from @st to make
346 * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
347 * @cache: Scatterlist traversal cache for fast lookups.
348 * @cache.sg: Pointer to the currently cached scatterlist segment.
349 * @cache.i: First index of @sg. PAGE_SIZE granularity.
350 * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity.
351 * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity.
352 */
353 struct ttm_kmap_iter_iomap {
354 struct ttm_kmap_iter base;
355 struct io_mapping *iomap;
356 struct sg_table *st;
357 resource_size_t start;
358 struct {
359 struct scatterlist *sg;
360 pgoff_t i;
361 pgoff_t end;
362 pgoff_t offs;
363 } cache;
364 };
365
366 /**
367 * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io
368 * @base: The base iterator
369 * @dmap: Points to the starting address of the region
370 * @needs_unmap: Whether we need to unmap on fini
371 */
372 struct ttm_kmap_iter_linear_io {
373 struct ttm_kmap_iter base;
374 struct iosys_map dmap;
375 bool needs_unmap;
376 };
377
378 /**
379 * ttm_resource_manager_set_used
380 *
381 * @man: A memory manager object.
382 * @used: usage state to set.
383 *
384 * Set the manager in use flag. If disabled the manager is no longer
385 * used for object placement.
386 */
387 static inline void
ttm_resource_manager_set_used(struct ttm_resource_manager * man,bool used)388 ttm_resource_manager_set_used(struct ttm_resource_manager *man, bool used)
389 {
390 int i;
391
392 for (i = 0; i < TTM_MAX_BO_PRIORITY; i++)
393 WARN_ON(!list_empty(&man->lru[i]));
394 man->use_type = used;
395 }
396
397 /**
398 * ttm_resource_manager_used
399 *
400 * @man: Manager to get used state for
401 *
402 * Get the in use flag for a manager.
403 * Returns:
404 * true is used, false if not.
405 */
ttm_resource_manager_used(struct ttm_resource_manager * man)406 static inline bool ttm_resource_manager_used(struct ttm_resource_manager *man)
407 {
408 return man->use_type;
409 }
410
411 /**
412 * ttm_resource_manager_cleanup
413 *
414 * @man: A memory manager object.
415 *
416 * Cleanup the move fences from the memory manager object.
417 */
418 static inline void
ttm_resource_manager_cleanup(struct ttm_resource_manager * man)419 ttm_resource_manager_cleanup(struct ttm_resource_manager *man)
420 {
421 dma_fence_put(man->move);
422 man->move = NULL;
423 }
424
425 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk);
426 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk);
427 void ttm_lru_bulk_move_fini(struct ttm_device *bdev,
428 struct ttm_lru_bulk_move *bulk);
429
430 void ttm_resource_add_bulk_move(struct ttm_resource *res,
431 struct ttm_buffer_object *bo);
432 void ttm_resource_del_bulk_move(struct ttm_resource *res,
433 struct ttm_buffer_object *bo);
434 void ttm_resource_move_to_lru_tail(struct ttm_resource *res);
435
436 void ttm_resource_init(struct ttm_buffer_object *bo,
437 const struct ttm_place *place,
438 struct ttm_resource *res);
439 void ttm_resource_fini(struct ttm_resource_manager *man,
440 struct ttm_resource *res);
441
442 int ttm_resource_alloc(struct ttm_buffer_object *bo,
443 const struct ttm_place *place,
444 struct ttm_resource **res,
445 struct dmem_cgroup_pool_state **ret_limit_pool);
446 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res);
447 bool ttm_resource_intersects(struct ttm_device *bdev,
448 struct ttm_resource *res,
449 const struct ttm_place *place,
450 size_t size);
451 bool ttm_resource_compatible(struct ttm_resource *res,
452 struct ttm_placement *placement,
453 bool evicting);
454 void ttm_resource_set_bo(struct ttm_resource *res,
455 struct ttm_buffer_object *bo);
456
457 void ttm_resource_manager_init(struct ttm_resource_manager *man,
458 struct ttm_device *bdev,
459 uint64_t size);
460
461 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
462 struct ttm_resource_manager *man);
463
464 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man);
465 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
466 struct drm_printer *p);
467
468 struct ttm_resource *
469 ttm_resource_manager_first(struct ttm_resource_manager *man,
470 struct ttm_resource_cursor *cursor);
471 struct ttm_resource *
472 ttm_resource_manager_next(struct ttm_resource_cursor *cursor);
473
474 struct ttm_resource *
475 ttm_lru_first_res_or_null(struct list_head *head);
476
477 /**
478 * ttm_resource_manager_for_each_res - iterate over all resources
479 * @man: the resource manager
480 * @cursor: struct ttm_resource_cursor for the current position
481 * @res: the current resource
482 *
483 * Iterate over all the evictable resources in a resource manager.
484 */
485 #define ttm_resource_manager_for_each_res(man, cursor, res) \
486 for (res = ttm_resource_manager_first(man, cursor); res; \
487 res = ttm_resource_manager_next(cursor))
488
489 struct ttm_kmap_iter *
490 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
491 struct io_mapping *iomap,
492 struct sg_table *st,
493 resource_size_t start);
494
495 struct ttm_kmap_iter_linear_io;
496
497 struct ttm_kmap_iter *
498 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
499 struct ttm_device *bdev,
500 struct ttm_resource *mem);
501
502 void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
503 struct ttm_device *bdev,
504 struct ttm_resource *mem);
505
506 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
507 struct dentry * parent,
508 const char *name);
509 #endif
510