xref: /linux/drivers/gpu/drm/ttm/ttm_resource.c (revision 9c62fb62c9f0761eeda8f2a9517e007ff2cdbe9a)
1 /*
2  * Copyright 2020 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Christian König
23  */
24 
25 #include <linux/debugfs.h>
26 #include <linux/io-mapping.h>
27 #include <linux/iosys-map.h>
28 #include <linux/scatterlist.h>
29 
30 #include <drm/ttm/ttm_bo.h>
31 #include <drm/ttm/ttm_placement.h>
32 #include <drm/ttm/ttm_resource.h>
33 
34 #include <drm/drm_util.h>
35 
36 /**
37  * ttm_lru_bulk_move_init - initialize a bulk move structure
38  * @bulk: the structure to init
39  *
40  * For now just memset the structure to zero.
41  */
42 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)
43 {
44 	memset(bulk, 0, sizeof(*bulk));
45 }
46 EXPORT_SYMBOL(ttm_lru_bulk_move_init);
47 
48 /**
49  * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail.
50  *
51  * @bulk: bulk move structure
52  *
53  * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that
54  * resource order never changes. Should be called with &ttm_device.lru_lock held.
55  */
56 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
57 {
58 	unsigned i, j;
59 
60 	for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) {
61 		for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
62 			struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j];
63 			struct ttm_resource_manager *man;
64 
65 			if (!pos->first)
66 				continue;
67 
68 			lockdep_assert_held(&pos->first->bo->bdev->lru_lock);
69 			dma_resv_assert_held(pos->first->bo->base.resv);
70 			dma_resv_assert_held(pos->last->bo->base.resv);
71 
72 			man = ttm_manager_type(pos->first->bo->bdev, i);
73 			list_bulk_move_tail(&man->lru[j], &pos->first->lru.link,
74 					    &pos->last->lru.link);
75 		}
76 	}
77 }
78 EXPORT_SYMBOL(ttm_lru_bulk_move_tail);
79 
80 /* Return the bulk move pos object for this resource */
81 static struct ttm_lru_bulk_move_pos *
82 ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res)
83 {
84 	return &bulk->pos[res->mem_type][res->bo->priority];
85 }
86 
87 /* Return the previous resource on the list (skip over non-resource list items) */
88 static struct ttm_resource *ttm_lru_prev_res(struct ttm_resource *cur)
89 {
90 	struct ttm_lru_item *lru = &cur->lru;
91 
92 	do {
93 		lru = list_prev_entry(lru, link);
94 	} while (!ttm_lru_item_is_res(lru));
95 
96 	return ttm_lru_item_to_res(lru);
97 }
98 
99 /* Return the next resource on the list (skip over non-resource list items) */
100 static struct ttm_resource *ttm_lru_next_res(struct ttm_resource *cur)
101 {
102 	struct ttm_lru_item *lru = &cur->lru;
103 
104 	do {
105 		lru = list_next_entry(lru, link);
106 	} while (!ttm_lru_item_is_res(lru));
107 
108 	return ttm_lru_item_to_res(lru);
109 }
110 
111 /* Move the resource to the tail of the bulk move range */
112 static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos,
113 				       struct ttm_resource *res)
114 {
115 	if (pos->last != res) {
116 		if (pos->first == res)
117 			pos->first = ttm_lru_next_res(res);
118 		list_move(&res->lru.link, &pos->last->lru.link);
119 		pos->last = res;
120 	}
121 }
122 
123 /* Add the resource to a bulk_move cursor */
124 static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk,
125 				  struct ttm_resource *res)
126 {
127 	struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
128 
129 	if (!pos->first) {
130 		pos->first = res;
131 		pos->last = res;
132 	} else {
133 		WARN_ON(pos->first->bo->base.resv != res->bo->base.resv);
134 		ttm_lru_bulk_move_pos_tail(pos, res);
135 	}
136 }
137 
138 /* Remove the resource from a bulk_move range */
139 static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk,
140 				  struct ttm_resource *res)
141 {
142 	struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
143 
144 	if (unlikely(WARN_ON(!pos->first || !pos->last) ||
145 		     (pos->first == res && pos->last == res))) {
146 		pos->first = NULL;
147 		pos->last = NULL;
148 	} else if (pos->first == res) {
149 		pos->first = ttm_lru_next_res(res);
150 	} else if (pos->last == res) {
151 		pos->last = ttm_lru_prev_res(res);
152 	} else {
153 		list_move(&res->lru.link, &pos->last->lru.link);
154 	}
155 }
156 
157 /* Add the resource to a bulk move if the BO is configured for it */
158 void ttm_resource_add_bulk_move(struct ttm_resource *res,
159 				struct ttm_buffer_object *bo)
160 {
161 	if (bo->bulk_move && !bo->pin_count)
162 		ttm_lru_bulk_move_add(bo->bulk_move, res);
163 }
164 
165 /* Remove the resource from a bulk move if the BO is configured for it */
166 void ttm_resource_del_bulk_move(struct ttm_resource *res,
167 				struct ttm_buffer_object *bo)
168 {
169 	if (bo->bulk_move && !bo->pin_count)
170 		ttm_lru_bulk_move_del(bo->bulk_move, res);
171 }
172 
173 /* Move a resource to the LRU or bulk tail */
174 void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
175 {
176 	struct ttm_buffer_object *bo = res->bo;
177 	struct ttm_device *bdev = bo->bdev;
178 
179 	lockdep_assert_held(&bo->bdev->lru_lock);
180 
181 	if (bo->pin_count) {
182 		list_move_tail(&res->lru.link, &bdev->pinned);
183 
184 	} else	if (bo->bulk_move) {
185 		struct ttm_lru_bulk_move_pos *pos =
186 			ttm_lru_bulk_move_pos(bo->bulk_move, res);
187 
188 		ttm_lru_bulk_move_pos_tail(pos, res);
189 	} else {
190 		struct ttm_resource_manager *man;
191 
192 		man = ttm_manager_type(bdev, res->mem_type);
193 		list_move_tail(&res->lru.link, &man->lru[bo->priority]);
194 	}
195 }
196 
197 /**
198  * ttm_resource_init - resource object constructure
199  * @bo: buffer object this resources is allocated for
200  * @place: placement of the resource
201  * @res: the resource object to inistilize
202  *
203  * Initialize a new resource object. Counterpart of ttm_resource_fini().
204  */
205 void ttm_resource_init(struct ttm_buffer_object *bo,
206                        const struct ttm_place *place,
207                        struct ttm_resource *res)
208 {
209 	struct ttm_resource_manager *man;
210 
211 	res->start = 0;
212 	res->size = bo->base.size;
213 	res->mem_type = place->mem_type;
214 	res->placement = place->flags;
215 	res->bus.addr = NULL;
216 	res->bus.offset = 0;
217 	res->bus.is_iomem = false;
218 	res->bus.caching = ttm_cached;
219 	res->bo = bo;
220 
221 	man = ttm_manager_type(bo->bdev, place->mem_type);
222 	spin_lock(&bo->bdev->lru_lock);
223 	if (bo->pin_count)
224 		list_add_tail(&res->lru.link, &bo->bdev->pinned);
225 	else
226 		list_add_tail(&res->lru.link, &man->lru[bo->priority]);
227 	man->usage += res->size;
228 	spin_unlock(&bo->bdev->lru_lock);
229 }
230 EXPORT_SYMBOL(ttm_resource_init);
231 
232 /**
233  * ttm_resource_fini - resource destructor
234  * @man: the resource manager this resource belongs to
235  * @res: the resource to clean up
236  *
237  * Should be used by resource manager backends to clean up the TTM resource
238  * objects before freeing the underlying structure. Makes sure the resource is
239  * removed from the LRU before destruction.
240  * Counterpart of ttm_resource_init().
241  */
242 void ttm_resource_fini(struct ttm_resource_manager *man,
243 		       struct ttm_resource *res)
244 {
245 	struct ttm_device *bdev = man->bdev;
246 
247 	spin_lock(&bdev->lru_lock);
248 	list_del_init(&res->lru.link);
249 	man->usage -= res->size;
250 	spin_unlock(&bdev->lru_lock);
251 }
252 EXPORT_SYMBOL(ttm_resource_fini);
253 
254 int ttm_resource_alloc(struct ttm_buffer_object *bo,
255 		       const struct ttm_place *place,
256 		       struct ttm_resource **res_ptr)
257 {
258 	struct ttm_resource_manager *man =
259 		ttm_manager_type(bo->bdev, place->mem_type);
260 	int ret;
261 
262 	ret = man->func->alloc(man, bo, place, res_ptr);
263 	if (ret)
264 		return ret;
265 
266 	spin_lock(&bo->bdev->lru_lock);
267 	ttm_resource_add_bulk_move(*res_ptr, bo);
268 	spin_unlock(&bo->bdev->lru_lock);
269 	return 0;
270 }
271 EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_resource_alloc);
272 
273 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
274 {
275 	struct ttm_resource_manager *man;
276 
277 	if (!*res)
278 		return;
279 
280 	spin_lock(&bo->bdev->lru_lock);
281 	ttm_resource_del_bulk_move(*res, bo);
282 	spin_unlock(&bo->bdev->lru_lock);
283 	man = ttm_manager_type(bo->bdev, (*res)->mem_type);
284 	man->func->free(man, *res);
285 	*res = NULL;
286 }
287 EXPORT_SYMBOL(ttm_resource_free);
288 
289 /**
290  * ttm_resource_intersects - test for intersection
291  *
292  * @bdev: TTM device structure
293  * @res: The resource to test
294  * @place: The placement to test
295  * @size: How many bytes the new allocation needs.
296  *
297  * Test if @res intersects with @place and @size. Used for testing if evictions
298  * are valueable or not.
299  *
300  * Returns true if the res placement intersects with @place and @size.
301  */
302 bool ttm_resource_intersects(struct ttm_device *bdev,
303 			     struct ttm_resource *res,
304 			     const struct ttm_place *place,
305 			     size_t size)
306 {
307 	struct ttm_resource_manager *man;
308 
309 	if (!res)
310 		return false;
311 
312 	man = ttm_manager_type(bdev, res->mem_type);
313 	if (!place || !man->func->intersects)
314 		return true;
315 
316 	return man->func->intersects(man, res, place, size);
317 }
318 
319 /**
320  * ttm_resource_compatible - check if resource is compatible with placement
321  *
322  * @res: the resource to check
323  * @placement: the placement to check against
324  * @evicting: true if the caller is doing evictions
325  *
326  * Returns true if the placement is compatible.
327  */
328 bool ttm_resource_compatible(struct ttm_resource *res,
329 			     struct ttm_placement *placement,
330 			     bool evicting)
331 {
332 	struct ttm_buffer_object *bo = res->bo;
333 	struct ttm_device *bdev = bo->bdev;
334 	unsigned i;
335 
336 	if (res->placement & TTM_PL_FLAG_TEMPORARY)
337 		return false;
338 
339 	for (i = 0; i < placement->num_placement; i++) {
340 		const struct ttm_place *place = &placement->placement[i];
341 		struct ttm_resource_manager *man;
342 
343 		if (res->mem_type != place->mem_type)
344 			continue;
345 
346 		if (place->flags & (evicting ? TTM_PL_FLAG_DESIRED :
347 				    TTM_PL_FLAG_FALLBACK))
348 			continue;
349 
350 		if (place->flags & TTM_PL_FLAG_CONTIGUOUS &&
351 		    !(res->placement & TTM_PL_FLAG_CONTIGUOUS))
352 			continue;
353 
354 		man = ttm_manager_type(bdev, res->mem_type);
355 		if (man->func->compatible &&
356 		    !man->func->compatible(man, res, place, bo->base.size))
357 			continue;
358 
359 		return true;
360 	}
361 	return false;
362 }
363 
364 void ttm_resource_set_bo(struct ttm_resource *res,
365 			 struct ttm_buffer_object *bo)
366 {
367 	spin_lock(&bo->bdev->lru_lock);
368 	res->bo = bo;
369 	spin_unlock(&bo->bdev->lru_lock);
370 }
371 
372 /**
373  * ttm_resource_manager_init
374  *
375  * @man: memory manager object to init
376  * @bdev: ttm device this manager belongs to
377  * @size: size of managed resources in arbitrary units
378  *
379  * Initialise core parts of a manager object.
380  */
381 void ttm_resource_manager_init(struct ttm_resource_manager *man,
382 			       struct ttm_device *bdev,
383 			       uint64_t size)
384 {
385 	unsigned i;
386 
387 	spin_lock_init(&man->move_lock);
388 	man->bdev = bdev;
389 	man->size = size;
390 	man->usage = 0;
391 
392 	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
393 		INIT_LIST_HEAD(&man->lru[i]);
394 	man->move = NULL;
395 }
396 EXPORT_SYMBOL(ttm_resource_manager_init);
397 
398 /*
399  * ttm_resource_manager_evict_all
400  *
401  * @bdev - device to use
402  * @man - manager to use
403  *
404  * Evict all the objects out of a memory manager until it is empty.
405  * Part of memory manager cleanup sequence.
406  */
407 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
408 				   struct ttm_resource_manager *man)
409 {
410 	struct ttm_operation_ctx ctx = {
411 		.interruptible = false,
412 		.no_wait_gpu = false,
413 		.force_alloc = true
414 	};
415 	struct dma_fence *fence;
416 	int ret;
417 	unsigned i;
418 
419 	/*
420 	 * Can't use standard list traversal since we're unlocking.
421 	 */
422 
423 	spin_lock(&bdev->lru_lock);
424 	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
425 		while (!list_empty(&man->lru[i])) {
426 			spin_unlock(&bdev->lru_lock);
427 			ret = ttm_mem_evict_first(bdev, man, NULL, &ctx,
428 						  NULL);
429 			if (ret)
430 				return ret;
431 			spin_lock(&bdev->lru_lock);
432 		}
433 	}
434 	spin_unlock(&bdev->lru_lock);
435 
436 	spin_lock(&man->move_lock);
437 	fence = dma_fence_get(man->move);
438 	spin_unlock(&man->move_lock);
439 
440 	if (fence) {
441 		ret = dma_fence_wait(fence, false);
442 		dma_fence_put(fence);
443 		if (ret)
444 			return ret;
445 	}
446 
447 	return 0;
448 }
449 EXPORT_SYMBOL(ttm_resource_manager_evict_all);
450 
451 /**
452  * ttm_resource_manager_usage
453  *
454  * @man: A memory manager object.
455  *
456  * Return how many resources are currently used.
457  */
458 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man)
459 {
460 	uint64_t usage;
461 
462 	spin_lock(&man->bdev->lru_lock);
463 	usage = man->usage;
464 	spin_unlock(&man->bdev->lru_lock);
465 	return usage;
466 }
467 EXPORT_SYMBOL(ttm_resource_manager_usage);
468 
469 /**
470  * ttm_resource_manager_debug
471  *
472  * @man: manager type to dump.
473  * @p: printer to use for debug.
474  */
475 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
476 				struct drm_printer *p)
477 {
478 	drm_printf(p, "  use_type: %d\n", man->use_type);
479 	drm_printf(p, "  use_tt: %d\n", man->use_tt);
480 	drm_printf(p, "  size: %llu\n", man->size);
481 	drm_printf(p, "  usage: %llu\n", ttm_resource_manager_usage(man));
482 	if (man->func->debug)
483 		man->func->debug(man, p);
484 }
485 EXPORT_SYMBOL(ttm_resource_manager_debug);
486 
487 /**
488  * ttm_resource_manager_first
489  *
490  * @man: resource manager to iterate over
491  * @cursor: cursor to record the position
492  *
493  * Returns the first resource from the resource manager.
494  */
495 struct ttm_resource *
496 ttm_resource_manager_first(struct ttm_resource_manager *man,
497 			   struct ttm_resource_cursor *cursor)
498 {
499 	lockdep_assert_held(&man->bdev->lru_lock);
500 
501 	cursor->priority = 0;
502 	cursor->man = man;
503 	cursor->cur = &man->lru[cursor->priority];
504 	return ttm_resource_manager_next(cursor);
505 }
506 
507 /**
508  * ttm_resource_manager_next
509  *
510  * @cursor: cursor to record the position
511  *
512  * Return: the next resource from the resource manager.
513  */
514 struct ttm_resource *
515 ttm_resource_manager_next(struct ttm_resource_cursor *cursor)
516 {
517 	struct ttm_resource_manager *man = cursor->man;
518 	struct ttm_lru_item *lru;
519 
520 	lockdep_assert_held(&man->bdev->lru_lock);
521 
522 	for (;;) {
523 		lru = list_entry(cursor->cur, typeof(*lru), link);
524 		list_for_each_entry_continue(lru, &man->lru[cursor->priority], link) {
525 			if (ttm_lru_item_is_res(lru)) {
526 				cursor->cur = &lru->link;
527 				return ttm_lru_item_to_res(lru);
528 			}
529 		}
530 
531 		if (++cursor->priority >= TTM_MAX_BO_PRIORITY)
532 			break;
533 
534 		cursor->cur = &man->lru[cursor->priority];
535 	}
536 
537 	return NULL;
538 }
539 
540 /**
541  * ttm_lru_first_res_or_null() - Return the first resource on an lru list
542  * @head: The list head of the lru list.
543  *
544  * Return: Pointer to the first resource on the lru list or NULL if
545  * there is none.
546  */
547 struct ttm_resource *ttm_lru_first_res_or_null(struct list_head *head)
548 {
549 	struct ttm_lru_item *lru;
550 
551 	list_for_each_entry(lru, head, link) {
552 		if (ttm_lru_item_is_res(lru))
553 			return ttm_lru_item_to_res(lru);
554 	}
555 
556 	return NULL;
557 }
558 
559 static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
560 					  struct iosys_map *dmap,
561 					  pgoff_t i)
562 {
563 	struct ttm_kmap_iter_iomap *iter_io =
564 		container_of(iter, typeof(*iter_io), base);
565 	void __iomem *addr;
566 
567 retry:
568 	while (i >= iter_io->cache.end) {
569 		iter_io->cache.sg = iter_io->cache.sg ?
570 			sg_next(iter_io->cache.sg) : iter_io->st->sgl;
571 		iter_io->cache.i = iter_io->cache.end;
572 		iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
573 			PAGE_SHIFT;
574 		iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
575 			iter_io->start;
576 	}
577 
578 	if (i < iter_io->cache.i) {
579 		iter_io->cache.end = 0;
580 		iter_io->cache.sg = NULL;
581 		goto retry;
582 	}
583 
584 	addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
585 				       (((resource_size_t)i - iter_io->cache.i)
586 					<< PAGE_SHIFT));
587 	iosys_map_set_vaddr_iomem(dmap, addr);
588 }
589 
590 static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
591 					    struct iosys_map *map)
592 {
593 	io_mapping_unmap_local(map->vaddr_iomem);
594 }
595 
596 static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
597 	.map_local =  ttm_kmap_iter_iomap_map_local,
598 	.unmap_local = ttm_kmap_iter_iomap_unmap_local,
599 	.maps_tt = false,
600 };
601 
602 /**
603  * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
604  * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
605  * @iomap: The struct io_mapping representing the underlying linear io_memory.
606  * @st: sg_table into @iomap, representing the memory of the struct
607  * ttm_resource.
608  * @start: Offset that needs to be subtracted from @st to make
609  * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
610  *
611  * Return: Pointer to the embedded struct ttm_kmap_iter.
612  */
613 struct ttm_kmap_iter *
614 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
615 			 struct io_mapping *iomap,
616 			 struct sg_table *st,
617 			 resource_size_t start)
618 {
619 	iter_io->base.ops = &ttm_kmap_iter_io_ops;
620 	iter_io->iomap = iomap;
621 	iter_io->st = st;
622 	iter_io->start = start;
623 	memset(&iter_io->cache, 0, sizeof(iter_io->cache));
624 
625 	return &iter_io->base;
626 }
627 EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
628 
629 /**
630  * DOC: Linear io iterator
631  *
632  * This code should die in the not too near future. Best would be if we could
633  * make io-mapping use memremap for all io memory, and have memremap
634  * implement a kmap_local functionality. We could then strip a huge amount of
635  * code. These linear io iterators are implemented to mimic old functionality,
636  * and they don't use kmap_local semantics at all internally. Rather ioremap or
637  * friends, and at least on 32-bit they add global TLB flushes and points
638  * of failure.
639  */
640 
641 static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
642 					      struct iosys_map *dmap,
643 					      pgoff_t i)
644 {
645 	struct ttm_kmap_iter_linear_io *iter_io =
646 		container_of(iter, typeof(*iter_io), base);
647 
648 	*dmap = iter_io->dmap;
649 	iosys_map_incr(dmap, i * PAGE_SIZE);
650 }
651 
652 static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
653 	.map_local =  ttm_kmap_iter_linear_io_map_local,
654 	.maps_tt = false,
655 };
656 
657 /**
658  * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
659  * @iter_io: The iterator to initialize
660  * @bdev: The TTM device
661  * @mem: The ttm resource representing the iomap.
662  *
663  * This function is for internal TTM use only. It sets up a memcpy kmap iterator
664  * pointing at a linear chunk of io memory.
665  *
666  * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
667  * failure.
668  */
669 struct ttm_kmap_iter *
670 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
671 			     struct ttm_device *bdev,
672 			     struct ttm_resource *mem)
673 {
674 	int ret;
675 
676 	ret = ttm_mem_io_reserve(bdev, mem);
677 	if (ret)
678 		goto out_err;
679 	if (!mem->bus.is_iomem) {
680 		ret = -EINVAL;
681 		goto out_io_free;
682 	}
683 
684 	if (mem->bus.addr) {
685 		iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
686 		iter_io->needs_unmap = false;
687 	} else {
688 		iter_io->needs_unmap = true;
689 		memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
690 		if (mem->bus.caching == ttm_write_combined)
691 			iosys_map_set_vaddr_iomem(&iter_io->dmap,
692 						  ioremap_wc(mem->bus.offset,
693 							     mem->size));
694 		else if (mem->bus.caching == ttm_cached)
695 			iosys_map_set_vaddr(&iter_io->dmap,
696 					    memremap(mem->bus.offset, mem->size,
697 						     MEMREMAP_WB |
698 						     MEMREMAP_WT |
699 						     MEMREMAP_WC));
700 
701 		/* If uncached requested or if mapping cached or wc failed */
702 		if (iosys_map_is_null(&iter_io->dmap))
703 			iosys_map_set_vaddr_iomem(&iter_io->dmap,
704 						  ioremap(mem->bus.offset,
705 							  mem->size));
706 
707 		if (iosys_map_is_null(&iter_io->dmap)) {
708 			ret = -ENOMEM;
709 			goto out_io_free;
710 		}
711 	}
712 
713 	iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
714 	return &iter_io->base;
715 
716 out_io_free:
717 	ttm_mem_io_free(bdev, mem);
718 out_err:
719 	return ERR_PTR(ret);
720 }
721 
722 /**
723  * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
724  * @iter_io: The iterator to initialize
725  * @bdev: The TTM device
726  * @mem: The ttm resource representing the iomap.
727  *
728  * This function is for internal TTM use only. It cleans up a memcpy kmap
729  * iterator initialized by ttm_kmap_iter_linear_io_init.
730  */
731 void
732 ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
733 			     struct ttm_device *bdev,
734 			     struct ttm_resource *mem)
735 {
736 	if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) {
737 		if (iter_io->dmap.is_iomem)
738 			iounmap(iter_io->dmap.vaddr_iomem);
739 		else
740 			memunmap(iter_io->dmap.vaddr);
741 	}
742 
743 	ttm_mem_io_free(bdev, mem);
744 }
745 
746 #if defined(CONFIG_DEBUG_FS)
747 
748 static int ttm_resource_manager_show(struct seq_file *m, void *unused)
749 {
750 	struct ttm_resource_manager *man =
751 		(struct ttm_resource_manager *)m->private;
752 	struct drm_printer p = drm_seq_file_printer(m);
753 	ttm_resource_manager_debug(man, &p);
754 	return 0;
755 }
756 DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager);
757 
758 #endif
759 
760 /**
761  * ttm_resource_manager_create_debugfs - Create debugfs entry for specified
762  * resource manager.
763  * @man: The TTM resource manager for which the debugfs stats file be creates
764  * @parent: debugfs directory in which the file will reside
765  * @name: The filename to create.
766  *
767  * This function setups up a debugfs file that can be used to look
768  * at debug statistics of the specified ttm_resource_manager.
769  */
770 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
771 					 struct dentry * parent,
772 					 const char *name)
773 {
774 #if defined(CONFIG_DEBUG_FS)
775 	debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops);
776 #endif
777 }
778 EXPORT_SYMBOL(ttm_resource_manager_create_debugfs);
779