xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c (revision e6f2a617ac53bc0753b885ffb94379ff48b2e2df)
1 /*
2  * Copyright 2009 Jerome Glisse.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sub license, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
20  *
21  * The above copyright notice and this permission notice (including the
22  * next paragraph) shall be included in all copies or substantial portions
23  * of the Software.
24  *
25  */
26 /*
27  * Authors:
28  *    Jerome Glisse <glisse@freedesktop.org>
29  *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
30  *    Dave Airlie
31  */
32 
33 #include <linux/dma-mapping.h>
34 #include <linux/iommu.h>
35 #include <linux/hmm.h>
36 #include <linux/pagemap.h>
37 #include <linux/sched/task.h>
38 #include <linux/sched/mm.h>
39 #include <linux/seq_file.h>
40 #include <linux/slab.h>
41 #include <linux/swap.h>
42 #include <linux/swiotlb.h>
43 #include <linux/dma-buf.h>
44 
45 #include <drm/ttm/ttm_bo_api.h>
46 #include <drm/ttm/ttm_bo_driver.h>
47 #include <drm/ttm/ttm_placement.h>
48 #include <drm/ttm/ttm_module.h>
49 #include <drm/ttm/ttm_page_alloc.h>
50 
51 #include <drm/drm_debugfs.h>
52 #include <drm/amdgpu_drm.h>
53 
54 #include "amdgpu.h"
55 #include "amdgpu_object.h"
56 #include "amdgpu_trace.h"
57 #include "amdgpu_amdkfd.h"
58 #include "amdgpu_sdma.h"
59 #include "amdgpu_ras.h"
60 #include "bif/bif_4_1_d.h"
61 
62 static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
63 			     struct ttm_mem_reg *mem, unsigned num_pages,
64 			     uint64_t offset, unsigned window,
65 			     struct amdgpu_ring *ring,
66 			     uint64_t *addr);
67 
68 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);
69 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev);
70 
71 static int amdgpu_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
72 {
73 	return 0;
74 }
75 
76 /**
77  * amdgpu_init_mem_type - Initialize a memory manager for a specific type of
78  * memory request.
79  *
80  * @bdev: The TTM BO device object (contains a reference to amdgpu_device)
81  * @type: The type of memory requested
82  * @man: The memory type manager for each domain
83  *
84  * This is called by ttm_bo_init_mm() when a buffer object is being
85  * initialized.
86  */
87 static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
88 				struct ttm_mem_type_manager *man)
89 {
90 	struct amdgpu_device *adev;
91 
92 	adev = amdgpu_ttm_adev(bdev);
93 
94 	switch (type) {
95 	case TTM_PL_SYSTEM:
96 		/* System memory */
97 		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
98 		man->available_caching = TTM_PL_MASK_CACHING;
99 		man->default_caching = TTM_PL_FLAG_CACHED;
100 		break;
101 	case TTM_PL_TT:
102 		/* GTT memory  */
103 		man->func = &amdgpu_gtt_mgr_func;
104 		man->gpu_offset = adev->gmc.gart_start;
105 		man->available_caching = TTM_PL_MASK_CACHING;
106 		man->default_caching = TTM_PL_FLAG_CACHED;
107 		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
108 		break;
109 	case TTM_PL_VRAM:
110 		/* "On-card" video ram */
111 		man->func = &amdgpu_vram_mgr_func;
112 		man->gpu_offset = adev->gmc.vram_start;
113 		man->flags = TTM_MEMTYPE_FLAG_FIXED |
114 			     TTM_MEMTYPE_FLAG_MAPPABLE;
115 		man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;
116 		man->default_caching = TTM_PL_FLAG_WC;
117 		break;
118 	case AMDGPU_PL_GDS:
119 	case AMDGPU_PL_GWS:
120 	case AMDGPU_PL_OA:
121 		/* On-chip GDS memory*/
122 		man->func = &ttm_bo_manager_func;
123 		man->gpu_offset = 0;
124 		man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_CMA;
125 		man->available_caching = TTM_PL_FLAG_UNCACHED;
126 		man->default_caching = TTM_PL_FLAG_UNCACHED;
127 		break;
128 	default:
129 		DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
130 		return -EINVAL;
131 	}
132 	return 0;
133 }
134 
135 /**
136  * amdgpu_evict_flags - Compute placement flags
137  *
138  * @bo: The buffer object to evict
139  * @placement: Possible destination(s) for evicted BO
140  *
141  * Fill in placement data when ttm_bo_evict() is called
142  */
143 static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
144 				struct ttm_placement *placement)
145 {
146 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
147 	struct amdgpu_bo *abo;
148 	static const struct ttm_place placements = {
149 		.fpfn = 0,
150 		.lpfn = 0,
151 		.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM
152 	};
153 
154 	/* Don't handle scatter gather BOs */
155 	if (bo->type == ttm_bo_type_sg) {
156 		placement->num_placement = 0;
157 		placement->num_busy_placement = 0;
158 		return;
159 	}
160 
161 	/* Object isn't an AMDGPU object so ignore */
162 	if (!amdgpu_bo_is_amdgpu_bo(bo)) {
163 		placement->placement = &placements;
164 		placement->busy_placement = &placements;
165 		placement->num_placement = 1;
166 		placement->num_busy_placement = 1;
167 		return;
168 	}
169 
170 	abo = ttm_to_amdgpu_bo(bo);
171 	switch (bo->mem.mem_type) {
172 	case AMDGPU_PL_GDS:
173 	case AMDGPU_PL_GWS:
174 	case AMDGPU_PL_OA:
175 		placement->num_placement = 0;
176 		placement->num_busy_placement = 0;
177 		return;
178 
179 	case TTM_PL_VRAM:
180 		if (!adev->mman.buffer_funcs_enabled) {
181 			/* Move to system memory */
182 			amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
183 		} else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
184 			   !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
185 			   amdgpu_bo_in_cpu_visible_vram(abo)) {
186 
187 			/* Try evicting to the CPU inaccessible part of VRAM
188 			 * first, but only set GTT as busy placement, so this
189 			 * BO will be evicted to GTT rather than causing other
190 			 * BOs to be evicted from VRAM
191 			 */
192 			amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
193 							 AMDGPU_GEM_DOMAIN_GTT);
194 			abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
195 			abo->placements[0].lpfn = 0;
196 			abo->placement.busy_placement = &abo->placements[1];
197 			abo->placement.num_busy_placement = 1;
198 		} else {
199 			/* Move to GTT memory */
200 			amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
201 		}
202 		break;
203 	case TTM_PL_TT:
204 	default:
205 		amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
206 		break;
207 	}
208 	*placement = abo->placement;
209 }
210 
211 /**
212  * amdgpu_verify_access - Verify access for a mmap call
213  *
214  * @bo:	The buffer object to map
215  * @filp: The file pointer from the process performing the mmap
216  *
217  * This is called by ttm_bo_mmap() to verify whether a process
218  * has the right to mmap a BO to their process space.
219  */
220 static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp)
221 {
222 	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
223 
224 	/*
225 	 * Don't verify access for KFD BOs. They don't have a GEM
226 	 * object associated with them.
227 	 */
228 	if (abo->kfd_bo)
229 		return 0;
230 
231 	if (amdgpu_ttm_tt_get_usermm(bo->ttm))
232 		return -EPERM;
233 	return drm_vma_node_verify_access(&abo->tbo.base.vma_node,
234 					  filp->private_data);
235 }
236 
237 /**
238  * amdgpu_move_null - Register memory for a buffer object
239  *
240  * @bo: The bo to assign the memory to
241  * @new_mem: The memory to be assigned.
242  *
243  * Assign the memory from new_mem to the memory of the buffer object bo.
244  */
245 static void amdgpu_move_null(struct ttm_buffer_object *bo,
246 			     struct ttm_mem_reg *new_mem)
247 {
248 	struct ttm_mem_reg *old_mem = &bo->mem;
249 
250 	BUG_ON(old_mem->mm_node != NULL);
251 	*old_mem = *new_mem;
252 	new_mem->mm_node = NULL;
253 }
254 
255 /**
256  * amdgpu_mm_node_addr - Compute the GPU relative offset of a GTT buffer.
257  *
258  * @bo: The bo to assign the memory to.
259  * @mm_node: Memory manager node for drm allocator.
260  * @mem: The region where the bo resides.
261  *
262  */
263 static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
264 				    struct drm_mm_node *mm_node,
265 				    struct ttm_mem_reg *mem)
266 {
267 	uint64_t addr = 0;
268 
269 	if (mm_node->start != AMDGPU_BO_INVALID_OFFSET) {
270 		addr = mm_node->start << PAGE_SHIFT;
271 		addr += bo->bdev->man[mem->mem_type].gpu_offset;
272 	}
273 	return addr;
274 }
275 
276 /**
277  * amdgpu_find_mm_node - Helper function finds the drm_mm_node corresponding to
278  * @offset. It also modifies the offset to be within the drm_mm_node returned
279  *
280  * @mem: The region where the bo resides.
281  * @offset: The offset that drm_mm_node is used for finding.
282  *
283  */
284 static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_mem_reg *mem,
285 					       unsigned long *offset)
286 {
287 	struct drm_mm_node *mm_node = mem->mm_node;
288 
289 	while (*offset >= (mm_node->size << PAGE_SHIFT)) {
290 		*offset -= (mm_node->size << PAGE_SHIFT);
291 		++mm_node;
292 	}
293 	return mm_node;
294 }
295 
296 /**
297  * amdgpu_copy_ttm_mem_to_mem - Helper function for copy
298  *
299  * The function copies @size bytes from {src->mem + src->offset} to
300  * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a
301  * move and different for a BO to BO copy.
302  *
303  * @f: Returns the last fence if multiple jobs are submitted.
304  */
305 int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
306 			       struct amdgpu_copy_mem *src,
307 			       struct amdgpu_copy_mem *dst,
308 			       uint64_t size,
309 			       struct dma_resv *resv,
310 			       struct dma_fence **f)
311 {
312 	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
313 	struct drm_mm_node *src_mm, *dst_mm;
314 	uint64_t src_node_start, dst_node_start, src_node_size,
315 		 dst_node_size, src_page_offset, dst_page_offset;
316 	struct dma_fence *fence = NULL;
317 	int r = 0;
318 	const uint64_t GTT_MAX_BYTES = (AMDGPU_GTT_MAX_TRANSFER_SIZE *
319 					AMDGPU_GPU_PAGE_SIZE);
320 
321 	if (!adev->mman.buffer_funcs_enabled) {
322 		DRM_ERROR("Trying to move memory with ring turned off.\n");
323 		return -EINVAL;
324 	}
325 
326 	src_mm = amdgpu_find_mm_node(src->mem, &src->offset);
327 	src_node_start = amdgpu_mm_node_addr(src->bo, src_mm, src->mem) +
328 					     src->offset;
329 	src_node_size = (src_mm->size << PAGE_SHIFT) - src->offset;
330 	src_page_offset = src_node_start & (PAGE_SIZE - 1);
331 
332 	dst_mm = amdgpu_find_mm_node(dst->mem, &dst->offset);
333 	dst_node_start = amdgpu_mm_node_addr(dst->bo, dst_mm, dst->mem) +
334 					     dst->offset;
335 	dst_node_size = (dst_mm->size << PAGE_SHIFT) - dst->offset;
336 	dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
337 
338 	mutex_lock(&adev->mman.gtt_window_lock);
339 
340 	while (size) {
341 		unsigned long cur_size;
342 		uint64_t from = src_node_start, to = dst_node_start;
343 		struct dma_fence *next;
344 
345 		/* Copy size cannot exceed GTT_MAX_BYTES. So if src or dst
346 		 * begins at an offset, then adjust the size accordingly
347 		 */
348 		cur_size = min3(min(src_node_size, dst_node_size), size,
349 				GTT_MAX_BYTES);
350 		if (cur_size + src_page_offset > GTT_MAX_BYTES ||
351 		    cur_size + dst_page_offset > GTT_MAX_BYTES)
352 			cur_size -= max(src_page_offset, dst_page_offset);
353 
354 		/* Map only what needs to be accessed. Map src to window 0 and
355 		 * dst to window 1
356 		 */
357 		if (src->mem->start == AMDGPU_BO_INVALID_OFFSET) {
358 			r = amdgpu_map_buffer(src->bo, src->mem,
359 					PFN_UP(cur_size + src_page_offset),
360 					src_node_start, 0, ring,
361 					&from);
362 			if (r)
363 				goto error;
364 			/* Adjust the offset because amdgpu_map_buffer returns
365 			 * start of mapped page
366 			 */
367 			from += src_page_offset;
368 		}
369 
370 		if (dst->mem->start == AMDGPU_BO_INVALID_OFFSET) {
371 			r = amdgpu_map_buffer(dst->bo, dst->mem,
372 					PFN_UP(cur_size + dst_page_offset),
373 					dst_node_start, 1, ring,
374 					&to);
375 			if (r)
376 				goto error;
377 			to += dst_page_offset;
378 		}
379 
380 		r = amdgpu_copy_buffer(ring, from, to, cur_size,
381 				       resv, &next, false, true);
382 		if (r)
383 			goto error;
384 
385 		dma_fence_put(fence);
386 		fence = next;
387 
388 		size -= cur_size;
389 		if (!size)
390 			break;
391 
392 		src_node_size -= cur_size;
393 		if (!src_node_size) {
394 			src_node_start = amdgpu_mm_node_addr(src->bo, ++src_mm,
395 							     src->mem);
396 			src_node_size = (src_mm->size << PAGE_SHIFT);
397 			src_page_offset = 0;
398 		} else {
399 			src_node_start += cur_size;
400 			src_page_offset = src_node_start & (PAGE_SIZE - 1);
401 		}
402 		dst_node_size -= cur_size;
403 		if (!dst_node_size) {
404 			dst_node_start = amdgpu_mm_node_addr(dst->bo, ++dst_mm,
405 							     dst->mem);
406 			dst_node_size = (dst_mm->size << PAGE_SHIFT);
407 			dst_page_offset = 0;
408 		} else {
409 			dst_node_start += cur_size;
410 			dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
411 		}
412 	}
413 error:
414 	mutex_unlock(&adev->mman.gtt_window_lock);
415 	if (f)
416 		*f = dma_fence_get(fence);
417 	dma_fence_put(fence);
418 	return r;
419 }
420 
421 /**
422  * amdgpu_move_blit - Copy an entire buffer to another buffer
423  *
424  * This is a helper called by amdgpu_bo_move() and amdgpu_move_vram_ram() to
425  * help move buffers to and from VRAM.
426  */
427 static int amdgpu_move_blit(struct ttm_buffer_object *bo,
428 			    bool evict, bool no_wait_gpu,
429 			    struct ttm_mem_reg *new_mem,
430 			    struct ttm_mem_reg *old_mem)
431 {
432 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
433 	struct amdgpu_copy_mem src, dst;
434 	struct dma_fence *fence = NULL;
435 	int r;
436 
437 	src.bo = bo;
438 	dst.bo = bo;
439 	src.mem = old_mem;
440 	dst.mem = new_mem;
441 	src.offset = 0;
442 	dst.offset = 0;
443 
444 	r = amdgpu_ttm_copy_mem_to_mem(adev, &src, &dst,
445 				       new_mem->num_pages << PAGE_SHIFT,
446 				       bo->base.resv, &fence);
447 	if (r)
448 		goto error;
449 
450 	/* clear the space being freed */
451 	if (old_mem->mem_type == TTM_PL_VRAM &&
452 	    (ttm_to_amdgpu_bo(bo)->flags &
453 	     AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) {
454 		struct dma_fence *wipe_fence = NULL;
455 
456 		r = amdgpu_fill_buffer(ttm_to_amdgpu_bo(bo), AMDGPU_POISON,
457 				       NULL, &wipe_fence);
458 		if (r) {
459 			goto error;
460 		} else if (wipe_fence) {
461 			dma_fence_put(fence);
462 			fence = wipe_fence;
463 		}
464 	}
465 
466 	/* Always block for VM page tables before committing the new location */
467 	if (bo->type == ttm_bo_type_kernel)
468 		r = ttm_bo_move_accel_cleanup(bo, fence, true, new_mem);
469 	else
470 		r = ttm_bo_pipeline_move(bo, fence, evict, new_mem);
471 	dma_fence_put(fence);
472 	return r;
473 
474 error:
475 	if (fence)
476 		dma_fence_wait(fence, false);
477 	dma_fence_put(fence);
478 	return r;
479 }
480 
481 /**
482  * amdgpu_move_vram_ram - Copy VRAM buffer to RAM buffer
483  *
484  * Called by amdgpu_bo_move().
485  */
486 static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict,
487 				struct ttm_operation_ctx *ctx,
488 				struct ttm_mem_reg *new_mem)
489 {
490 	struct ttm_mem_reg *old_mem = &bo->mem;
491 	struct ttm_mem_reg tmp_mem;
492 	struct ttm_place placements;
493 	struct ttm_placement placement;
494 	int r;
495 
496 	/* create space/pages for new_mem in GTT space */
497 	tmp_mem = *new_mem;
498 	tmp_mem.mm_node = NULL;
499 	placement.num_placement = 1;
500 	placement.placement = &placements;
501 	placement.num_busy_placement = 1;
502 	placement.busy_placement = &placements;
503 	placements.fpfn = 0;
504 	placements.lpfn = 0;
505 	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
506 	r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
507 	if (unlikely(r)) {
508 		pr_err("Failed to find GTT space for blit from VRAM\n");
509 		return r;
510 	}
511 
512 	/* set caching flags */
513 	r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);
514 	if (unlikely(r)) {
515 		goto out_cleanup;
516 	}
517 
518 	/* Bind the memory to the GTT space */
519 	r = ttm_tt_bind(bo->ttm, &tmp_mem, ctx);
520 	if (unlikely(r)) {
521 		goto out_cleanup;
522 	}
523 
524 	/* blit VRAM to GTT */
525 	r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, &tmp_mem, old_mem);
526 	if (unlikely(r)) {
527 		goto out_cleanup;
528 	}
529 
530 	/* move BO (in tmp_mem) to new_mem */
531 	r = ttm_bo_move_ttm(bo, ctx, new_mem);
532 out_cleanup:
533 	ttm_bo_mem_put(bo, &tmp_mem);
534 	return r;
535 }
536 
537 /**
538  * amdgpu_move_ram_vram - Copy buffer from RAM to VRAM
539  *
540  * Called by amdgpu_bo_move().
541  */
542 static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict,
543 				struct ttm_operation_ctx *ctx,
544 				struct ttm_mem_reg *new_mem)
545 {
546 	struct ttm_mem_reg *old_mem = &bo->mem;
547 	struct ttm_mem_reg tmp_mem;
548 	struct ttm_placement placement;
549 	struct ttm_place placements;
550 	int r;
551 
552 	/* make space in GTT for old_mem buffer */
553 	tmp_mem = *new_mem;
554 	tmp_mem.mm_node = NULL;
555 	placement.num_placement = 1;
556 	placement.placement = &placements;
557 	placement.num_busy_placement = 1;
558 	placement.busy_placement = &placements;
559 	placements.fpfn = 0;
560 	placements.lpfn = 0;
561 	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
562 	r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
563 	if (unlikely(r)) {
564 		pr_err("Failed to find GTT space for blit to VRAM\n");
565 		return r;
566 	}
567 
568 	/* move/bind old memory to GTT space */
569 	r = ttm_bo_move_ttm(bo, ctx, &tmp_mem);
570 	if (unlikely(r)) {
571 		goto out_cleanup;
572 	}
573 
574 	/* copy to VRAM */
575 	r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, new_mem, old_mem);
576 	if (unlikely(r)) {
577 		goto out_cleanup;
578 	}
579 out_cleanup:
580 	ttm_bo_mem_put(bo, &tmp_mem);
581 	return r;
582 }
583 
584 /**
585  * amdgpu_mem_visible - Check that memory can be accessed by ttm_bo_move_memcpy
586  *
587  * Called by amdgpu_bo_move()
588  */
589 static bool amdgpu_mem_visible(struct amdgpu_device *adev,
590 			       struct ttm_mem_reg *mem)
591 {
592 	struct drm_mm_node *nodes = mem->mm_node;
593 
594 	if (mem->mem_type == TTM_PL_SYSTEM ||
595 	    mem->mem_type == TTM_PL_TT)
596 		return true;
597 	if (mem->mem_type != TTM_PL_VRAM)
598 		return false;
599 
600 	/* ttm_mem_reg_ioremap only supports contiguous memory */
601 	if (nodes->size != mem->num_pages)
602 		return false;
603 
604 	return ((nodes->start + nodes->size) << PAGE_SHIFT)
605 		<= adev->gmc.visible_vram_size;
606 }
607 
608 /**
609  * amdgpu_bo_move - Move a buffer object to a new memory location
610  *
611  * Called by ttm_bo_handle_move_mem()
612  */
613 static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
614 			  struct ttm_operation_ctx *ctx,
615 			  struct ttm_mem_reg *new_mem)
616 {
617 	struct amdgpu_device *adev;
618 	struct amdgpu_bo *abo;
619 	struct ttm_mem_reg *old_mem = &bo->mem;
620 	int r;
621 
622 	/* Can't move a pinned BO */
623 	abo = ttm_to_amdgpu_bo(bo);
624 	if (WARN_ON_ONCE(abo->pin_count > 0))
625 		return -EINVAL;
626 
627 	adev = amdgpu_ttm_adev(bo->bdev);
628 
629 	if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
630 		amdgpu_move_null(bo, new_mem);
631 		return 0;
632 	}
633 	if ((old_mem->mem_type == TTM_PL_TT &&
634 	     new_mem->mem_type == TTM_PL_SYSTEM) ||
635 	    (old_mem->mem_type == TTM_PL_SYSTEM &&
636 	     new_mem->mem_type == TTM_PL_TT)) {
637 		/* bind is enough */
638 		amdgpu_move_null(bo, new_mem);
639 		return 0;
640 	}
641 	if (old_mem->mem_type == AMDGPU_PL_GDS ||
642 	    old_mem->mem_type == AMDGPU_PL_GWS ||
643 	    old_mem->mem_type == AMDGPU_PL_OA ||
644 	    new_mem->mem_type == AMDGPU_PL_GDS ||
645 	    new_mem->mem_type == AMDGPU_PL_GWS ||
646 	    new_mem->mem_type == AMDGPU_PL_OA) {
647 		/* Nothing to save here */
648 		amdgpu_move_null(bo, new_mem);
649 		return 0;
650 	}
651 
652 	if (!adev->mman.buffer_funcs_enabled) {
653 		r = -ENODEV;
654 		goto memcpy;
655 	}
656 
657 	if (old_mem->mem_type == TTM_PL_VRAM &&
658 	    new_mem->mem_type == TTM_PL_SYSTEM) {
659 		r = amdgpu_move_vram_ram(bo, evict, ctx, new_mem);
660 	} else if (old_mem->mem_type == TTM_PL_SYSTEM &&
661 		   new_mem->mem_type == TTM_PL_VRAM) {
662 		r = amdgpu_move_ram_vram(bo, evict, ctx, new_mem);
663 	} else {
664 		r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu,
665 				     new_mem, old_mem);
666 	}
667 
668 	if (r) {
669 memcpy:
670 		/* Check that all memory is CPU accessible */
671 		if (!amdgpu_mem_visible(adev, old_mem) ||
672 		    !amdgpu_mem_visible(adev, new_mem)) {
673 			pr_err("Move buffer fallback to memcpy unavailable\n");
674 			return r;
675 		}
676 
677 		r = ttm_bo_move_memcpy(bo, ctx, new_mem);
678 		if (r)
679 			return r;
680 	}
681 
682 	if (bo->type == ttm_bo_type_device &&
683 	    new_mem->mem_type == TTM_PL_VRAM &&
684 	    old_mem->mem_type != TTM_PL_VRAM) {
685 		/* amdgpu_bo_fault_reserve_notify will re-set this if the CPU
686 		 * accesses the BO after it's moved.
687 		 */
688 		abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
689 	}
690 
691 	/* update statistics */
692 	atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved);
693 	return 0;
694 }
695 
696 /**
697  * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault
698  *
699  * Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault()
700  */
701 static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
702 {
703 	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
704 	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
705 	struct drm_mm_node *mm_node = mem->mm_node;
706 
707 	mem->bus.addr = NULL;
708 	mem->bus.offset = 0;
709 	mem->bus.size = mem->num_pages << PAGE_SHIFT;
710 	mem->bus.base = 0;
711 	mem->bus.is_iomem = false;
712 	if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
713 		return -EINVAL;
714 	switch (mem->mem_type) {
715 	case TTM_PL_SYSTEM:
716 		/* system memory */
717 		return 0;
718 	case TTM_PL_TT:
719 		break;
720 	case TTM_PL_VRAM:
721 		mem->bus.offset = mem->start << PAGE_SHIFT;
722 		/* check if it's visible */
723 		if ((mem->bus.offset + mem->bus.size) > adev->gmc.visible_vram_size)
724 			return -EINVAL;
725 		/* Only physically contiguous buffers apply. In a contiguous
726 		 * buffer, size of the first mm_node would match the number of
727 		 * pages in ttm_mem_reg.
728 		 */
729 		if (adev->mman.aper_base_kaddr &&
730 		    (mm_node->size == mem->num_pages))
731 			mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr +
732 					mem->bus.offset;
733 
734 		mem->bus.base = adev->gmc.aper_base;
735 		mem->bus.is_iomem = true;
736 		break;
737 	default:
738 		return -EINVAL;
739 	}
740 	return 0;
741 }
742 
743 static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
744 {
745 }
746 
747 static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
748 					   unsigned long page_offset)
749 {
750 	struct drm_mm_node *mm;
751 	unsigned long offset = (page_offset << PAGE_SHIFT);
752 
753 	mm = amdgpu_find_mm_node(&bo->mem, &offset);
754 	return (bo->mem.bus.base >> PAGE_SHIFT) + mm->start +
755 		(offset >> PAGE_SHIFT);
756 }
757 
758 /*
759  * TTM backend functions.
760  */
761 struct amdgpu_ttm_tt {
762 	struct ttm_dma_tt	ttm;
763 	struct drm_gem_object	*gobj;
764 	u64			offset;
765 	uint64_t		userptr;
766 	struct task_struct	*usertask;
767 	uint32_t		userflags;
768 #if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
769 	struct hmm_range	*range;
770 #endif
771 };
772 
773 #ifdef CONFIG_DRM_AMDGPU_USERPTR
774 /* flags used by HMM internal, not related to CPU/GPU PTE flags */
775 static const uint64_t hmm_range_flags[HMM_PFN_FLAG_MAX] = {
776 	(1 << 0), /* HMM_PFN_VALID */
777 	(1 << 1), /* HMM_PFN_WRITE */
778 	0 /* HMM_PFN_DEVICE_PRIVATE */
779 };
780 
781 static const uint64_t hmm_range_values[HMM_PFN_VALUE_MAX] = {
782 	0xfffffffffffffffeUL, /* HMM_PFN_ERROR */
783 	0, /* HMM_PFN_NONE */
784 	0xfffffffffffffffcUL /* HMM_PFN_SPECIAL */
785 };
786 
787 /**
788  * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user
789  * memory and start HMM tracking CPU page table update
790  *
791  * Calling function must call amdgpu_ttm_tt_userptr_range_done() once and only
792  * once afterwards to stop HMM tracking
793  */
794 int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages)
795 {
796 	struct ttm_tt *ttm = bo->tbo.ttm;
797 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
798 	unsigned long start = gtt->userptr;
799 	struct vm_area_struct *vma;
800 	struct hmm_range *range;
801 	unsigned long timeout;
802 	struct mm_struct *mm;
803 	unsigned long i;
804 	int r = 0;
805 
806 	mm = bo->notifier.mm;
807 	if (unlikely(!mm)) {
808 		DRM_DEBUG_DRIVER("BO is not registered?\n");
809 		return -EFAULT;
810 	}
811 
812 	/* Another get_user_pages is running at the same time?? */
813 	if (WARN_ON(gtt->range))
814 		return -EFAULT;
815 
816 	if (!mmget_not_zero(mm)) /* Happens during process shutdown */
817 		return -ESRCH;
818 
819 	range = kzalloc(sizeof(*range), GFP_KERNEL);
820 	if (unlikely(!range)) {
821 		r = -ENOMEM;
822 		goto out;
823 	}
824 	range->notifier = &bo->notifier;
825 	range->flags = hmm_range_flags;
826 	range->values = hmm_range_values;
827 	range->pfn_shift = PAGE_SHIFT;
828 	range->start = bo->notifier.interval_tree.start;
829 	range->end = bo->notifier.interval_tree.last + 1;
830 	range->default_flags = hmm_range_flags[HMM_PFN_VALID];
831 	if (!amdgpu_ttm_tt_is_readonly(ttm))
832 		range->default_flags |= range->flags[HMM_PFN_WRITE];
833 
834 	range->pfns = kvmalloc_array(ttm->num_pages, sizeof(*range->pfns),
835 				     GFP_KERNEL);
836 	if (unlikely(!range->pfns)) {
837 		r = -ENOMEM;
838 		goto out_free_ranges;
839 	}
840 
841 	down_read(&mm->mmap_sem);
842 	vma = find_vma(mm, start);
843 	if (unlikely(!vma || start < vma->vm_start)) {
844 		r = -EFAULT;
845 		goto out_unlock;
846 	}
847 	if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) &&
848 		vma->vm_file)) {
849 		r = -EPERM;
850 		goto out_unlock;
851 	}
852 	up_read(&mm->mmap_sem);
853 	timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
854 
855 retry:
856 	range->notifier_seq = mmu_interval_read_begin(&bo->notifier);
857 
858 	down_read(&mm->mmap_sem);
859 	r = hmm_range_fault(range, 0);
860 	up_read(&mm->mmap_sem);
861 	if (unlikely(r <= 0)) {
862 		/*
863 		 * FIXME: This timeout should encompass the retry from
864 		 * mmu_interval_read_retry() as well.
865 		 */
866 		if ((r == 0 || r == -EBUSY) && !time_after(jiffies, timeout))
867 			goto retry;
868 		goto out_free_pfns;
869 	}
870 
871 	for (i = 0; i < ttm->num_pages; i++) {
872 		/* FIXME: The pages cannot be touched outside the notifier_lock */
873 		pages[i] = hmm_device_entry_to_page(range, range->pfns[i]);
874 		if (unlikely(!pages[i])) {
875 			pr_err("Page fault failed for pfn[%lu] = 0x%llx\n",
876 			       i, range->pfns[i]);
877 			r = -ENOMEM;
878 
879 			goto out_free_pfns;
880 		}
881 	}
882 
883 	gtt->range = range;
884 	mmput(mm);
885 
886 	return 0;
887 
888 out_unlock:
889 	up_read(&mm->mmap_sem);
890 out_free_pfns:
891 	kvfree(range->pfns);
892 out_free_ranges:
893 	kfree(range);
894 out:
895 	mmput(mm);
896 	return r;
897 }
898 
899 /**
900  * amdgpu_ttm_tt_userptr_range_done - stop HMM track the CPU page table change
901  * Check if the pages backing this ttm range have been invalidated
902  *
903  * Returns: true if pages are still valid
904  */
905 bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm)
906 {
907 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
908 	bool r = false;
909 
910 	if (!gtt || !gtt->userptr)
911 		return false;
912 
913 	DRM_DEBUG_DRIVER("user_pages_done 0x%llx pages 0x%lx\n",
914 		gtt->userptr, ttm->num_pages);
915 
916 	WARN_ONCE(!gtt->range || !gtt->range->pfns,
917 		"No user pages to check\n");
918 
919 	if (gtt->range) {
920 		/*
921 		 * FIXME: Must always hold notifier_lock for this, and must
922 		 * not ignore the return code.
923 		 */
924 		r = mmu_interval_read_retry(gtt->range->notifier,
925 					 gtt->range->notifier_seq);
926 		kvfree(gtt->range->pfns);
927 		kfree(gtt->range);
928 		gtt->range = NULL;
929 	}
930 
931 	return !r;
932 }
933 #endif
934 
935 /**
936  * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
937  *
938  * Called by amdgpu_cs_list_validate(). This creates the page list
939  * that backs user memory and will ultimately be mapped into the device
940  * address space.
941  */
942 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
943 {
944 	unsigned long i;
945 
946 	for (i = 0; i < ttm->num_pages; ++i)
947 		ttm->pages[i] = pages ? pages[i] : NULL;
948 }
949 
950 /**
951  * amdgpu_ttm_tt_pin_userptr - 	prepare the sg table with the user pages
952  *
953  * Called by amdgpu_ttm_backend_bind()
954  **/
955 static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
956 {
957 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
958 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
959 	unsigned nents;
960 	int r;
961 
962 	int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
963 	enum dma_data_direction direction = write ?
964 		DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
965 
966 	/* Allocate an SG array and squash pages into it */
967 	r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,
968 				      ttm->num_pages << PAGE_SHIFT,
969 				      GFP_KERNEL);
970 	if (r)
971 		goto release_sg;
972 
973 	/* Map SG to device */
974 	r = -ENOMEM;
975 	nents = dma_map_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
976 	if (nents != ttm->sg->nents)
977 		goto release_sg;
978 
979 	/* convert SG to linear array of pages and dma addresses */
980 	drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
981 					 gtt->ttm.dma_address, ttm->num_pages);
982 
983 	return 0;
984 
985 release_sg:
986 	kfree(ttm->sg);
987 	return r;
988 }
989 
990 /**
991  * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages
992  */
993 static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
994 {
995 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
996 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
997 
998 	int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
999 	enum dma_data_direction direction = write ?
1000 		DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
1001 
1002 	/* double check that we don't free the table twice */
1003 	if (!ttm->sg->sgl)
1004 		return;
1005 
1006 	/* unmap the pages mapped to the device */
1007 	dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
1008 
1009 	sg_free_table(ttm->sg);
1010 
1011 #if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
1012 	if (gtt->range) {
1013 		unsigned long i;
1014 
1015 		for (i = 0; i < ttm->num_pages; i++) {
1016 			if (ttm->pages[i] !=
1017 				hmm_device_entry_to_page(gtt->range,
1018 					      gtt->range->pfns[i]))
1019 				break;
1020 		}
1021 
1022 		WARN((i == ttm->num_pages), "Missing get_user_page_done\n");
1023 	}
1024 #endif
1025 }
1026 
1027 int amdgpu_ttm_gart_bind(struct amdgpu_device *adev,
1028 				struct ttm_buffer_object *tbo,
1029 				uint64_t flags)
1030 {
1031 	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo);
1032 	struct ttm_tt *ttm = tbo->ttm;
1033 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
1034 	int r;
1035 
1036 	if (abo->flags & AMDGPU_GEM_CREATE_MQD_GFX9) {
1037 		uint64_t page_idx = 1;
1038 
1039 		r = amdgpu_gart_bind(adev, gtt->offset, page_idx,
1040 				ttm->pages, gtt->ttm.dma_address, flags);
1041 		if (r)
1042 			goto gart_bind_fail;
1043 
1044 		/* Patch mtype of the second part BO */
1045 		flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1046 		flags |= AMDGPU_PTE_MTYPE_VG10(AMDGPU_MTYPE_NC);
1047 
1048 		r = amdgpu_gart_bind(adev,
1049 				gtt->offset + (page_idx << PAGE_SHIFT),
1050 				ttm->num_pages - page_idx,
1051 				&ttm->pages[page_idx],
1052 				&(gtt->ttm.dma_address[page_idx]), flags);
1053 	} else {
1054 		r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
1055 				     ttm->pages, gtt->ttm.dma_address, flags);
1056 	}
1057 
1058 gart_bind_fail:
1059 	if (r)
1060 		DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
1061 			  ttm->num_pages, gtt->offset);
1062 
1063 	return r;
1064 }
1065 
1066 /**
1067  * amdgpu_ttm_backend_bind - Bind GTT memory
1068  *
1069  * Called by ttm_tt_bind() on behalf of ttm_bo_handle_move_mem().
1070  * This handles binding GTT memory to the device address space.
1071  */
1072 static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
1073 				   struct ttm_mem_reg *bo_mem)
1074 {
1075 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
1076 	struct amdgpu_ttm_tt *gtt = (void*)ttm;
1077 	uint64_t flags;
1078 	int r = 0;
1079 
1080 	if (gtt->userptr) {
1081 		r = amdgpu_ttm_tt_pin_userptr(ttm);
1082 		if (r) {
1083 			DRM_ERROR("failed to pin userptr\n");
1084 			return r;
1085 		}
1086 	}
1087 	if (!ttm->num_pages) {
1088 		WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n",
1089 		     ttm->num_pages, bo_mem, ttm);
1090 	}
1091 
1092 	if (bo_mem->mem_type == AMDGPU_PL_GDS ||
1093 	    bo_mem->mem_type == AMDGPU_PL_GWS ||
1094 	    bo_mem->mem_type == AMDGPU_PL_OA)
1095 		return -EINVAL;
1096 
1097 	if (!amdgpu_gtt_mgr_has_gart_addr(bo_mem)) {
1098 		gtt->offset = AMDGPU_BO_INVALID_OFFSET;
1099 		return 0;
1100 	}
1101 
1102 	/* compute PTE flags relevant to this BO memory */
1103 	flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem);
1104 
1105 	/* bind pages into GART page tables */
1106 	gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
1107 	r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
1108 		ttm->pages, gtt->ttm.dma_address, flags);
1109 
1110 	if (r)
1111 		DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
1112 			  ttm->num_pages, gtt->offset);
1113 	return r;
1114 }
1115 
1116 /**
1117  * amdgpu_ttm_alloc_gart - Allocate GART memory for buffer object
1118  */
1119 int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
1120 {
1121 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1122 	struct ttm_operation_ctx ctx = { false, false };
1123 	struct amdgpu_ttm_tt *gtt = (void*)bo->ttm;
1124 	struct ttm_mem_reg tmp;
1125 	struct ttm_placement placement;
1126 	struct ttm_place placements;
1127 	uint64_t addr, flags;
1128 	int r;
1129 
1130 	if (bo->mem.start != AMDGPU_BO_INVALID_OFFSET)
1131 		return 0;
1132 
1133 	addr = amdgpu_gmc_agp_addr(bo);
1134 	if (addr != AMDGPU_BO_INVALID_OFFSET) {
1135 		bo->mem.start = addr >> PAGE_SHIFT;
1136 	} else {
1137 
1138 		/* allocate GART space */
1139 		tmp = bo->mem;
1140 		tmp.mm_node = NULL;
1141 		placement.num_placement = 1;
1142 		placement.placement = &placements;
1143 		placement.num_busy_placement = 1;
1144 		placement.busy_placement = &placements;
1145 		placements.fpfn = 0;
1146 		placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT;
1147 		placements.flags = (bo->mem.placement & ~TTM_PL_MASK_MEM) |
1148 			TTM_PL_FLAG_TT;
1149 
1150 		r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx);
1151 		if (unlikely(r))
1152 			return r;
1153 
1154 		/* compute PTE flags for this buffer object */
1155 		flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp);
1156 
1157 		/* Bind pages */
1158 		gtt->offset = (u64)tmp.start << PAGE_SHIFT;
1159 		r = amdgpu_ttm_gart_bind(adev, bo, flags);
1160 		if (unlikely(r)) {
1161 			ttm_bo_mem_put(bo, &tmp);
1162 			return r;
1163 		}
1164 
1165 		ttm_bo_mem_put(bo, &bo->mem);
1166 		bo->mem = tmp;
1167 	}
1168 
1169 	bo->offset = (bo->mem.start << PAGE_SHIFT) +
1170 		bo->bdev->man[bo->mem.mem_type].gpu_offset;
1171 
1172 	return 0;
1173 }
1174 
1175 /**
1176  * amdgpu_ttm_recover_gart - Rebind GTT pages
1177  *
1178  * Called by amdgpu_gtt_mgr_recover() from amdgpu_device_reset() to
1179  * rebind GTT pages during a GPU reset.
1180  */
1181 int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo)
1182 {
1183 	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
1184 	uint64_t flags;
1185 	int r;
1186 
1187 	if (!tbo->ttm)
1188 		return 0;
1189 
1190 	flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, &tbo->mem);
1191 	r = amdgpu_ttm_gart_bind(adev, tbo, flags);
1192 
1193 	return r;
1194 }
1195 
1196 /**
1197  * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages
1198  *
1199  * Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and
1200  * ttm_tt_destroy().
1201  */
1202 static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
1203 {
1204 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
1205 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
1206 	int r;
1207 
1208 	/* if the pages have userptr pinning then clear that first */
1209 	if (gtt->userptr)
1210 		amdgpu_ttm_tt_unpin_userptr(ttm);
1211 
1212 	if (gtt->offset == AMDGPU_BO_INVALID_OFFSET)
1213 		return 0;
1214 
1215 	/* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */
1216 	r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages);
1217 	if (r)
1218 		DRM_ERROR("failed to unbind %lu pages at 0x%08llX\n",
1219 			  gtt->ttm.ttm.num_pages, gtt->offset);
1220 	return r;
1221 }
1222 
1223 static void amdgpu_ttm_backend_destroy(struct ttm_tt *ttm)
1224 {
1225 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
1226 
1227 	if (gtt->usertask)
1228 		put_task_struct(gtt->usertask);
1229 
1230 	ttm_dma_tt_fini(&gtt->ttm);
1231 	kfree(gtt);
1232 }
1233 
1234 static struct ttm_backend_func amdgpu_backend_func = {
1235 	.bind = &amdgpu_ttm_backend_bind,
1236 	.unbind = &amdgpu_ttm_backend_unbind,
1237 	.destroy = &amdgpu_ttm_backend_destroy,
1238 };
1239 
1240 /**
1241  * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
1242  *
1243  * @bo: The buffer object to create a GTT ttm_tt object around
1244  *
1245  * Called by ttm_tt_create().
1246  */
1247 static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
1248 					   uint32_t page_flags)
1249 {
1250 	struct amdgpu_ttm_tt *gtt;
1251 
1252 	gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL);
1253 	if (gtt == NULL) {
1254 		return NULL;
1255 	}
1256 	gtt->ttm.ttm.func = &amdgpu_backend_func;
1257 	gtt->gobj = &bo->base;
1258 
1259 	/* allocate space for the uninitialized page entries */
1260 	if (ttm_sg_tt_init(&gtt->ttm, bo, page_flags)) {
1261 		kfree(gtt);
1262 		return NULL;
1263 	}
1264 	return &gtt->ttm.ttm;
1265 }
1266 
1267 /**
1268  * amdgpu_ttm_tt_populate - Map GTT pages visible to the device
1269  *
1270  * Map the pages of a ttm_tt object to an address space visible
1271  * to the underlying device.
1272  */
1273 static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm,
1274 			struct ttm_operation_ctx *ctx)
1275 {
1276 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
1277 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
1278 
1279 	/* user pages are bound by amdgpu_ttm_tt_pin_userptr() */
1280 	if (gtt && gtt->userptr) {
1281 		ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
1282 		if (!ttm->sg)
1283 			return -ENOMEM;
1284 
1285 		ttm->page_flags |= TTM_PAGE_FLAG_SG;
1286 		ttm->state = tt_unbound;
1287 		return 0;
1288 	}
1289 
1290 	if (ttm->page_flags & TTM_PAGE_FLAG_SG) {
1291 		if (!ttm->sg) {
1292 			struct dma_buf_attachment *attach;
1293 			struct sg_table *sgt;
1294 
1295 			attach = gtt->gobj->import_attach;
1296 			sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
1297 			if (IS_ERR(sgt))
1298 				return PTR_ERR(sgt);
1299 
1300 			ttm->sg = sgt;
1301 		}
1302 
1303 		drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
1304 						 gtt->ttm.dma_address,
1305 						 ttm->num_pages);
1306 		ttm->state = tt_unbound;
1307 		return 0;
1308 	}
1309 
1310 #ifdef CONFIG_SWIOTLB
1311 	if (adev->need_swiotlb && swiotlb_nr_tbl()) {
1312 		return ttm_dma_populate(&gtt->ttm, adev->dev, ctx);
1313 	}
1314 #endif
1315 
1316 	/* fall back to generic helper to populate the page array
1317 	 * and map them to the device */
1318 	return ttm_populate_and_map_pages(adev->dev, &gtt->ttm, ctx);
1319 }
1320 
1321 /**
1322  * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays
1323  *
1324  * Unmaps pages of a ttm_tt object from the device address space and
1325  * unpopulates the page array backing it.
1326  */
1327 static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
1328 {
1329 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
1330 	struct amdgpu_device *adev;
1331 
1332 	if (gtt && gtt->userptr) {
1333 		amdgpu_ttm_tt_set_user_pages(ttm, NULL);
1334 		kfree(ttm->sg);
1335 		ttm->page_flags &= ~TTM_PAGE_FLAG_SG;
1336 		return;
1337 	}
1338 
1339 	if (ttm->sg && gtt->gobj->import_attach) {
1340 		struct dma_buf_attachment *attach;
1341 
1342 		attach = gtt->gobj->import_attach;
1343 		dma_buf_unmap_attachment(attach, ttm->sg, DMA_BIDIRECTIONAL);
1344 		ttm->sg = NULL;
1345 		return;
1346 	}
1347 
1348 	if (ttm->page_flags & TTM_PAGE_FLAG_SG)
1349 		return;
1350 
1351 	adev = amdgpu_ttm_adev(ttm->bdev);
1352 
1353 #ifdef CONFIG_SWIOTLB
1354 	if (adev->need_swiotlb && swiotlb_nr_tbl()) {
1355 		ttm_dma_unpopulate(&gtt->ttm, adev->dev);
1356 		return;
1357 	}
1358 #endif
1359 
1360 	/* fall back to generic helper to unmap and unpopulate array */
1361 	ttm_unmap_and_unpopulate_pages(adev->dev, &gtt->ttm);
1362 }
1363 
1364 /**
1365  * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current
1366  * task
1367  *
1368  * @ttm: The ttm_tt object to bind this userptr object to
1369  * @addr:  The address in the current tasks VM space to use
1370  * @flags: Requirements of userptr object.
1371  *
1372  * Called by amdgpu_gem_userptr_ioctl() to bind userptr pages
1373  * to current task
1374  */
1375 int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
1376 			      uint32_t flags)
1377 {
1378 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
1379 
1380 	if (gtt == NULL)
1381 		return -EINVAL;
1382 
1383 	gtt->userptr = addr;
1384 	gtt->userflags = flags;
1385 
1386 	if (gtt->usertask)
1387 		put_task_struct(gtt->usertask);
1388 	gtt->usertask = current->group_leader;
1389 	get_task_struct(gtt->usertask);
1390 
1391 	return 0;
1392 }
1393 
1394 /**
1395  * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object
1396  */
1397 struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)
1398 {
1399 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
1400 
1401 	if (gtt == NULL)
1402 		return NULL;
1403 
1404 	if (gtt->usertask == NULL)
1405 		return NULL;
1406 
1407 	return gtt->usertask->mm;
1408 }
1409 
1410 /**
1411  * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an
1412  * address range for the current task.
1413  *
1414  */
1415 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
1416 				  unsigned long end)
1417 {
1418 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
1419 	unsigned long size;
1420 
1421 	if (gtt == NULL || !gtt->userptr)
1422 		return false;
1423 
1424 	/* Return false if no part of the ttm_tt object lies within
1425 	 * the range
1426 	 */
1427 	size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE;
1428 	if (gtt->userptr > end || gtt->userptr + size <= start)
1429 		return false;
1430 
1431 	return true;
1432 }
1433 
1434 /**
1435  * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr?
1436  */
1437 bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm)
1438 {
1439 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
1440 
1441 	if (gtt == NULL || !gtt->userptr)
1442 		return false;
1443 
1444 	return true;
1445 }
1446 
1447 /**
1448  * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only?
1449  */
1450 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
1451 {
1452 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
1453 
1454 	if (gtt == NULL)
1455 		return false;
1456 
1457 	return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
1458 }
1459 
1460 /**
1461  * amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object
1462  *
1463  * @ttm: The ttm_tt object to compute the flags for
1464  * @mem: The memory registry backing this ttm_tt object
1465  *
1466  * Figure out the flags to use for a VM PDE (Page Directory Entry).
1467  */
1468 uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
1469 {
1470 	uint64_t flags = 0;
1471 
1472 	if (mem && mem->mem_type != TTM_PL_SYSTEM)
1473 		flags |= AMDGPU_PTE_VALID;
1474 
1475 	if (mem && mem->mem_type == TTM_PL_TT) {
1476 		flags |= AMDGPU_PTE_SYSTEM;
1477 
1478 		if (ttm->caching_state == tt_cached)
1479 			flags |= AMDGPU_PTE_SNOOPED;
1480 	}
1481 
1482 	return flags;
1483 }
1484 
1485 /**
1486  * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object
1487  *
1488  * @ttm: The ttm_tt object to compute the flags for
1489  * @mem: The memory registry backing this ttm_tt object
1490 
1491  * Figure out the flags to use for a VM PTE (Page Table Entry).
1492  */
1493 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
1494 				 struct ttm_mem_reg *mem)
1495 {
1496 	uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem);
1497 
1498 	flags |= adev->gart.gart_pte_flags;
1499 	flags |= AMDGPU_PTE_READABLE;
1500 
1501 	if (!amdgpu_ttm_tt_is_readonly(ttm))
1502 		flags |= AMDGPU_PTE_WRITEABLE;
1503 
1504 	return flags;
1505 }
1506 
1507 /**
1508  * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer
1509  * object.
1510  *
1511  * Return true if eviction is sensible. Called by ttm_mem_evict_first() on
1512  * behalf of ttm_bo_mem_force_space() which tries to evict buffer objects until
1513  * it can find space for a new object and by ttm_bo_force_list_clean() which is
1514  * used to clean out a memory space.
1515  */
1516 static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
1517 					    const struct ttm_place *place)
1518 {
1519 	unsigned long num_pages = bo->mem.num_pages;
1520 	struct drm_mm_node *node = bo->mem.mm_node;
1521 	struct dma_resv_list *flist;
1522 	struct dma_fence *f;
1523 	int i;
1524 
1525 	/* Don't evict VM page tables while they are busy, otherwise we can't
1526 	 * cleanly handle page faults.
1527 	 */
1528 	if (bo->type == ttm_bo_type_kernel &&
1529 	    !dma_resv_test_signaled_rcu(bo->base.resv, true))
1530 		return false;
1531 
1532 	/* If bo is a KFD BO, check if the bo belongs to the current process.
1533 	 * If true, then return false as any KFD process needs all its BOs to
1534 	 * be resident to run successfully
1535 	 */
1536 	flist = dma_resv_get_list(bo->base.resv);
1537 	if (flist) {
1538 		for (i = 0; i < flist->shared_count; ++i) {
1539 			f = rcu_dereference_protected(flist->shared[i],
1540 				dma_resv_held(bo->base.resv));
1541 			if (amdkfd_fence_check_mm(f, current->mm))
1542 				return false;
1543 		}
1544 	}
1545 
1546 	switch (bo->mem.mem_type) {
1547 	case TTM_PL_TT:
1548 		return true;
1549 
1550 	case TTM_PL_VRAM:
1551 		/* Check each drm MM node individually */
1552 		while (num_pages) {
1553 			if (place->fpfn < (node->start + node->size) &&
1554 			    !(place->lpfn && place->lpfn <= node->start))
1555 				return true;
1556 
1557 			num_pages -= node->size;
1558 			++node;
1559 		}
1560 		return false;
1561 
1562 	default:
1563 		break;
1564 	}
1565 
1566 	return ttm_bo_eviction_valuable(bo, place);
1567 }
1568 
1569 /**
1570  * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object.
1571  *
1572  * @bo:  The buffer object to read/write
1573  * @offset:  Offset into buffer object
1574  * @buf:  Secondary buffer to write/read from
1575  * @len: Length in bytes of access
1576  * @write:  true if writing
1577  *
1578  * This is used to access VRAM that backs a buffer object via MMIO
1579  * access for debugging purposes.
1580  */
1581 static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
1582 				    unsigned long offset,
1583 				    void *buf, int len, int write)
1584 {
1585 	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1586 	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
1587 	struct drm_mm_node *nodes;
1588 	uint32_t value = 0;
1589 	int ret = 0;
1590 	uint64_t pos;
1591 	unsigned long flags;
1592 
1593 	if (bo->mem.mem_type != TTM_PL_VRAM)
1594 		return -EIO;
1595 
1596 	nodes = amdgpu_find_mm_node(&abo->tbo.mem, &offset);
1597 	pos = (nodes->start << PAGE_SHIFT) + offset;
1598 
1599 	while (len && pos < adev->gmc.mc_vram_size) {
1600 		uint64_t aligned_pos = pos & ~(uint64_t)3;
1601 		uint32_t bytes = 4 - (pos & 3);
1602 		uint32_t shift = (pos & 3) * 8;
1603 		uint32_t mask = 0xffffffff << shift;
1604 
1605 		if (len < bytes) {
1606 			mask &= 0xffffffff >> (bytes - len) * 8;
1607 			bytes = len;
1608 		}
1609 
1610 		spin_lock_irqsave(&adev->mmio_idx_lock, flags);
1611 		WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000);
1612 		WREG32_NO_KIQ(mmMM_INDEX_HI, aligned_pos >> 31);
1613 		if (!write || mask != 0xffffffff)
1614 			value = RREG32_NO_KIQ(mmMM_DATA);
1615 		if (write) {
1616 			value &= ~mask;
1617 			value |= (*(uint32_t *)buf << shift) & mask;
1618 			WREG32_NO_KIQ(mmMM_DATA, value);
1619 		}
1620 		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
1621 		if (!write) {
1622 			value = (value & mask) >> shift;
1623 			memcpy(buf, &value, bytes);
1624 		}
1625 
1626 		ret += bytes;
1627 		buf = (uint8_t *)buf + bytes;
1628 		pos += bytes;
1629 		len -= bytes;
1630 		if (pos >= (nodes->start + nodes->size) << PAGE_SHIFT) {
1631 			++nodes;
1632 			pos = (nodes->start << PAGE_SHIFT);
1633 		}
1634 	}
1635 
1636 	return ret;
1637 }
1638 
1639 static struct ttm_bo_driver amdgpu_bo_driver = {
1640 	.ttm_tt_create = &amdgpu_ttm_tt_create,
1641 	.ttm_tt_populate = &amdgpu_ttm_tt_populate,
1642 	.ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate,
1643 	.invalidate_caches = &amdgpu_invalidate_caches,
1644 	.init_mem_type = &amdgpu_init_mem_type,
1645 	.eviction_valuable = amdgpu_ttm_bo_eviction_valuable,
1646 	.evict_flags = &amdgpu_evict_flags,
1647 	.move = &amdgpu_bo_move,
1648 	.verify_access = &amdgpu_verify_access,
1649 	.move_notify = &amdgpu_bo_move_notify,
1650 	.release_notify = &amdgpu_bo_release_notify,
1651 	.fault_reserve_notify = &amdgpu_bo_fault_reserve_notify,
1652 	.io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
1653 	.io_mem_free = &amdgpu_ttm_io_mem_free,
1654 	.io_mem_pfn = amdgpu_ttm_io_mem_pfn,
1655 	.access_memory = &amdgpu_ttm_access_memory,
1656 	.del_from_lru_notify = &amdgpu_vm_del_from_lru_notify
1657 };
1658 
1659 /*
1660  * Firmware Reservation functions
1661  */
1662 /**
1663  * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram
1664  *
1665  * @adev: amdgpu_device pointer
1666  *
1667  * free fw reserved vram if it has been reserved.
1668  */
1669 static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
1670 {
1671 	amdgpu_bo_free_kernel(&adev->fw_vram_usage.reserved_bo,
1672 		NULL, &adev->fw_vram_usage.va);
1673 }
1674 
1675 /**
1676  * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
1677  *
1678  * @adev: amdgpu_device pointer
1679  *
1680  * create bo vram reservation from fw.
1681  */
1682 static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
1683 {
1684 	uint64_t vram_size = adev->gmc.visible_vram_size;
1685 
1686 	adev->fw_vram_usage.va = NULL;
1687 	adev->fw_vram_usage.reserved_bo = NULL;
1688 
1689 	if (adev->fw_vram_usage.size == 0 ||
1690 	    adev->fw_vram_usage.size > vram_size)
1691 		return 0;
1692 
1693 	return amdgpu_bo_create_kernel_at(adev,
1694 					  adev->fw_vram_usage.start_offset,
1695 					  adev->fw_vram_usage.size,
1696 					  AMDGPU_GEM_DOMAIN_VRAM,
1697 					  &adev->fw_vram_usage.reserved_bo,
1698 					  &adev->fw_vram_usage.va);
1699 }
1700 
1701 /*
1702  * Memoy training reservation functions
1703  */
1704 
1705 /**
1706  * amdgpu_ttm_training_reserve_vram_fini - free memory training reserved vram
1707  *
1708  * @adev: amdgpu_device pointer
1709  *
1710  * free memory training reserved vram if it has been reserved.
1711  */
1712 static int amdgpu_ttm_training_reserve_vram_fini(struct amdgpu_device *adev)
1713 {
1714 	struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx;
1715 
1716 	ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT;
1717 	amdgpu_bo_free_kernel(&ctx->c2p_bo, NULL, NULL);
1718 	ctx->c2p_bo = NULL;
1719 
1720 	amdgpu_bo_free_kernel(&ctx->p2c_bo, NULL, NULL);
1721 	ctx->p2c_bo = NULL;
1722 
1723 	return 0;
1724 }
1725 
1726 /**
1727  * amdgpu_ttm_training_reserve_vram_init - create bo vram reservation from memory training
1728  *
1729  * @adev: amdgpu_device pointer
1730  *
1731  * create bo vram reservation from memory training.
1732  */
1733 static int amdgpu_ttm_training_reserve_vram_init(struct amdgpu_device *adev)
1734 {
1735 	int ret;
1736 	struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx;
1737 
1738 	memset(ctx, 0, sizeof(*ctx));
1739 	if (!adev->fw_vram_usage.mem_train_support) {
1740 		DRM_DEBUG("memory training does not support!\n");
1741 		return 0;
1742 	}
1743 
1744 	ctx->c2p_train_data_offset = adev->fw_vram_usage.mem_train_fb_loc;
1745 	ctx->p2c_train_data_offset = (adev->gmc.mc_vram_size - GDDR6_MEM_TRAINING_OFFSET);
1746 	ctx->train_data_size = GDDR6_MEM_TRAINING_DATA_SIZE_IN_BYTES;
1747 
1748 	DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n",
1749 		  ctx->train_data_size,
1750 		  ctx->p2c_train_data_offset,
1751 		  ctx->c2p_train_data_offset);
1752 
1753 	ret = amdgpu_bo_create_kernel_at(adev,
1754 					 ctx->p2c_train_data_offset,
1755 					 ctx->train_data_size,
1756 					 AMDGPU_GEM_DOMAIN_VRAM,
1757 					 &ctx->p2c_bo,
1758 					 NULL);
1759 	if (ret) {
1760 		DRM_ERROR("alloc p2c_bo failed(%d)!\n", ret);
1761 		goto Err_out;
1762 	}
1763 
1764 	ret = amdgpu_bo_create_kernel_at(adev,
1765 					 ctx->c2p_train_data_offset,
1766 					 ctx->train_data_size,
1767 					 AMDGPU_GEM_DOMAIN_VRAM,
1768 					 &ctx->c2p_bo,
1769 					 NULL);
1770 	if (ret) {
1771 		DRM_ERROR("alloc c2p_bo failed(%d)!\n", ret);
1772 		goto Err_out;
1773 	}
1774 
1775 	ctx->init = PSP_MEM_TRAIN_RESERVE_SUCCESS;
1776 	return 0;
1777 
1778 Err_out:
1779 	amdgpu_ttm_training_reserve_vram_fini(adev);
1780 	return ret;
1781 }
1782 
1783 /**
1784  * amdgpu_ttm_init - Init the memory management (ttm) as well as various
1785  * gtt/vram related fields.
1786  *
1787  * This initializes all of the memory space pools that the TTM layer
1788  * will need such as the GTT space (system memory mapped to the device),
1789  * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which
1790  * can be mapped per VMID.
1791  */
1792 int amdgpu_ttm_init(struct amdgpu_device *adev)
1793 {
1794 	uint64_t gtt_size;
1795 	int r;
1796 	u64 vis_vram_limit;
1797 	void *stolen_vga_buf;
1798 
1799 	mutex_init(&adev->mman.gtt_window_lock);
1800 
1801 	/* No others user of address space so set it to 0 */
1802 	r = ttm_bo_device_init(&adev->mman.bdev,
1803 			       &amdgpu_bo_driver,
1804 			       adev->ddev->anon_inode->i_mapping,
1805 			       adev->ddev->vma_offset_manager,
1806 			       dma_addressing_limited(adev->dev));
1807 	if (r) {
1808 		DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
1809 		return r;
1810 	}
1811 	adev->mman.initialized = true;
1812 
1813 	/* We opt to avoid OOM on system pages allocations */
1814 	adev->mman.bdev.no_retry = true;
1815 
1816 	/* Initialize VRAM pool with all of VRAM divided into pages */
1817 	r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_VRAM,
1818 				adev->gmc.real_vram_size >> PAGE_SHIFT);
1819 	if (r) {
1820 		DRM_ERROR("Failed initializing VRAM heap.\n");
1821 		return r;
1822 	}
1823 
1824 	/* Reduce size of CPU-visible VRAM if requested */
1825 	vis_vram_limit = (u64)amdgpu_vis_vram_limit * 1024 * 1024;
1826 	if (amdgpu_vis_vram_limit > 0 &&
1827 	    vis_vram_limit <= adev->gmc.visible_vram_size)
1828 		adev->gmc.visible_vram_size = vis_vram_limit;
1829 
1830 	/* Change the size here instead of the init above so only lpfn is affected */
1831 	amdgpu_ttm_set_buffer_funcs_status(adev, false);
1832 #ifdef CONFIG_64BIT
1833 	adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base,
1834 						adev->gmc.visible_vram_size);
1835 #endif
1836 
1837 	/*
1838 	 *The reserved vram for firmware must be pinned to the specified
1839 	 *place on the VRAM, so reserve it early.
1840 	 */
1841 	r = amdgpu_ttm_fw_reserve_vram_init(adev);
1842 	if (r) {
1843 		return r;
1844 	}
1845 
1846 	/*
1847 	 *The reserved vram for memory training must be pinned to the specified
1848 	 *place on the VRAM, so reserve it early.
1849 	 */
1850 	r = amdgpu_ttm_training_reserve_vram_init(adev);
1851 	if (r)
1852 		return r;
1853 
1854 	/* allocate memory as required for VGA
1855 	 * This is used for VGA emulation and pre-OS scanout buffers to
1856 	 * avoid display artifacts while transitioning between pre-OS
1857 	 * and driver.  */
1858 	r = amdgpu_bo_create_kernel(adev, adev->gmc.stolen_size, PAGE_SIZE,
1859 				    AMDGPU_GEM_DOMAIN_VRAM,
1860 				    &adev->stolen_vga_memory,
1861 				    NULL, &stolen_vga_buf);
1862 	if (r)
1863 		return r;
1864 
1865 	/*
1866 	 * reserve one TMR (64K) memory at the top of VRAM which holds
1867 	 * IP Discovery data and is protected by PSP.
1868 	 */
1869 	r = amdgpu_bo_create_kernel_at(adev,
1870 				       adev->gmc.real_vram_size - DISCOVERY_TMR_SIZE,
1871 				       DISCOVERY_TMR_SIZE,
1872 				       AMDGPU_GEM_DOMAIN_VRAM,
1873 				       &adev->discovery_memory,
1874 				       NULL);
1875 	if (r)
1876 		return r;
1877 
1878 	DRM_INFO("amdgpu: %uM of VRAM memory ready\n",
1879 		 (unsigned) (adev->gmc.real_vram_size / (1024 * 1024)));
1880 
1881 	/* Compute GTT size, either bsaed on 3/4th the size of RAM size
1882 	 * or whatever the user passed on module init */
1883 	if (amdgpu_gtt_size == -1) {
1884 		struct sysinfo si;
1885 
1886 		si_meminfo(&si);
1887 		gtt_size = min(max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
1888 			       adev->gmc.mc_vram_size),
1889 			       ((uint64_t)si.totalram * si.mem_unit * 3/4));
1890 	}
1891 	else
1892 		gtt_size = (uint64_t)amdgpu_gtt_size << 20;
1893 
1894 	/* Initialize GTT memory pool */
1895 	r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT, gtt_size >> PAGE_SHIFT);
1896 	if (r) {
1897 		DRM_ERROR("Failed initializing GTT heap.\n");
1898 		return r;
1899 	}
1900 	DRM_INFO("amdgpu: %uM of GTT memory ready.\n",
1901 		 (unsigned)(gtt_size / (1024 * 1024)));
1902 
1903 	/* Initialize various on-chip memory pools */
1904 	r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GDS,
1905 			   adev->gds.gds_size);
1906 	if (r) {
1907 		DRM_ERROR("Failed initializing GDS heap.\n");
1908 		return r;
1909 	}
1910 
1911 	r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GWS,
1912 			   adev->gds.gws_size);
1913 	if (r) {
1914 		DRM_ERROR("Failed initializing gws heap.\n");
1915 		return r;
1916 	}
1917 
1918 	r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_OA,
1919 			   adev->gds.oa_size);
1920 	if (r) {
1921 		DRM_ERROR("Failed initializing oa heap.\n");
1922 		return r;
1923 	}
1924 
1925 	/* Register debugfs entries for amdgpu_ttm */
1926 	r = amdgpu_ttm_debugfs_init(adev);
1927 	if (r) {
1928 		DRM_ERROR("Failed to init debugfs\n");
1929 		return r;
1930 	}
1931 	return 0;
1932 }
1933 
1934 /**
1935  * amdgpu_ttm_late_init - Handle any late initialization for amdgpu_ttm
1936  */
1937 void amdgpu_ttm_late_init(struct amdgpu_device *adev)
1938 {
1939 	void *stolen_vga_buf;
1940 	/* return the VGA stolen memory (if any) back to VRAM */
1941 	amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, &stolen_vga_buf);
1942 }
1943 
1944 /**
1945  * amdgpu_ttm_fini - De-initialize the TTM memory pools
1946  */
1947 void amdgpu_ttm_fini(struct amdgpu_device *adev)
1948 {
1949 	if (!adev->mman.initialized)
1950 		return;
1951 
1952 	amdgpu_ttm_debugfs_fini(adev);
1953 	amdgpu_ttm_training_reserve_vram_fini(adev);
1954 	/* return the IP Discovery TMR memory back to VRAM */
1955 	amdgpu_bo_free_kernel(&adev->discovery_memory, NULL, NULL);
1956 	amdgpu_ttm_fw_reserve_vram_fini(adev);
1957 
1958 	if (adev->mman.aper_base_kaddr)
1959 		iounmap(adev->mman.aper_base_kaddr);
1960 	adev->mman.aper_base_kaddr = NULL;
1961 
1962 	ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_VRAM);
1963 	ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_TT);
1964 	ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GDS);
1965 	ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GWS);
1966 	ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_OA);
1967 	ttm_bo_device_release(&adev->mman.bdev);
1968 	adev->mman.initialized = false;
1969 	DRM_INFO("amdgpu: ttm finalized\n");
1970 }
1971 
1972 /**
1973  * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions
1974  *
1975  * @adev: amdgpu_device pointer
1976  * @enable: true when we can use buffer functions.
1977  *
1978  * Enable/disable use of buffer functions during suspend/resume. This should
1979  * only be called at bootup or when userspace isn't running.
1980  */
1981 void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
1982 {
1983 	struct ttm_mem_type_manager *man = &adev->mman.bdev.man[TTM_PL_VRAM];
1984 	uint64_t size;
1985 	int r;
1986 
1987 	if (!adev->mman.initialized || adev->in_gpu_reset ||
1988 	    adev->mman.buffer_funcs_enabled == enable)
1989 		return;
1990 
1991 	if (enable) {
1992 		struct amdgpu_ring *ring;
1993 		struct drm_sched_rq *rq;
1994 
1995 		ring = adev->mman.buffer_funcs_ring;
1996 		rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
1997 		r = drm_sched_entity_init(&adev->mman.entity, &rq, 1, NULL);
1998 		if (r) {
1999 			DRM_ERROR("Failed setting up TTM BO move entity (%d)\n",
2000 				  r);
2001 			return;
2002 		}
2003 	} else {
2004 		drm_sched_entity_destroy(&adev->mman.entity);
2005 		dma_fence_put(man->move);
2006 		man->move = NULL;
2007 	}
2008 
2009 	/* this just adjusts TTM size idea, which sets lpfn to the correct value */
2010 	if (enable)
2011 		size = adev->gmc.real_vram_size;
2012 	else
2013 		size = adev->gmc.visible_vram_size;
2014 	man->size = size >> PAGE_SHIFT;
2015 	adev->mman.buffer_funcs_enabled = enable;
2016 }
2017 
2018 int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma)
2019 {
2020 	struct drm_file *file_priv = filp->private_data;
2021 	struct amdgpu_device *adev = file_priv->minor->dev->dev_private;
2022 
2023 	if (adev == NULL)
2024 		return -EINVAL;
2025 
2026 	return ttm_bo_mmap(filp, vma, &adev->mman.bdev);
2027 }
2028 
2029 static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
2030 			     struct ttm_mem_reg *mem, unsigned num_pages,
2031 			     uint64_t offset, unsigned window,
2032 			     struct amdgpu_ring *ring,
2033 			     uint64_t *addr)
2034 {
2035 	struct amdgpu_ttm_tt *gtt = (void *)bo->ttm;
2036 	struct amdgpu_device *adev = ring->adev;
2037 	struct ttm_tt *ttm = bo->ttm;
2038 	struct amdgpu_job *job;
2039 	unsigned num_dw, num_bytes;
2040 	dma_addr_t *dma_address;
2041 	struct dma_fence *fence;
2042 	uint64_t src_addr, dst_addr;
2043 	uint64_t flags;
2044 	int r;
2045 
2046 	BUG_ON(adev->mman.buffer_funcs->copy_max_bytes <
2047 	       AMDGPU_GTT_MAX_TRANSFER_SIZE * 8);
2048 
2049 	*addr = adev->gmc.gart_start;
2050 	*addr += (u64)window * AMDGPU_GTT_MAX_TRANSFER_SIZE *
2051 		AMDGPU_GPU_PAGE_SIZE;
2052 
2053 	num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
2054 	num_bytes = num_pages * 8;
2055 
2056 	r = amdgpu_job_alloc_with_ib(adev, num_dw * 4 + num_bytes, &job);
2057 	if (r)
2058 		return r;
2059 
2060 	src_addr = num_dw * 4;
2061 	src_addr += job->ibs[0].gpu_addr;
2062 
2063 	dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
2064 	dst_addr += window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 8;
2065 	amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
2066 				dst_addr, num_bytes);
2067 
2068 	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2069 	WARN_ON(job->ibs[0].length_dw > num_dw);
2070 
2071 	dma_address = &gtt->ttm.dma_address[offset >> PAGE_SHIFT];
2072 	flags = amdgpu_ttm_tt_pte_flags(adev, ttm, mem);
2073 	r = amdgpu_gart_map(adev, 0, num_pages, dma_address, flags,
2074 			    &job->ibs[0].ptr[num_dw]);
2075 	if (r)
2076 		goto error_free;
2077 
2078 	r = amdgpu_job_submit(job, &adev->mman.entity,
2079 			      AMDGPU_FENCE_OWNER_UNDEFINED, &fence);
2080 	if (r)
2081 		goto error_free;
2082 
2083 	dma_fence_put(fence);
2084 
2085 	return r;
2086 
2087 error_free:
2088 	amdgpu_job_free(job);
2089 	return r;
2090 }
2091 
2092 int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
2093 		       uint64_t dst_offset, uint32_t byte_count,
2094 		       struct dma_resv *resv,
2095 		       struct dma_fence **fence, bool direct_submit,
2096 		       bool vm_needs_flush)
2097 {
2098 	struct amdgpu_device *adev = ring->adev;
2099 	struct amdgpu_job *job;
2100 
2101 	uint32_t max_bytes;
2102 	unsigned num_loops, num_dw;
2103 	unsigned i;
2104 	int r;
2105 
2106 	if (direct_submit && !ring->sched.ready) {
2107 		DRM_ERROR("Trying to move memory with ring turned off.\n");
2108 		return -EINVAL;
2109 	}
2110 
2111 	max_bytes = adev->mman.buffer_funcs->copy_max_bytes;
2112 	num_loops = DIV_ROUND_UP(byte_count, max_bytes);
2113 	num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->copy_num_dw, 8);
2114 
2115 	r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
2116 	if (r)
2117 		return r;
2118 
2119 	if (vm_needs_flush) {
2120 		job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
2121 		job->vm_needs_flush = true;
2122 	}
2123 	if (resv) {
2124 		r = amdgpu_sync_resv(adev, &job->sync, resv,
2125 				     AMDGPU_FENCE_OWNER_UNDEFINED,
2126 				     false);
2127 		if (r) {
2128 			DRM_ERROR("sync failed (%d).\n", r);
2129 			goto error_free;
2130 		}
2131 	}
2132 
2133 	for (i = 0; i < num_loops; i++) {
2134 		uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
2135 
2136 		amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset,
2137 					dst_offset, cur_size_in_bytes);
2138 
2139 		src_offset += cur_size_in_bytes;
2140 		dst_offset += cur_size_in_bytes;
2141 		byte_count -= cur_size_in_bytes;
2142 	}
2143 
2144 	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2145 	WARN_ON(job->ibs[0].length_dw > num_dw);
2146 	if (direct_submit)
2147 		r = amdgpu_job_submit_direct(job, ring, fence);
2148 	else
2149 		r = amdgpu_job_submit(job, &adev->mman.entity,
2150 				      AMDGPU_FENCE_OWNER_UNDEFINED, fence);
2151 	if (r)
2152 		goto error_free;
2153 
2154 	return r;
2155 
2156 error_free:
2157 	amdgpu_job_free(job);
2158 	DRM_ERROR("Error scheduling IBs (%d)\n", r);
2159 	return r;
2160 }
2161 
2162 int amdgpu_fill_buffer(struct amdgpu_bo *bo,
2163 		       uint32_t src_data,
2164 		       struct dma_resv *resv,
2165 		       struct dma_fence **fence)
2166 {
2167 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
2168 	uint32_t max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
2169 	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2170 
2171 	struct drm_mm_node *mm_node;
2172 	unsigned long num_pages;
2173 	unsigned int num_loops, num_dw;
2174 
2175 	struct amdgpu_job *job;
2176 	int r;
2177 
2178 	if (!adev->mman.buffer_funcs_enabled) {
2179 		DRM_ERROR("Trying to clear memory with ring turned off.\n");
2180 		return -EINVAL;
2181 	}
2182 
2183 	if (bo->tbo.mem.mem_type == TTM_PL_TT) {
2184 		r = amdgpu_ttm_alloc_gart(&bo->tbo);
2185 		if (r)
2186 			return r;
2187 	}
2188 
2189 	num_pages = bo->tbo.num_pages;
2190 	mm_node = bo->tbo.mem.mm_node;
2191 	num_loops = 0;
2192 	while (num_pages) {
2193 		uint64_t byte_count = mm_node->size << PAGE_SHIFT;
2194 
2195 		num_loops += DIV_ROUND_UP_ULL(byte_count, max_bytes);
2196 		num_pages -= mm_node->size;
2197 		++mm_node;
2198 	}
2199 	num_dw = num_loops * adev->mman.buffer_funcs->fill_num_dw;
2200 
2201 	/* for IB padding */
2202 	num_dw += 64;
2203 
2204 	r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
2205 	if (r)
2206 		return r;
2207 
2208 	if (resv) {
2209 		r = amdgpu_sync_resv(adev, &job->sync, resv,
2210 				     AMDGPU_FENCE_OWNER_UNDEFINED, false);
2211 		if (r) {
2212 			DRM_ERROR("sync failed (%d).\n", r);
2213 			goto error_free;
2214 		}
2215 	}
2216 
2217 	num_pages = bo->tbo.num_pages;
2218 	mm_node = bo->tbo.mem.mm_node;
2219 
2220 	while (num_pages) {
2221 		uint64_t byte_count = mm_node->size << PAGE_SHIFT;
2222 		uint64_t dst_addr;
2223 
2224 		dst_addr = amdgpu_mm_node_addr(&bo->tbo, mm_node, &bo->tbo.mem);
2225 		while (byte_count) {
2226 			uint32_t cur_size_in_bytes = min_t(uint64_t, byte_count,
2227 							   max_bytes);
2228 
2229 			amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data,
2230 						dst_addr, cur_size_in_bytes);
2231 
2232 			dst_addr += cur_size_in_bytes;
2233 			byte_count -= cur_size_in_bytes;
2234 		}
2235 
2236 		num_pages -= mm_node->size;
2237 		++mm_node;
2238 	}
2239 
2240 	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2241 	WARN_ON(job->ibs[0].length_dw > num_dw);
2242 	r = amdgpu_job_submit(job, &adev->mman.entity,
2243 			      AMDGPU_FENCE_OWNER_UNDEFINED, fence);
2244 	if (r)
2245 		goto error_free;
2246 
2247 	return 0;
2248 
2249 error_free:
2250 	amdgpu_job_free(job);
2251 	return r;
2252 }
2253 
2254 #if defined(CONFIG_DEBUG_FS)
2255 
2256 static int amdgpu_mm_dump_table(struct seq_file *m, void *data)
2257 {
2258 	struct drm_info_node *node = (struct drm_info_node *)m->private;
2259 	unsigned ttm_pl = (uintptr_t)node->info_ent->data;
2260 	struct drm_device *dev = node->minor->dev;
2261 	struct amdgpu_device *adev = dev->dev_private;
2262 	struct ttm_mem_type_manager *man = &adev->mman.bdev.man[ttm_pl];
2263 	struct drm_printer p = drm_seq_file_printer(m);
2264 
2265 	man->func->debug(man, &p);
2266 	return 0;
2267 }
2268 
2269 static const struct drm_info_list amdgpu_ttm_debugfs_list[] = {
2270 	{"amdgpu_vram_mm", amdgpu_mm_dump_table, 0, (void *)TTM_PL_VRAM},
2271 	{"amdgpu_gtt_mm", amdgpu_mm_dump_table, 0, (void *)TTM_PL_TT},
2272 	{"amdgpu_gds_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_GDS},
2273 	{"amdgpu_gws_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_GWS},
2274 	{"amdgpu_oa_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_OA},
2275 	{"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL},
2276 #ifdef CONFIG_SWIOTLB
2277 	{"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL}
2278 #endif
2279 };
2280 
2281 /**
2282  * amdgpu_ttm_vram_read - Linear read access to VRAM
2283  *
2284  * Accesses VRAM via MMIO for debugging purposes.
2285  */
2286 static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
2287 				    size_t size, loff_t *pos)
2288 {
2289 	struct amdgpu_device *adev = file_inode(f)->i_private;
2290 	ssize_t result = 0;
2291 	int r;
2292 
2293 	if (size & 0x3 || *pos & 0x3)
2294 		return -EINVAL;
2295 
2296 	if (*pos >= adev->gmc.mc_vram_size)
2297 		return -ENXIO;
2298 
2299 	while (size) {
2300 		unsigned long flags;
2301 		uint32_t value;
2302 
2303 		if (*pos >= adev->gmc.mc_vram_size)
2304 			return result;
2305 
2306 		spin_lock_irqsave(&adev->mmio_idx_lock, flags);
2307 		WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
2308 		WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
2309 		value = RREG32_NO_KIQ(mmMM_DATA);
2310 		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
2311 
2312 		r = put_user(value, (uint32_t *)buf);
2313 		if (r)
2314 			return r;
2315 
2316 		result += 4;
2317 		buf += 4;
2318 		*pos += 4;
2319 		size -= 4;
2320 	}
2321 
2322 	return result;
2323 }
2324 
2325 /**
2326  * amdgpu_ttm_vram_write - Linear write access to VRAM
2327  *
2328  * Accesses VRAM via MMIO for debugging purposes.
2329  */
2330 static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf,
2331 				    size_t size, loff_t *pos)
2332 {
2333 	struct amdgpu_device *adev = file_inode(f)->i_private;
2334 	ssize_t result = 0;
2335 	int r;
2336 
2337 	if (size & 0x3 || *pos & 0x3)
2338 		return -EINVAL;
2339 
2340 	if (*pos >= adev->gmc.mc_vram_size)
2341 		return -ENXIO;
2342 
2343 	while (size) {
2344 		unsigned long flags;
2345 		uint32_t value;
2346 
2347 		if (*pos >= adev->gmc.mc_vram_size)
2348 			return result;
2349 
2350 		r = get_user(value, (uint32_t *)buf);
2351 		if (r)
2352 			return r;
2353 
2354 		spin_lock_irqsave(&adev->mmio_idx_lock, flags);
2355 		WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
2356 		WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
2357 		WREG32_NO_KIQ(mmMM_DATA, value);
2358 		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
2359 
2360 		result += 4;
2361 		buf += 4;
2362 		*pos += 4;
2363 		size -= 4;
2364 	}
2365 
2366 	return result;
2367 }
2368 
2369 static const struct file_operations amdgpu_ttm_vram_fops = {
2370 	.owner = THIS_MODULE,
2371 	.read = amdgpu_ttm_vram_read,
2372 	.write = amdgpu_ttm_vram_write,
2373 	.llseek = default_llseek,
2374 };
2375 
2376 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
2377 
2378 /**
2379  * amdgpu_ttm_gtt_read - Linear read access to GTT memory
2380  */
2381 static ssize_t amdgpu_ttm_gtt_read(struct file *f, char __user *buf,
2382 				   size_t size, loff_t *pos)
2383 {
2384 	struct amdgpu_device *adev = file_inode(f)->i_private;
2385 	ssize_t result = 0;
2386 	int r;
2387 
2388 	while (size) {
2389 		loff_t p = *pos / PAGE_SIZE;
2390 		unsigned off = *pos & ~PAGE_MASK;
2391 		size_t cur_size = min_t(size_t, size, PAGE_SIZE - off);
2392 		struct page *page;
2393 		void *ptr;
2394 
2395 		if (p >= adev->gart.num_cpu_pages)
2396 			return result;
2397 
2398 		page = adev->gart.pages[p];
2399 		if (page) {
2400 			ptr = kmap(page);
2401 			ptr += off;
2402 
2403 			r = copy_to_user(buf, ptr, cur_size);
2404 			kunmap(adev->gart.pages[p]);
2405 		} else
2406 			r = clear_user(buf, cur_size);
2407 
2408 		if (r)
2409 			return -EFAULT;
2410 
2411 		result += cur_size;
2412 		buf += cur_size;
2413 		*pos += cur_size;
2414 		size -= cur_size;
2415 	}
2416 
2417 	return result;
2418 }
2419 
2420 static const struct file_operations amdgpu_ttm_gtt_fops = {
2421 	.owner = THIS_MODULE,
2422 	.read = amdgpu_ttm_gtt_read,
2423 	.llseek = default_llseek
2424 };
2425 
2426 #endif
2427 
2428 /**
2429  * amdgpu_iomem_read - Virtual read access to GPU mapped memory
2430  *
2431  * This function is used to read memory that has been mapped to the
2432  * GPU and the known addresses are not physical addresses but instead
2433  * bus addresses (e.g., what you'd put in an IB or ring buffer).
2434  */
2435 static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf,
2436 				 size_t size, loff_t *pos)
2437 {
2438 	struct amdgpu_device *adev = file_inode(f)->i_private;
2439 	struct iommu_domain *dom;
2440 	ssize_t result = 0;
2441 	int r;
2442 
2443 	/* retrieve the IOMMU domain if any for this device */
2444 	dom = iommu_get_domain_for_dev(adev->dev);
2445 
2446 	while (size) {
2447 		phys_addr_t addr = *pos & PAGE_MASK;
2448 		loff_t off = *pos & ~PAGE_MASK;
2449 		size_t bytes = PAGE_SIZE - off;
2450 		unsigned long pfn;
2451 		struct page *p;
2452 		void *ptr;
2453 
2454 		bytes = bytes < size ? bytes : size;
2455 
2456 		/* Translate the bus address to a physical address.  If
2457 		 * the domain is NULL it means there is no IOMMU active
2458 		 * and the address translation is the identity
2459 		 */
2460 		addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
2461 
2462 		pfn = addr >> PAGE_SHIFT;
2463 		if (!pfn_valid(pfn))
2464 			return -EPERM;
2465 
2466 		p = pfn_to_page(pfn);
2467 		if (p->mapping != adev->mman.bdev.dev_mapping)
2468 			return -EPERM;
2469 
2470 		ptr = kmap(p);
2471 		r = copy_to_user(buf, ptr + off, bytes);
2472 		kunmap(p);
2473 		if (r)
2474 			return -EFAULT;
2475 
2476 		size -= bytes;
2477 		*pos += bytes;
2478 		result += bytes;
2479 	}
2480 
2481 	return result;
2482 }
2483 
2484 /**
2485  * amdgpu_iomem_write - Virtual write access to GPU mapped memory
2486  *
2487  * This function is used to write memory that has been mapped to the
2488  * GPU and the known addresses are not physical addresses but instead
2489  * bus addresses (e.g., what you'd put in an IB or ring buffer).
2490  */
2491 static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf,
2492 				 size_t size, loff_t *pos)
2493 {
2494 	struct amdgpu_device *adev = file_inode(f)->i_private;
2495 	struct iommu_domain *dom;
2496 	ssize_t result = 0;
2497 	int r;
2498 
2499 	dom = iommu_get_domain_for_dev(adev->dev);
2500 
2501 	while (size) {
2502 		phys_addr_t addr = *pos & PAGE_MASK;
2503 		loff_t off = *pos & ~PAGE_MASK;
2504 		size_t bytes = PAGE_SIZE - off;
2505 		unsigned long pfn;
2506 		struct page *p;
2507 		void *ptr;
2508 
2509 		bytes = bytes < size ? bytes : size;
2510 
2511 		addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
2512 
2513 		pfn = addr >> PAGE_SHIFT;
2514 		if (!pfn_valid(pfn))
2515 			return -EPERM;
2516 
2517 		p = pfn_to_page(pfn);
2518 		if (p->mapping != adev->mman.bdev.dev_mapping)
2519 			return -EPERM;
2520 
2521 		ptr = kmap(p);
2522 		r = copy_from_user(ptr + off, buf, bytes);
2523 		kunmap(p);
2524 		if (r)
2525 			return -EFAULT;
2526 
2527 		size -= bytes;
2528 		*pos += bytes;
2529 		result += bytes;
2530 	}
2531 
2532 	return result;
2533 }
2534 
2535 static const struct file_operations amdgpu_ttm_iomem_fops = {
2536 	.owner = THIS_MODULE,
2537 	.read = amdgpu_iomem_read,
2538 	.write = amdgpu_iomem_write,
2539 	.llseek = default_llseek
2540 };
2541 
2542 static const struct {
2543 	char *name;
2544 	const struct file_operations *fops;
2545 	int domain;
2546 } ttm_debugfs_entries[] = {
2547 	{ "amdgpu_vram", &amdgpu_ttm_vram_fops, TTM_PL_VRAM },
2548 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
2549 	{ "amdgpu_gtt", &amdgpu_ttm_gtt_fops, TTM_PL_TT },
2550 #endif
2551 	{ "amdgpu_iomem", &amdgpu_ttm_iomem_fops, TTM_PL_SYSTEM },
2552 };
2553 
2554 #endif
2555 
2556 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
2557 {
2558 #if defined(CONFIG_DEBUG_FS)
2559 	unsigned count;
2560 
2561 	struct drm_minor *minor = adev->ddev->primary;
2562 	struct dentry *ent, *root = minor->debugfs_root;
2563 
2564 	for (count = 0; count < ARRAY_SIZE(ttm_debugfs_entries); count++) {
2565 		ent = debugfs_create_file(
2566 				ttm_debugfs_entries[count].name,
2567 				S_IFREG | S_IRUGO, root,
2568 				adev,
2569 				ttm_debugfs_entries[count].fops);
2570 		if (IS_ERR(ent))
2571 			return PTR_ERR(ent);
2572 		if (ttm_debugfs_entries[count].domain == TTM_PL_VRAM)
2573 			i_size_write(ent->d_inode, adev->gmc.mc_vram_size);
2574 		else if (ttm_debugfs_entries[count].domain == TTM_PL_TT)
2575 			i_size_write(ent->d_inode, adev->gmc.gart_size);
2576 		adev->mman.debugfs_entries[count] = ent;
2577 	}
2578 
2579 	count = ARRAY_SIZE(amdgpu_ttm_debugfs_list);
2580 
2581 #ifdef CONFIG_SWIOTLB
2582 	if (!(adev->need_swiotlb && swiotlb_nr_tbl()))
2583 		--count;
2584 #endif
2585 
2586 	return amdgpu_debugfs_add_files(adev, amdgpu_ttm_debugfs_list, count);
2587 #else
2588 	return 0;
2589 #endif
2590 }
2591 
2592 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev)
2593 {
2594 #if defined(CONFIG_DEBUG_FS)
2595 	unsigned i;
2596 
2597 	for (i = 0; i < ARRAY_SIZE(ttm_debugfs_entries); i++)
2598 		debugfs_remove(adev->mman.debugfs_entries[i]);
2599 #endif
2600 }
2601