xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 /*
2  * Copyright 2009 Jerome Glisse.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sub license, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
20  *
21  * The above copyright notice and this permission notice (including the
22  * next paragraph) shall be included in all copies or substantial portions
23  * of the Software.
24  *
25  */
26 /*
27  * Authors:
28  *    Jerome Glisse <glisse@freedesktop.org>
29  *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
30  *    Dave Airlie
31  */
32 #include <linux/list.h>
33 #include <linux/slab.h>
34 #include <linux/dma-buf.h>
35 #include <linux/export.h>
36 
37 #include <drm/drm_drv.h>
38 #include <drm/amdgpu_drm.h>
39 #include <drm/drm_cache.h>
40 #include "amdgpu.h"
41 #include "amdgpu_trace.h"
42 #include "amdgpu_amdkfd.h"
43 #include "amdgpu_vram_mgr.h"
44 #include "amdgpu_vm.h"
45 #include "amdgpu_dma_buf.h"
46 #include "kfd_svm.h"
47 
48 /**
49  * DOC: amdgpu_object
50  *
51  * This defines the interfaces to operate on an &amdgpu_bo buffer object which
52  * represents memory used by driver (VRAM, system memory, etc.). The driver
53  * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces
54  * to create/destroy/set buffer object which are then managed by the kernel TTM
55  * memory manager.
56  * The interfaces are also used internally by kernel clients, including gfx,
57  * uvd, etc. for kernel managed allocations used by the GPU.
58  *
59  */
60 
61 static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo)
62 {
63 	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
64 
65 	amdgpu_bo_kunmap(bo);
66 
67 	if (drm_gem_is_imported(&bo->tbo.base))
68 		drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg);
69 	drm_gem_object_release(&bo->tbo.base);
70 	amdgpu_bo_unref(&bo->parent);
71 	kvfree(bo);
72 }
73 
74 static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo)
75 {
76 	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
77 	struct amdgpu_bo_user *ubo;
78 
79 	ubo = to_amdgpu_bo_user(bo);
80 	kfree(ubo->metadata);
81 	amdgpu_bo_destroy(tbo);
82 }
83 
84 /**
85  * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo
86  * @bo: buffer object to be checked
87  *
88  * Uses destroy function associated with the object to determine if this is
89  * an &amdgpu_bo.
90  *
91  * Returns:
92  * true if the object belongs to &amdgpu_bo, false if not.
93  */
94 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
95 {
96 	if (bo->destroy == &amdgpu_bo_destroy ||
97 	    bo->destroy == &amdgpu_bo_user_destroy ||
98 	    bo->destroy == &svm_range_bo_destroy)
99 		return true;
100 
101 	return false;
102 }
103 
104 /**
105  * amdgpu_bo_placement_from_domain - set buffer's placement
106  * @abo: &amdgpu_bo buffer object whose placement is to be set
107  * @domain: requested domain
108  *
109  * Sets buffer's placement according to requested domain and the buffer's
110  * flags.
111  */
112 void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
113 {
114 	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
115 	struct ttm_placement *placement = &abo->placement;
116 	struct ttm_place *places = abo->placements;
117 	u64 flags = abo->flags;
118 	u32 c = 0;
119 
120 	if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
121 		unsigned int visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
122 		int8_t mem_id = KFD_XCP_MEM_ID(adev, abo->xcp_id);
123 
124 		if (adev->gmc.mem_partitions && mem_id >= 0) {
125 			places[c].fpfn = adev->gmc.mem_partitions[mem_id].range.fpfn;
126 			/*
127 			 * memory partition range lpfn is inclusive start + size - 1
128 			 * TTM place lpfn is exclusive start + size
129 			 */
130 			places[c].lpfn = adev->gmc.mem_partitions[mem_id].range.lpfn + 1;
131 		} else {
132 			places[c].fpfn = 0;
133 			places[c].lpfn = 0;
134 		}
135 		places[c].mem_type = TTM_PL_VRAM;
136 		places[c].flags = 0;
137 
138 		if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
139 			places[c].lpfn = min_not_zero(places[c].lpfn, visible_pfn);
140 		else
141 			places[c].flags |= TTM_PL_FLAG_TOPDOWN;
142 
143 		if (abo->tbo.type == ttm_bo_type_kernel &&
144 		    flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
145 			places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;
146 
147 		c++;
148 	}
149 
150 	if (domain & AMDGPU_GEM_DOMAIN_DOORBELL) {
151 		places[c].fpfn = 0;
152 		places[c].lpfn = 0;
153 		places[c].mem_type = AMDGPU_PL_DOORBELL;
154 		places[c].flags = 0;
155 		c++;
156 	}
157 
158 	if (domain & AMDGPU_GEM_DOMAIN_GTT) {
159 		places[c].fpfn = 0;
160 		places[c].lpfn = 0;
161 		places[c].mem_type =
162 			abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ?
163 			AMDGPU_PL_PREEMPT : TTM_PL_TT;
164 		places[c].flags = 0;
165 		/*
166 		 * When GTT is just an alternative to VRAM make sure that we
167 		 * only use it as fallback and still try to fill up VRAM first.
168 		 */
169 		if (abo->tbo.resource && !(adev->flags & AMD_IS_APU) &&
170 		    domain & abo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)
171 			places[c].flags |= TTM_PL_FLAG_FALLBACK;
172 		c++;
173 	}
174 
175 	if (domain & AMDGPU_GEM_DOMAIN_CPU) {
176 		places[c].fpfn = 0;
177 		places[c].lpfn = 0;
178 		places[c].mem_type = TTM_PL_SYSTEM;
179 		places[c].flags = 0;
180 		c++;
181 	}
182 
183 	if (domain & AMDGPU_GEM_DOMAIN_GDS) {
184 		places[c].fpfn = 0;
185 		places[c].lpfn = 0;
186 		places[c].mem_type = AMDGPU_PL_GDS;
187 		places[c].flags = 0;
188 		c++;
189 	}
190 
191 	if (domain & AMDGPU_GEM_DOMAIN_GWS) {
192 		places[c].fpfn = 0;
193 		places[c].lpfn = 0;
194 		places[c].mem_type = AMDGPU_PL_GWS;
195 		places[c].flags = 0;
196 		c++;
197 	}
198 
199 	if (domain & AMDGPU_GEM_DOMAIN_OA) {
200 		places[c].fpfn = 0;
201 		places[c].lpfn = 0;
202 		places[c].mem_type = AMDGPU_PL_OA;
203 		places[c].flags = 0;
204 		c++;
205 	}
206 
207 	if (domain & AMDGPU_GEM_DOMAIN_NPA) {
208 		places[c].fpfn = 0;
209 		places[c].lpfn = 0;
210 		places[c].mem_type = AMDGPU_PL_NPA;
211 		places[c].flags = 0;
212 		c++;
213 	}
214 
215 	if (!c) {
216 		places[c].fpfn = 0;
217 		places[c].lpfn = 0;
218 		places[c].mem_type = TTM_PL_SYSTEM;
219 		places[c].flags = 0;
220 		c++;
221 	}
222 
223 	BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS);
224 
225 	placement->num_placement = c;
226 	placement->placement = places;
227 }
228 
229 /**
230  * amdgpu_bo_create_reserved - create reserved BO for kernel use
231  *
232  * @adev: amdgpu device object
233  * @size: size for the new BO
234  * @align: alignment for the new BO
235  * @domain: where to place it
236  * @bo_ptr: used to initialize BOs in structures
237  * @gpu_addr: GPU addr of the pinned BO
238  * @cpu_addr: optional CPU address mapping
239  *
240  * Allocates and pins a BO for kernel internal use, and returns it still
241  * reserved.
242  *
243  * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
244  *
245  * Returns:
246  * 0 on success, negative error code otherwise.
247  */
248 int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
249 			      unsigned long size, int align,
250 			      u32 domain, struct amdgpu_bo **bo_ptr,
251 			      u64 *gpu_addr, void **cpu_addr)
252 {
253 	struct amdgpu_bo_param bp;
254 	bool free = false;
255 	int r;
256 
257 	if (!size) {
258 		amdgpu_bo_unref(bo_ptr);
259 		return 0;
260 	}
261 
262 	memset(&bp, 0, sizeof(bp));
263 	bp.size = size;
264 	bp.byte_align = align;
265 	bp.domain = domain;
266 	bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
267 		: AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
268 	bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
269 	bp.type = ttm_bo_type_kernel;
270 	bp.resv = NULL;
271 	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
272 
273 	if (!*bo_ptr) {
274 		r = amdgpu_bo_create(adev, &bp, bo_ptr);
275 		if (r) {
276 			dev_err(adev->dev, "(%d) failed to allocate kernel bo\n",
277 				r);
278 			return r;
279 		}
280 		free = true;
281 	}
282 
283 	r = amdgpu_bo_reserve(*bo_ptr, false);
284 	if (r) {
285 		dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
286 		goto error_free;
287 	}
288 
289 	if (free) {
290 		r = amdgpu_bo_pin(*bo_ptr, domain);
291 		if (r) {
292 			dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
293 			goto error_unreserve;
294 		}
295 	}
296 
297 	r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo);
298 	if (r) {
299 		dev_err(adev->dev, "%p bind failed\n", *bo_ptr);
300 		goto error_unpin;
301 	}
302 
303 	if (gpu_addr)
304 		*gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr);
305 
306 	if (cpu_addr) {
307 		r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
308 		if (r) {
309 			dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
310 			goto error_unpin;
311 		}
312 	}
313 
314 	return 0;
315 
316 error_unpin:
317 	if (free)
318 		amdgpu_bo_unpin(*bo_ptr);
319 error_unreserve:
320 	amdgpu_bo_unreserve(*bo_ptr);
321 
322 error_free:
323 	if (free)
324 		amdgpu_bo_unref(bo_ptr);
325 
326 	return r;
327 }
328 
329 /**
330  * amdgpu_bo_create_kernel - create BO for kernel use
331  *
332  * @adev: amdgpu device object
333  * @size: size for the new BO
334  * @align: alignment for the new BO
335  * @domain: where to place it
336  * @bo_ptr:  used to initialize BOs in structures
337  * @gpu_addr: GPU addr of the pinned BO
338  * @cpu_addr: optional CPU address mapping
339  *
340  * Allocates and pins a BO for kernel internal use.
341  *
342  * This function is exported to allow the V4L2 isp device
343  * external to drm device to create and access the kernel BO.
344  *
345  * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
346  *
347  * Returns:
348  * 0 on success, negative error code otherwise.
349  */
350 int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
351 			    unsigned long size, int align,
352 			    u32 domain, struct amdgpu_bo **bo_ptr,
353 			    u64 *gpu_addr, void **cpu_addr)
354 {
355 	int r;
356 
357 	r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr,
358 				      gpu_addr, cpu_addr);
359 
360 	if (r)
361 		return r;
362 
363 	if (*bo_ptr)
364 		amdgpu_bo_unreserve(*bo_ptr);
365 
366 	return 0;
367 }
368 
369 /**
370  * amdgpu_bo_create_isp_user - create user BO for isp
371  *
372  * @adev: amdgpu device object
373  * @dma_buf: DMABUF handle for isp buffer
374  * @domain: where to place it
375  * @bo:  used to initialize BOs in structures
376  * @gpu_addr: GPU addr of the pinned BO
377  *
378  * Imports isp DMABUF to allocate and pin a user BO for isp internal use. It does
379  * GART alloc to generate gpu_addr for BO to make it accessible through the
380  * GART aperture for ISP HW.
381  *
382  * This function is exported to allow the V4L2 isp device external to drm device
383  * to create and access the isp user BO.
384  *
385  * Returns:
386  * 0 on success, negative error code otherwise.
387  */
388 int amdgpu_bo_create_isp_user(struct amdgpu_device *adev,
389 			   struct dma_buf *dma_buf, u32 domain, struct amdgpu_bo **bo,
390 			   u64 *gpu_addr)
391 
392 {
393 	struct drm_gem_object *gem_obj;
394 	int r;
395 
396 	gem_obj = amdgpu_gem_prime_import(&adev->ddev, dma_buf);
397 	*bo = gem_to_amdgpu_bo(gem_obj);
398 	if (!(*bo)) {
399 		dev_err(adev->dev, "failed to get valid isp user bo\n");
400 		return -EINVAL;
401 	}
402 
403 	r = amdgpu_bo_reserve(*bo, false);
404 	if (r) {
405 		dev_err(adev->dev, "(%d) failed to reserve isp user bo\n", r);
406 		return r;
407 	}
408 
409 	r = amdgpu_bo_pin(*bo, domain);
410 	if (r) {
411 		dev_err(adev->dev, "(%d) isp user bo pin failed\n", r);
412 		goto error_unreserve;
413 	}
414 
415 	r = amdgpu_ttm_alloc_gart(&(*bo)->tbo);
416 	if (r) {
417 		dev_err(adev->dev, "%p bind failed\n", *bo);
418 		goto error_unpin;
419 	}
420 
421 	if (!WARN_ON(!gpu_addr))
422 		*gpu_addr = amdgpu_bo_gpu_offset(*bo);
423 
424 	amdgpu_bo_unreserve(*bo);
425 
426 	return 0;
427 
428 error_unpin:
429 	amdgpu_bo_unpin(*bo);
430 error_unreserve:
431 	amdgpu_bo_unreserve(*bo);
432 	amdgpu_bo_unref(bo);
433 
434 	return r;
435 }
436 
437 /**
438  * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location
439  *
440  * @adev: amdgpu device object
441  * @offset: offset of the BO
442  * @size: size of the BO
443  * @bo_ptr:  used to initialize BOs in structures
444  * @cpu_addr: optional CPU address mapping
445  *
446  * Creates a kernel BO at a specific offset in VRAM.
447  *
448  * Returns:
449  * 0 on success, negative error code otherwise.
450  */
451 int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev,
452 			       uint64_t offset, uint64_t size,
453 			       struct amdgpu_bo **bo_ptr, void **cpu_addr)
454 {
455 	struct ttm_operation_ctx ctx = { false, false };
456 	unsigned int i;
457 	int r;
458 
459 	offset &= PAGE_MASK;
460 	size = ALIGN(size, PAGE_SIZE);
461 
462 	r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE,
463 				      AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL,
464 				      cpu_addr);
465 	if (r)
466 		return r;
467 
468 	if ((*bo_ptr) == NULL)
469 		return 0;
470 
471 	/*
472 	 * Remove the original mem node and create a new one at the request
473 	 * position.
474 	 */
475 	if (cpu_addr)
476 		amdgpu_bo_kunmap(*bo_ptr);
477 
478 	ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource);
479 
480 	for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) {
481 		(*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT;
482 		(*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
483 	}
484 	r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement,
485 			     &(*bo_ptr)->tbo.resource, &ctx);
486 	if (r)
487 		goto error;
488 
489 	if (cpu_addr) {
490 		r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
491 		if (r)
492 			goto error;
493 	}
494 
495 	amdgpu_bo_unreserve(*bo_ptr);
496 	return 0;
497 
498 error:
499 	amdgpu_bo_unreserve(*bo_ptr);
500 	amdgpu_bo_unref(bo_ptr);
501 	return r;
502 }
503 
504 /**
505  * amdgpu_bo_free_kernel - free BO for kernel use
506  *
507  * @bo: amdgpu BO to free
508  * @gpu_addr: pointer to where the BO's GPU memory space address was stored
509  * @cpu_addr: pointer to where the BO's CPU memory space address was stored
510  *
511  * unmaps and unpin a BO for kernel internal use.
512  *
513  * This function is exported to allow the V4L2 isp device
514  * external to drm device to free the kernel BO.
515  */
516 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
517 			   void **cpu_addr)
518 {
519 	if (*bo == NULL)
520 		return;
521 
522 	WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend);
523 
524 	if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {
525 		if (cpu_addr)
526 			amdgpu_bo_kunmap(*bo);
527 
528 		amdgpu_bo_unpin(*bo);
529 		amdgpu_bo_unreserve(*bo);
530 	}
531 	amdgpu_bo_unref(bo);
532 
533 	if (gpu_addr)
534 		*gpu_addr = 0;
535 
536 	if (cpu_addr)
537 		*cpu_addr = NULL;
538 }
539 
540 /**
541  * amdgpu_bo_free_isp_user - free BO for isp use
542  *
543  * @bo: amdgpu isp user BO to free
544  *
545  * unpin and unref BO for isp internal use.
546  *
547  * This function is exported to allow the V4L2 isp device
548  * external to drm device to free the isp user BO.
549  */
550 void amdgpu_bo_free_isp_user(struct amdgpu_bo *bo)
551 {
552 	if (bo == NULL)
553 		return;
554 
555 	if (amdgpu_bo_reserve(bo, true) == 0) {
556 		amdgpu_bo_unpin(bo);
557 		amdgpu_bo_unreserve(bo);
558 	}
559 	amdgpu_bo_unref(&bo);
560 }
561 
562 /* Validate bo size is bit bigger than the request domain */
563 static bool amdgpu_bo_validate_size(struct amdgpu_device *adev,
564 					  unsigned long size, u32 domain)
565 {
566 	struct ttm_resource_manager *man = NULL;
567 
568 	/*
569 	 * If GTT is part of requested domains the check must succeed to
570 	 * allow fall back to GTT.
571 	 */
572 	if (domain & AMDGPU_GEM_DOMAIN_GTT)
573 		man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
574 	else if (domain & AMDGPU_GEM_DOMAIN_VRAM)
575 		man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
576 	else
577 		return true;
578 
579 	if (!man) {
580 		if (domain & AMDGPU_GEM_DOMAIN_GTT)
581 			WARN_ON_ONCE("GTT domain requested but GTT mem manager uninitialized");
582 		return false;
583 	}
584 
585 	/* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU, _DOMAIN_DOORBELL */
586 	if (size < man->size)
587 		return true;
588 
589 	DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, man->size);
590 	return false;
591 }
592 
593 bool amdgpu_bo_support_uswc(u64 bo_flags)
594 {
595 
596 #ifdef CONFIG_X86_32
597 	/* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit
598 	 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627
599 	 */
600 	return false;
601 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
602 	/* Don't try to enable write-combining when it can't work, or things
603 	 * may be slow
604 	 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758
605 	 */
606 
607 #ifndef CONFIG_COMPILE_TEST
608 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
609 	 thanks to write-combining
610 #endif
611 
612 	if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
613 		DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
614 			      "better performance thanks to write-combining\n");
615 	return false;
616 #else
617 	/* For architectures that don't support WC memory,
618 	 * mask out the WC flag from the BO
619 	 */
620 	if (!drm_arch_can_wc_memory())
621 		return false;
622 
623 	return true;
624 #endif
625 }
626 
627 /**
628  * amdgpu_bo_create - create an &amdgpu_bo buffer object
629  * @adev: amdgpu device object
630  * @bp: parameters to be used for the buffer object
631  * @bo_ptr: pointer to the buffer object pointer
632  *
633  * Creates an &amdgpu_bo buffer object.
634  *
635  * Returns:
636  * 0 for success or a negative error code on failure.
637  */
638 int amdgpu_bo_create(struct amdgpu_device *adev,
639 			       struct amdgpu_bo_param *bp,
640 			       struct amdgpu_bo **bo_ptr)
641 {
642 	struct ttm_operation_ctx ctx = {
643 		.interruptible = (bp->type != ttm_bo_type_kernel),
644 		.no_wait_gpu = bp->no_wait_gpu,
645 		/* We opt to avoid OOM on system pages allocations */
646 		.gfp_retry_mayfail = true,
647 		.allow_res_evict = bp->type != ttm_bo_type_kernel,
648 		.resv = bp->resv
649 	};
650 	struct amdgpu_bo *bo;
651 	unsigned long page_align, size = bp->size;
652 	int r;
653 
654 	/* Note that GDS/GWS/OA allocates 1 page per byte/resource. */
655 	if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
656 		/* GWS and OA don't need any alignment. */
657 		page_align = bp->byte_align;
658 		size <<= PAGE_SHIFT;
659 
660 	} else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) {
661 		/* Both size and alignment must be a multiple of 4. */
662 		page_align = ALIGN(bp->byte_align, 4);
663 		size = ALIGN(size, 4) << PAGE_SHIFT;
664 	} else {
665 		/* Memory should be aligned at least to a page size. */
666 		page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT;
667 		size = ALIGN(size, PAGE_SIZE);
668 	}
669 
670 	if (!amdgpu_bo_validate_size(adev, size, bp->domain))
671 		return -ENOMEM;
672 
673 	BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo));
674 
675 	*bo_ptr = NULL;
676 	bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL);
677 	if (bo == NULL)
678 		return -ENOMEM;
679 	drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size);
680 	bo->tbo.base.funcs = &amdgpu_gem_object_funcs;
681 	bo->vm_bo = NULL;
682 	bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain :
683 		bp->domain;
684 	bo->allowed_domains = bo->preferred_domains;
685 	if (bp->type != ttm_bo_type_kernel &&
686 	    !(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE) &&
687 	    bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
688 		bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
689 
690 	bo->flags = bp->flags;
691 
692 	if (adev->gmc.mem_partitions)
693 		/* For GPUs with spatial partitioning, bo->xcp_id=-1 means any partition */
694 		bo->xcp_id = bp->xcp_id_plus1 - 1;
695 	else
696 		/* For GPUs without spatial partitioning */
697 		bo->xcp_id = 0;
698 
699 	if (!amdgpu_bo_support_uswc(bo->flags))
700 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
701 
702 	bo->tbo.bdev = &adev->mman.bdev;
703 	if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA |
704 			  AMDGPU_GEM_DOMAIN_GDS))
705 		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
706 	else
707 		amdgpu_bo_placement_from_domain(bo, bp->domain);
708 	if (bp->type == ttm_bo_type_kernel)
709 		bo->tbo.priority = 2;
710 	else if (!(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE))
711 		bo->tbo.priority = 1;
712 
713 	if (!bp->destroy)
714 		bp->destroy = &amdgpu_bo_destroy;
715 
716 	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type,
717 				 &bo->placement, page_align, &ctx,  NULL,
718 				 bp->resv, bp->destroy);
719 	if (unlikely(r != 0))
720 		return r;
721 
722 	if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
723 	    amdgpu_res_cpu_visible(adev, bo->tbo.resource))
724 		amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved,
725 					     ctx.bytes_moved);
726 	else
727 		amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0);
728 
729 	if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
730 	    bo->tbo.resource->mem_type == TTM_PL_VRAM) {
731 		struct dma_fence *fence;
732 
733 		r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev),
734 					    bo, bo->tbo.base.resv, &fence,
735 					    true, AMDGPU_KERNEL_JOB_ID_TTM_CLEAR_BUFFER);
736 		if (unlikely(r))
737 			goto fail_unreserve;
738 
739 		if (fence) {
740 			dma_resv_add_fence(bo->tbo.base.resv, fence,
741 					   DMA_RESV_USAGE_KERNEL);
742 			dma_fence_put(fence);
743 		}
744 	}
745 	if (!bp->resv)
746 		amdgpu_bo_unreserve(bo);
747 	*bo_ptr = bo;
748 
749 	trace_amdgpu_bo_create(bo);
750 
751 	/* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */
752 	if (bp->type == ttm_bo_type_device)
753 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
754 
755 	return 0;
756 
757 fail_unreserve:
758 	if (!bp->resv)
759 		dma_resv_unlock(bo->tbo.base.resv);
760 	amdgpu_bo_unref(&bo);
761 	return r;
762 }
763 
764 /**
765  * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object
766  * @adev: amdgpu device object
767  * @bp: parameters to be used for the buffer object
768  * @ubo_ptr: pointer to the buffer object pointer
769  *
770  * Create a BO to be used by user application;
771  *
772  * Returns:
773  * 0 for success or a negative error code on failure.
774  */
775 
776 int amdgpu_bo_create_user(struct amdgpu_device *adev,
777 			  struct amdgpu_bo_param *bp,
778 			  struct amdgpu_bo_user **ubo_ptr)
779 {
780 	struct amdgpu_bo *bo_ptr;
781 	int r;
782 
783 	bp->bo_ptr_size = sizeof(struct amdgpu_bo_user);
784 	bp->destroy = &amdgpu_bo_user_destroy;
785 	r = amdgpu_bo_create(adev, bp, &bo_ptr);
786 	if (r)
787 		return r;
788 
789 	*ubo_ptr = to_amdgpu_bo_user(bo_ptr);
790 	return r;
791 }
792 
793 /**
794  * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object
795  * @adev: amdgpu device object
796  * @bp: parameters to be used for the buffer object
797  * @vmbo_ptr: pointer to the buffer object pointer
798  *
799  * Create a BO to be for GPUVM.
800  *
801  * Returns:
802  * 0 for success or a negative error code on failure.
803  */
804 
805 int amdgpu_bo_create_vm(struct amdgpu_device *adev,
806 			struct amdgpu_bo_param *bp,
807 			struct amdgpu_bo_vm **vmbo_ptr)
808 {
809 	struct amdgpu_bo *bo_ptr;
810 	int r;
811 
812 	/* bo_ptr_size will be determined by the caller and it depends on
813 	 * num of amdgpu_vm_pt entries.
814 	 */
815 	BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm));
816 	r = amdgpu_bo_create(adev, bp, &bo_ptr);
817 	if (r)
818 		return r;
819 
820 	*vmbo_ptr = to_amdgpu_bo_vm(bo_ptr);
821 	return r;
822 }
823 
824 /**
825  * amdgpu_bo_kmap - map an &amdgpu_bo buffer object
826  * @bo: &amdgpu_bo buffer object to be mapped
827  * @ptr: kernel virtual address to be returned
828  *
829  * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls
830  * amdgpu_bo_kptr() to get the kernel virtual address.
831  *
832  * Returns:
833  * 0 for success or a negative error code on failure.
834  */
835 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
836 {
837 	void *kptr;
838 	long r;
839 
840 	if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
841 		return -EPERM;
842 
843 	r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL,
844 				  false, MAX_SCHEDULE_TIMEOUT);
845 	if (r < 0)
846 		return r;
847 
848 	kptr = amdgpu_bo_kptr(bo);
849 	if (kptr) {
850 		if (ptr)
851 			*ptr = kptr;
852 		return 0;
853 	}
854 
855 	r = ttm_bo_kmap(&bo->tbo, 0, PFN_UP(bo->tbo.base.size), &bo->kmap);
856 	if (r)
857 		return r;
858 
859 	if (ptr)
860 		*ptr = amdgpu_bo_kptr(bo);
861 
862 	return 0;
863 }
864 
865 /**
866  * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object
867  * @bo: &amdgpu_bo buffer object
868  *
869  * Calls ttm_kmap_obj_virtual() to get the kernel virtual address
870  *
871  * Returns:
872  * the virtual address of a buffer object area.
873  */
874 void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
875 {
876 	bool is_iomem;
877 
878 	return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
879 }
880 
881 /**
882  * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object
883  * @bo: &amdgpu_bo buffer object to be unmapped
884  *
885  * Unmaps a kernel map set up by amdgpu_bo_kmap().
886  */
887 void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
888 {
889 	if (bo->kmap.bo)
890 		ttm_bo_kunmap(&bo->kmap);
891 }
892 
893 /**
894  * amdgpu_bo_ref - reference an &amdgpu_bo buffer object
895  * @bo: &amdgpu_bo buffer object
896  *
897  * References the contained &ttm_buffer_object.
898  *
899  * Returns:
900  * a refcounted pointer to the &amdgpu_bo buffer object.
901  */
902 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
903 {
904 	if (bo == NULL)
905 		return NULL;
906 
907 	drm_gem_object_get(&bo->tbo.base);
908 	return bo;
909 }
910 
911 /**
912  * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object
913  * @bo: &amdgpu_bo buffer object
914  *
915  * Unreferences the contained &ttm_buffer_object and clear the pointer
916  */
917 void amdgpu_bo_unref(struct amdgpu_bo **bo)
918 {
919 	if ((*bo) == NULL)
920 		return;
921 
922 	drm_gem_object_put(&(*bo)->tbo.base);
923 	*bo = NULL;
924 }
925 
926 /**
927  * amdgpu_bo_pin - pin an &amdgpu_bo buffer object
928  * @bo: &amdgpu_bo buffer object to be pinned
929  * @domain: domain to be pinned to
930  *
931  * Pins the buffer object according to requested domain. If the memory is
932  * unbound gart memory, binds the pages into gart table. Adjusts pin_count and
933  * pin_size accordingly.
934  *
935  * Pinning means to lock pages in memory along with keeping them at a fixed
936  * offset. It is required when a buffer can not be moved, for example, when
937  * a display buffer is being scanned out.
938  *
939  * Returns:
940  * 0 for success or a negative error code on failure.
941  */
942 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain)
943 {
944 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
945 	struct ttm_operation_ctx ctx = { false, false };
946 	int r, i;
947 
948 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
949 		return -EPERM;
950 
951 	/* Check domain to be pinned to against preferred domains */
952 	if (bo->preferred_domains & domain)
953 		domain = bo->preferred_domains & domain;
954 
955 	/* A shared bo cannot be migrated to VRAM */
956 	if (drm_gem_is_imported(&bo->tbo.base)) {
957 		if (domain & AMDGPU_GEM_DOMAIN_GTT)
958 			domain = AMDGPU_GEM_DOMAIN_GTT;
959 		else
960 			return -EINVAL;
961 	}
962 
963 	if (bo->tbo.pin_count) {
964 		uint32_t mem_type = bo->tbo.resource->mem_type;
965 		uint32_t mem_flags = bo->tbo.resource->placement;
966 
967 		if (!(domain & amdgpu_mem_type_to_domain(mem_type)))
968 			return -EINVAL;
969 
970 		if ((mem_type == TTM_PL_VRAM) &&
971 		    (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) &&
972 		    !(mem_flags & TTM_PL_FLAG_CONTIGUOUS))
973 			return -EINVAL;
974 
975 		ttm_bo_pin(&bo->tbo);
976 		return 0;
977 	}
978 
979 	/* This assumes only APU display buffers are pinned with (VRAM|GTT).
980 	 * See function amdgpu_display_supported_domains()
981 	 */
982 	domain = amdgpu_bo_get_preferred_domain(adev, domain);
983 
984 	if (drm_gem_is_imported(&bo->tbo.base))
985 		dma_buf_pin(bo->tbo.base.import_attach);
986 
987 	/* force to pin into visible video ram */
988 	if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))
989 		bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
990 	amdgpu_bo_placement_from_domain(bo, domain);
991 	for (i = 0; i < bo->placement.num_placement; i++) {
992 		if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS &&
993 		    bo->placements[i].mem_type == TTM_PL_VRAM)
994 			bo->placements[i].flags |= TTM_PL_FLAG_CONTIGUOUS;
995 	}
996 
997 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
998 	if (unlikely(r)) {
999 		dev_err(adev->dev, "%p pin failed\n", bo);
1000 		goto error;
1001 	}
1002 
1003 	ttm_bo_pin(&bo->tbo);
1004 
1005 	if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {
1006 		atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size);
1007 		atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo),
1008 			     &adev->visible_pin_size);
1009 	} else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
1010 		atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size);
1011 	}
1012 
1013 error:
1014 	return r;
1015 }
1016 
1017 /**
1018  * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object
1019  * @bo: &amdgpu_bo buffer object to be unpinned
1020  *
1021  * Decreases the pin_count, and clears the flags if pin_count reaches 0.
1022  * Changes placement and pin size accordingly.
1023  *
1024  * Returns:
1025  * 0 for success or a negative error code on failure.
1026  */
1027 void amdgpu_bo_unpin(struct amdgpu_bo *bo)
1028 {
1029 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1030 
1031 	ttm_bo_unpin(&bo->tbo);
1032 	if (bo->tbo.pin_count)
1033 		return;
1034 
1035 	if (drm_gem_is_imported(&bo->tbo.base))
1036 		dma_buf_unpin(bo->tbo.base.import_attach);
1037 
1038 	if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {
1039 		atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size);
1040 		atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo),
1041 			     &adev->visible_pin_size);
1042 	} else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
1043 		atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size);
1044 	}
1045 
1046 }
1047 
1048 static const char * const amdgpu_vram_names[] = {
1049 	"UNKNOWN",
1050 	"GDDR1",
1051 	"DDR2",
1052 	"GDDR3",
1053 	"GDDR4",
1054 	"GDDR5",
1055 	"HBM",
1056 	"DDR3",
1057 	"DDR4",
1058 	"GDDR6",
1059 	"DDR5",
1060 	"LPDDR4",
1061 	"LPDDR5",
1062 	"HBM3E",
1063 	"HBM4",
1064 	"GDDR7"
1065 };
1066 
1067 /**
1068  * amdgpu_bo_init - initialize memory manager
1069  * @adev: amdgpu device object
1070  *
1071  * Calls amdgpu_ttm_init() to initialize amdgpu memory manager.
1072  *
1073  * Returns:
1074  * 0 for success or a negative error code on failure.
1075  */
1076 int amdgpu_bo_init(struct amdgpu_device *adev)
1077 {
1078 	/* On A+A platform, VRAM can be mapped as WB */
1079 	if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
1080 		/* reserve PAT memory space to WC for VRAM */
1081 		int r = arch_io_reserve_memtype_wc(adev->gmc.aper_base,
1082 				adev->gmc.aper_size);
1083 
1084 		if (r) {
1085 			DRM_ERROR("Unable to set WC memtype for the aperture base\n");
1086 			return r;
1087 		}
1088 
1089 		/* Add an MTRR for the VRAM */
1090 		adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base,
1091 				adev->gmc.aper_size);
1092 	}
1093 
1094 	drm_info(adev_to_drm(adev), "Detected VRAM RAM=%lluM, BAR=%lluM\n",
1095 		 adev->gmc.mc_vram_size >> 20,
1096 		 (unsigned long long)adev->gmc.aper_size >> 20);
1097 	drm_info(adev_to_drm(adev), "RAM width %dbits %s\n",
1098 		 adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]);
1099 	return amdgpu_ttm_init(adev);
1100 }
1101 
1102 /**
1103  * amdgpu_bo_fini - tear down memory manager
1104  * @adev: amdgpu device object
1105  *
1106  * Reverses amdgpu_bo_init() to tear down memory manager.
1107  */
1108 void amdgpu_bo_fini(struct amdgpu_device *adev)
1109 {
1110 	int idx;
1111 
1112 	amdgpu_ttm_fini(adev);
1113 
1114 	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
1115 		if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
1116 			arch_phys_wc_del(adev->gmc.vram_mtrr);
1117 			arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
1118 		}
1119 		drm_dev_exit(idx);
1120 	}
1121 }
1122 
1123 /**
1124  * amdgpu_bo_set_tiling_flags - set tiling flags
1125  * @bo: &amdgpu_bo buffer object
1126  * @tiling_flags: new flags
1127  *
1128  * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or
1129  * kernel driver to set the tiling flags on a buffer.
1130  *
1131  * Returns:
1132  * 0 for success or a negative error code on failure.
1133  */
1134 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
1135 {
1136 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1137 	struct amdgpu_bo_user *ubo;
1138 
1139 	/* MMIO_REMAP is BAR I/O space; tiling should never be used here. */
1140 	WARN_ON_ONCE(bo->tbo.resource &&
1141 		     bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP);
1142 
1143 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1144 	if (adev->family <= AMDGPU_FAMILY_CZ &&
1145 	    AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
1146 		return -EINVAL;
1147 
1148 	ubo = to_amdgpu_bo_user(bo);
1149 	ubo->tiling_flags = tiling_flags;
1150 	return 0;
1151 }
1152 
1153 /**
1154  * amdgpu_bo_get_tiling_flags - get tiling flags
1155  * @bo: &amdgpu_bo buffer object
1156  * @tiling_flags: returned flags
1157  *
1158  * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to
1159  * set the tiling flags on a buffer.
1160  */
1161 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
1162 {
1163 	struct amdgpu_bo_user *ubo;
1164 
1165 	/*
1166 	 * MMIO_REMAP BOs are not real VRAM/GTT memory but a fixed BAR I/O window.
1167 	 * They should never go through GEM tiling helpers.
1168 	 */
1169 	WARN_ON_ONCE(bo->tbo.resource &&
1170 		     bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP);
1171 
1172 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1173 	dma_resv_assert_held(bo->tbo.base.resv);
1174 	ubo = to_amdgpu_bo_user(bo);
1175 
1176 	if (tiling_flags)
1177 		*tiling_flags = ubo->tiling_flags;
1178 }
1179 
1180 /**
1181  * amdgpu_bo_set_metadata - set metadata
1182  * @bo: &amdgpu_bo buffer object
1183  * @metadata: new metadata
1184  * @metadata_size: size of the new metadata
1185  * @flags: flags of the new metadata
1186  *
1187  * Sets buffer object's metadata, its size and flags.
1188  * Used via GEM ioctl.
1189  *
1190  * Returns:
1191  * 0 for success or a negative error code on failure.
1192  */
1193 int amdgpu_bo_set_metadata(struct amdgpu_bo *bo, void *metadata,
1194 			   u32 metadata_size, uint64_t flags)
1195 {
1196 	struct amdgpu_bo_user *ubo;
1197 	void *buffer;
1198 
1199 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1200 	ubo = to_amdgpu_bo_user(bo);
1201 	if (!metadata_size) {
1202 		if (ubo->metadata_size) {
1203 			kfree(ubo->metadata);
1204 			ubo->metadata = NULL;
1205 			ubo->metadata_size = 0;
1206 		}
1207 		return 0;
1208 	}
1209 
1210 	if (metadata == NULL)
1211 		return -EINVAL;
1212 
1213 	buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
1214 	if (buffer == NULL)
1215 		return -ENOMEM;
1216 
1217 	kfree(ubo->metadata);
1218 	ubo->metadata_flags = flags;
1219 	ubo->metadata = buffer;
1220 	ubo->metadata_size = metadata_size;
1221 
1222 	return 0;
1223 }
1224 
1225 /**
1226  * amdgpu_bo_get_metadata - get metadata
1227  * @bo: &amdgpu_bo buffer object
1228  * @buffer: returned metadata
1229  * @buffer_size: size of the buffer
1230  * @metadata_size: size of the returned metadata
1231  * @flags: flags of the returned metadata
1232  *
1233  * Gets buffer object's metadata, its size and flags. buffer_size shall not be
1234  * less than metadata_size.
1235  * Used via GEM ioctl.
1236  *
1237  * Returns:
1238  * 0 for success or a negative error code on failure.
1239  */
1240 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
1241 			   size_t buffer_size, uint32_t *metadata_size,
1242 			   uint64_t *flags)
1243 {
1244 	struct amdgpu_bo_user *ubo;
1245 
1246 	if (!buffer && !metadata_size)
1247 		return -EINVAL;
1248 
1249 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1250 	ubo = to_amdgpu_bo_user(bo);
1251 	if (metadata_size)
1252 		*metadata_size = ubo->metadata_size;
1253 
1254 	if (buffer) {
1255 		if (buffer_size < ubo->metadata_size)
1256 			return -EINVAL;
1257 
1258 		if (ubo->metadata_size)
1259 			memcpy(buffer, ubo->metadata, ubo->metadata_size);
1260 	}
1261 
1262 	if (flags)
1263 		*flags = ubo->metadata_flags;
1264 
1265 	return 0;
1266 }
1267 
1268 /**
1269  * amdgpu_bo_move_notify - notification about a memory move
1270  * @bo: pointer to a buffer object
1271  * @evict: if this move is evicting the buffer from the graphics address space
1272  * @new_mem: new resource for backing the BO
1273  *
1274  * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs
1275  * bookkeeping.
1276  * TTM driver callback which is called when ttm moves a buffer.
1277  */
1278 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
1279 			   bool evict,
1280 			   struct ttm_resource *new_mem)
1281 {
1282 	struct ttm_resource *old_mem = bo->resource;
1283 	struct amdgpu_bo *abo;
1284 
1285 	if (!amdgpu_bo_is_amdgpu_bo(bo))
1286 		return;
1287 
1288 	abo = ttm_to_amdgpu_bo(bo);
1289 	amdgpu_vm_bo_move(abo, new_mem, evict);
1290 
1291 	amdgpu_bo_kunmap(abo);
1292 
1293 	if (abo->tbo.base.dma_buf && !drm_gem_is_imported(&abo->tbo.base) &&
1294 	    old_mem && old_mem->mem_type != TTM_PL_SYSTEM)
1295 		dma_buf_invalidate_mappings(abo->tbo.base.dma_buf);
1296 
1297 	/* move_notify is called before move happens */
1298 	trace_amdgpu_bo_move(abo, new_mem ? new_mem->mem_type : -1,
1299 			     old_mem ? old_mem->mem_type : -1);
1300 }
1301 
1302 /**
1303  * amdgpu_bo_release_notify - notification about a BO being released
1304  * @bo: pointer to a buffer object
1305  *
1306  * Wipes VRAM buffers whose contents should not be leaked before the
1307  * memory is released.
1308  */
1309 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)
1310 {
1311 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1312 	struct dma_fence *fence = NULL;
1313 	struct amdgpu_bo *abo;
1314 	int r;
1315 
1316 	if (!amdgpu_bo_is_amdgpu_bo(bo))
1317 		return;
1318 
1319 	abo = ttm_to_amdgpu_bo(bo);
1320 
1321 	WARN_ON(abo->vm_bo);
1322 
1323 	if (abo->kfd_bo)
1324 		amdgpu_amdkfd_release_notify(abo);
1325 
1326 	/*
1327 	 * We lock the private dma_resv object here and since the BO is about to
1328 	 * be released nobody else should have a pointer to it.
1329 	 * So when this locking here fails something is wrong with the reference
1330 	 * counting.
1331 	 */
1332 	if (WARN_ON_ONCE(!dma_resv_trylock(&bo->base._resv)))
1333 		return;
1334 
1335 	amdgpu_amdkfd_remove_all_eviction_fences(abo);
1336 
1337 	if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM ||
1338 	    !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) ||
1339 	    adev->in_suspend || drm_dev_is_unplugged(adev_to_drm(adev)))
1340 		goto out;
1341 
1342 	r = dma_resv_reserve_fences(&bo->base._resv, 1);
1343 	if (r)
1344 		goto out;
1345 
1346 	r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev),
1347 				    abo, &bo->base._resv, &fence,
1348 				    false, AMDGPU_KERNEL_JOB_ID_CLEAR_ON_RELEASE);
1349 	if (WARN_ON(r))
1350 		goto out;
1351 
1352 	amdgpu_vram_mgr_set_cleared(bo->resource);
1353 	dma_resv_add_fence(&bo->base._resv, fence, DMA_RESV_USAGE_KERNEL);
1354 	dma_fence_put(fence);
1355 
1356 out:
1357 	dma_resv_unlock(&bo->base._resv);
1358 }
1359 
1360 /**
1361  * amdgpu_bo_fault_reserve_notify - notification about a memory fault
1362  * @bo: pointer to a buffer object
1363  *
1364  * Notifies the driver we are taking a fault on this BO and have reserved it,
1365  * also performs bookkeeping.
1366  * TTM driver callback for dealing with vm faults.
1367  *
1368  * Returns:
1369  * 0 for success or a negative error code on failure.
1370  */
1371 vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
1372 {
1373 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1374 	struct ttm_operation_ctx ctx = { false, false };
1375 	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1376 	int r;
1377 
1378 	/* Remember that this BO was accessed by the CPU */
1379 	abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1380 
1381 	if (amdgpu_res_cpu_visible(adev, bo->resource))
1382 		return 0;
1383 
1384 	/* Can't move a pinned BO to visible VRAM */
1385 	if (abo->tbo.pin_count > 0)
1386 		return VM_FAULT_SIGBUS;
1387 
1388 	/* hurrah the memory is not visible ! */
1389 	atomic64_inc(&adev->num_vram_cpu_page_faults);
1390 	amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
1391 					AMDGPU_GEM_DOMAIN_GTT);
1392 
1393 	/* Avoid costly evictions; only set GTT as a busy placement */
1394 	abo->placements[0].flags |= TTM_PL_FLAG_DESIRED;
1395 
1396 	r = ttm_bo_validate(bo, &abo->placement, &ctx);
1397 	if (unlikely(r == -EBUSY || r == -ERESTARTSYS))
1398 		return VM_FAULT_NOPAGE;
1399 	else if (unlikely(r))
1400 		return VM_FAULT_SIGBUS;
1401 
1402 	/* this should never happen */
1403 	if (bo->resource->mem_type == TTM_PL_VRAM &&
1404 	    !amdgpu_res_cpu_visible(adev, bo->resource))
1405 		return VM_FAULT_SIGBUS;
1406 
1407 	ttm_bo_move_to_lru_tail_unlocked(bo);
1408 	return 0;
1409 }
1410 
1411 /**
1412  * amdgpu_bo_fence - add fence to buffer object
1413  *
1414  * @bo: buffer object in question
1415  * @fence: fence to add
1416  * @shared: true if fence should be added shared
1417  *
1418  */
1419 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
1420 		     bool shared)
1421 {
1422 	struct dma_resv *resv = bo->tbo.base.resv;
1423 	int r;
1424 
1425 	r = dma_resv_reserve_fences(resv, 1);
1426 	if (r) {
1427 		/* As last resort on OOM we block for the fence */
1428 		dma_fence_wait(fence, false);
1429 		return;
1430 	}
1431 
1432 	dma_resv_add_fence(resv, fence, shared ? DMA_RESV_USAGE_READ :
1433 			   DMA_RESV_USAGE_WRITE);
1434 }
1435 
1436 /**
1437  * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences
1438  *
1439  * @adev: amdgpu device pointer
1440  * @resv: reservation object to sync to
1441  * @sync_mode: synchronization mode
1442  * @owner: fence owner
1443  * @intr: Whether the wait is interruptible
1444  *
1445  * Extract the fences from the reservation object and waits for them to finish.
1446  *
1447  * Returns:
1448  * 0 on success, errno otherwise.
1449  */
1450 int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv,
1451 			     enum amdgpu_sync_mode sync_mode, void *owner,
1452 			     bool intr)
1453 {
1454 	struct amdgpu_sync sync;
1455 	int r;
1456 
1457 	amdgpu_sync_create(&sync);
1458 	amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner);
1459 	r = amdgpu_sync_wait(&sync, intr);
1460 	amdgpu_sync_free(&sync);
1461 	return r;
1462 }
1463 
1464 /**
1465  * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv
1466  * @bo: buffer object to wait for
1467  * @owner: fence owner
1468  * @intr: Whether the wait is interruptible
1469  *
1470  * Wrapper to wait for fences in a BO.
1471  * Returns:
1472  * 0 on success, errno otherwise.
1473  */
1474 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr)
1475 {
1476 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1477 
1478 	return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv,
1479 					AMDGPU_SYNC_NE_OWNER, owner, intr);
1480 }
1481 
1482 /**
1483  * amdgpu_bo_gpu_offset - return GPU offset of bo
1484  * @bo:	amdgpu object for which we query the offset
1485  *
1486  * Note: object should either be pinned or reserved when calling this
1487  * function, it might be useful to add check for this for debugging.
1488  *
1489  * Returns:
1490  * current GPU offset of the object.
1491  */
1492 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
1493 {
1494 	WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM);
1495 	WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) &&
1496 		     !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel);
1497 	WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET);
1498 	WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM &&
1499 		     !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));
1500 
1501 	return amdgpu_bo_gpu_offset_no_check(bo);
1502 }
1503 
1504 /**
1505  * amdgpu_bo_fb_aper_addr - return FB aperture GPU offset of the VRAM bo
1506  * @bo:	amdgpu VRAM buffer object for which we query the offset
1507  *
1508  * Returns:
1509  * current FB aperture GPU offset of the object.
1510  */
1511 u64 amdgpu_bo_fb_aper_addr(struct amdgpu_bo *bo)
1512 {
1513 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1514 	uint64_t offset, fb_base;
1515 
1516 	WARN_ON_ONCE(bo->tbo.resource->mem_type != TTM_PL_VRAM);
1517 
1518 	fb_base = adev->gmc.fb_start;
1519 	fb_base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1520 	offset = (bo->tbo.resource->start << PAGE_SHIFT) + fb_base;
1521 	return amdgpu_gmc_sign_extend(offset);
1522 }
1523 
1524 /**
1525  * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo
1526  * @bo:	amdgpu object for which we query the offset
1527  *
1528  * Returns:
1529  * current GPU offset of the object without raising warnings.
1530  */
1531 u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo)
1532 {
1533 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1534 	uint64_t offset = AMDGPU_BO_INVALID_OFFSET;
1535 
1536 	if (bo->tbo.resource->mem_type == TTM_PL_TT)
1537 		offset = amdgpu_gmc_agp_addr(&bo->tbo);
1538 
1539 	if (offset == AMDGPU_BO_INVALID_OFFSET)
1540 		offset = (bo->tbo.resource->start << PAGE_SHIFT) +
1541 			amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type);
1542 
1543 	return amdgpu_gmc_sign_extend(offset);
1544 }
1545 
1546 /**
1547  * amdgpu_bo_mem_stats_placement - bo placement for memory accounting
1548  * @bo:	the buffer object we should look at
1549  *
1550  * BO can have multiple preferred placements, to avoid double counting we want
1551  * to file it under a single placement for memory stats.
1552  * Luckily, if we take the highest set bit in preferred_domains the result is
1553  * quite sensible.
1554  *
1555  * Returns:
1556  * Which of the placements should the BO be accounted under.
1557  */
1558 uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo)
1559 {
1560 	u32 domain;
1561 
1562 	/*
1563 	 * MMIO_REMAP is internal now, so it no longer maps from a userspace
1564 	 * domain bit. Keep fdinfo/mem-stats visibility by checking the actual
1565 	 * TTM placement.
1566 	 */
1567 	if (bo->tbo.resource && bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP)
1568 		return AMDGPU_PL_MMIO_REMAP;
1569 
1570 	domain = bo->preferred_domains & AMDGPU_GEM_DOMAIN_MASK;
1571 	if (!domain)
1572 		return TTM_PL_SYSTEM;
1573 
1574 	switch (rounddown_pow_of_two(domain)) {
1575 	case AMDGPU_GEM_DOMAIN_CPU:
1576 		return TTM_PL_SYSTEM;
1577 	case AMDGPU_GEM_DOMAIN_GTT:
1578 		return TTM_PL_TT;
1579 	case AMDGPU_GEM_DOMAIN_VRAM:
1580 		return TTM_PL_VRAM;
1581 	case AMDGPU_GEM_DOMAIN_GDS:
1582 		return AMDGPU_PL_GDS;
1583 	case AMDGPU_GEM_DOMAIN_GWS:
1584 		return AMDGPU_PL_GWS;
1585 	case AMDGPU_GEM_DOMAIN_OA:
1586 		return AMDGPU_PL_OA;
1587 	case AMDGPU_GEM_DOMAIN_DOORBELL:
1588 		return AMDGPU_PL_DOORBELL;
1589 	default:
1590 		return TTM_PL_SYSTEM;
1591 	}
1592 }
1593 
1594 /**
1595  * amdgpu_bo_get_preferred_domain - get preferred domain
1596  * @adev: amdgpu device object
1597  * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>`
1598  *
1599  * Returns:
1600  * Which of the allowed domains is preferred for allocating the BO.
1601  */
1602 uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,
1603 					    uint32_t domain)
1604 {
1605 	if ((domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) &&
1606 	    ((adev->asic_type == CHIP_CARRIZO) || (adev->asic_type == CHIP_STONEY))) {
1607 		domain = AMDGPU_GEM_DOMAIN_VRAM;
1608 		if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD)
1609 			domain = AMDGPU_GEM_DOMAIN_GTT;
1610 	}
1611 	return domain;
1612 }
1613 
1614 #if defined(CONFIG_DEBUG_FS)
1615 #define amdgpu_bo_print_flag(m, bo, flag)		        \
1616 	do {							\
1617 		if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) {	\
1618 			seq_printf((m), " " #flag);		\
1619 		}						\
1620 	} while (0)
1621 
1622 /**
1623  * amdgpu_bo_print_info - print BO info in debugfs file
1624  *
1625  * @id: Index or Id of the BO
1626  * @bo: Requested BO for printing info
1627  * @m: debugfs file
1628  *
1629  * Print BO information in debugfs file
1630  *
1631  * Returns:
1632  * Size of the BO in bytes.
1633  */
1634 u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m)
1635 {
1636 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1637 	struct dma_buf_attachment *attachment;
1638 	struct dma_buf *dma_buf;
1639 	const char *placement;
1640 	unsigned int pin_count;
1641 	u64 size;
1642 
1643 	if (dma_resv_trylock(bo->tbo.base.resv)) {
1644 		if (!bo->tbo.resource) {
1645 			placement = "NONE";
1646 		} else {
1647 			switch (bo->tbo.resource->mem_type) {
1648 			case TTM_PL_VRAM:
1649 				if (amdgpu_res_cpu_visible(adev, bo->tbo.resource))
1650 					placement = "VRAM VISIBLE";
1651 				else
1652 					placement = "VRAM";
1653 				break;
1654 			case TTM_PL_TT:
1655 				placement = "GTT";
1656 				break;
1657 			case AMDGPU_PL_GDS:
1658 				placement = "GDS";
1659 				break;
1660 			case AMDGPU_PL_GWS:
1661 				placement = "GWS";
1662 				break;
1663 			case AMDGPU_PL_OA:
1664 				placement = "OA";
1665 				break;
1666 			case AMDGPU_PL_PREEMPT:
1667 				placement = "PREEMPTIBLE";
1668 				break;
1669 			case AMDGPU_PL_DOORBELL:
1670 				placement = "DOORBELL";
1671 				break;
1672 			case AMDGPU_PL_MMIO_REMAP:
1673 				placement = "MMIO REMAP";
1674 				break;
1675 			case AMDGPU_PL_NPA:
1676 				placement = "NPA";
1677 				break;
1678 			case TTM_PL_SYSTEM:
1679 			default:
1680 				placement = "CPU";
1681 				break;
1682 			}
1683 		}
1684 		dma_resv_unlock(bo->tbo.base.resv);
1685 	} else {
1686 		placement = "UNKNOWN";
1687 	}
1688 
1689 	size = amdgpu_bo_size(bo);
1690 	seq_printf(m, "\t\t0x%08x: %12lld byte %s",
1691 			id, size, placement);
1692 
1693 	pin_count = READ_ONCE(bo->tbo.pin_count);
1694 	if (pin_count)
1695 		seq_printf(m, " pin count %d", pin_count);
1696 
1697 	dma_buf = READ_ONCE(bo->tbo.base.dma_buf);
1698 	attachment = READ_ONCE(bo->tbo.base.import_attach);
1699 
1700 	if (attachment)
1701 		seq_printf(m, " imported from ino:%llu", file_inode(dma_buf->file)->i_ino);
1702 	else if (dma_buf)
1703 		seq_printf(m, " exported as ino:%llu", file_inode(dma_buf->file)->i_ino);
1704 
1705 	amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED);
1706 	amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS);
1707 	amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC);
1708 	amdgpu_bo_print_flag(m, bo, VRAM_CLEARED);
1709 	amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS);
1710 	amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID);
1711 	amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC);
1712 	/* Add the gem obj resv fence dump*/
1713 	if (dma_resv_trylock(bo->tbo.base.resv)) {
1714 		dma_resv_describe(bo->tbo.base.resv, m);
1715 		dma_resv_unlock(bo->tbo.base.resv);
1716 	}
1717 	seq_puts(m, "\n");
1718 
1719 	return size;
1720 }
1721 #endif
1722