xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c (revision 4dd10fbbeb9c33709edc89916e19c03c780a0751)
1 /*
2  * Copyright 2009 Jerome Glisse.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sub license, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
20  *
21  * The above copyright notice and this permission notice (including the
22  * next paragraph) shall be included in all copies or substantial portions
23  * of the Software.
24  *
25  */
26 /*
27  * Authors:
28  *    Jerome Glisse <glisse@freedesktop.org>
29  *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
30  *    Dave Airlie
31  */
32 #include <linux/list.h>
33 #include <linux/slab.h>
34 #include <linux/dma-buf.h>
35 #include <linux/export.h>
36 
37 #include <drm/drm_drv.h>
38 #include <drm/amdgpu_drm.h>
39 #include <drm/drm_cache.h>
40 #include "amdgpu.h"
41 #include "amdgpu_trace.h"
42 #include "amdgpu_amdkfd.h"
43 #include "amdgpu_vram_mgr.h"
44 #include "amdgpu_vm.h"
45 #include "amdgpu_dma_buf.h"
46 #include "kfd_svm.h"
47 
48 /**
49  * DOC: amdgpu_object
50  *
51  * This defines the interfaces to operate on an &amdgpu_bo buffer object which
52  * represents memory used by driver (VRAM, system memory, etc.). The driver
53  * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces
54  * to create/destroy/set buffer object which are then managed by the kernel TTM
55  * memory manager.
56  * The interfaces are also used internally by kernel clients, including gfx,
57  * uvd, etc. for kernel managed allocations used by the GPU.
58  *
59  */
60 
61 static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo)
62 {
63 	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
64 
65 	amdgpu_bo_kunmap(bo);
66 
67 	if (drm_gem_is_imported(&bo->tbo.base))
68 		drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg);
69 	drm_gem_object_release(&bo->tbo.base);
70 	amdgpu_bo_unref(&bo->parent);
71 	kvfree(bo);
72 }
73 
74 static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo)
75 {
76 	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
77 	struct amdgpu_bo_user *ubo;
78 
79 	ubo = to_amdgpu_bo_user(bo);
80 	kfree(ubo->metadata);
81 	amdgpu_bo_destroy(tbo);
82 }
83 
84 /**
85  * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo
86  * @bo: buffer object to be checked
87  *
88  * Uses destroy function associated with the object to determine if this is
89  * an &amdgpu_bo.
90  *
91  * Returns:
92  * true if the object belongs to &amdgpu_bo, false if not.
93  */
94 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
95 {
96 	if (bo->destroy == &amdgpu_bo_destroy ||
97 	    bo->destroy == &amdgpu_bo_user_destroy ||
98 	    bo->destroy == &svm_range_bo_destroy)
99 		return true;
100 
101 	return false;
102 }
103 
104 /**
105  * amdgpu_bo_placement_from_domain - set buffer's placement
106  * @abo: &amdgpu_bo buffer object whose placement is to be set
107  * @domain: requested domain
108  *
109  * Sets buffer's placement according to requested domain and the buffer's
110  * flags.
111  */
112 void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
113 {
114 	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
115 	struct ttm_placement *placement = &abo->placement;
116 	struct ttm_place *places = abo->placements;
117 	u64 flags = abo->flags;
118 	u32 c = 0;
119 
120 	if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
121 		unsigned int visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
122 		int8_t mem_id = KFD_XCP_MEM_ID(adev, abo->xcp_id);
123 
124 		if (adev->gmc.mem_partitions && mem_id >= 0) {
125 			places[c].fpfn = adev->gmc.mem_partitions[mem_id].range.fpfn;
126 			/*
127 			 * memory partition range lpfn is inclusive start + size - 1
128 			 * TTM place lpfn is exclusive start + size
129 			 */
130 			places[c].lpfn = adev->gmc.mem_partitions[mem_id].range.lpfn + 1;
131 		} else {
132 			places[c].fpfn = 0;
133 			places[c].lpfn = 0;
134 		}
135 		places[c].mem_type = TTM_PL_VRAM;
136 		places[c].flags = 0;
137 
138 		if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
139 			places[c].lpfn = min_not_zero(places[c].lpfn, visible_pfn);
140 		else
141 			places[c].flags |= TTM_PL_FLAG_TOPDOWN;
142 
143 		if (abo->tbo.type == ttm_bo_type_kernel &&
144 		    flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
145 			places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;
146 
147 		c++;
148 	}
149 
150 	if (domain & AMDGPU_GEM_DOMAIN_DOORBELL) {
151 		places[c].fpfn = 0;
152 		places[c].lpfn = 0;
153 		places[c].mem_type = AMDGPU_PL_DOORBELL;
154 		places[c].flags = 0;
155 		c++;
156 	}
157 
158 	if (domain & AMDGPU_GEM_DOMAIN_GTT) {
159 		places[c].fpfn = 0;
160 		places[c].lpfn = 0;
161 		places[c].mem_type =
162 			abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ?
163 			AMDGPU_PL_PREEMPT : TTM_PL_TT;
164 		places[c].flags = 0;
165 		/*
166 		 * When GTT is just an alternative to VRAM make sure that we
167 		 * only use it as fallback and still try to fill up VRAM first.
168 		 */
169 		if (abo->tbo.resource && !(adev->flags & AMD_IS_APU) &&
170 		    domain & abo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)
171 			places[c].flags |= TTM_PL_FLAG_FALLBACK;
172 		c++;
173 	}
174 
175 	if (domain & AMDGPU_GEM_DOMAIN_CPU) {
176 		places[c].fpfn = 0;
177 		places[c].lpfn = 0;
178 		places[c].mem_type = TTM_PL_SYSTEM;
179 		places[c].flags = 0;
180 		c++;
181 	}
182 
183 	if (domain & AMDGPU_GEM_DOMAIN_GDS) {
184 		places[c].fpfn = 0;
185 		places[c].lpfn = 0;
186 		places[c].mem_type = AMDGPU_PL_GDS;
187 		places[c].flags = 0;
188 		c++;
189 	}
190 
191 	if (domain & AMDGPU_GEM_DOMAIN_GWS) {
192 		places[c].fpfn = 0;
193 		places[c].lpfn = 0;
194 		places[c].mem_type = AMDGPU_PL_GWS;
195 		places[c].flags = 0;
196 		c++;
197 	}
198 
199 	if (domain & AMDGPU_GEM_DOMAIN_OA) {
200 		places[c].fpfn = 0;
201 		places[c].lpfn = 0;
202 		places[c].mem_type = AMDGPU_PL_OA;
203 		places[c].flags = 0;
204 		c++;
205 	}
206 
207 	if (domain & AMDGPU_GEM_DOMAIN_NPA) {
208 		places[c].fpfn = 0;
209 		places[c].lpfn = 0;
210 		places[c].mem_type = AMDGPU_PL_NPA;
211 		places[c].flags = 0;
212 		c++;
213 	}
214 
215 	if (!c) {
216 		places[c].fpfn = 0;
217 		places[c].lpfn = 0;
218 		places[c].mem_type = TTM_PL_SYSTEM;
219 		places[c].flags = 0;
220 		c++;
221 	}
222 
223 	BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS);
224 
225 	placement->num_placement = c;
226 	placement->placement = places;
227 }
228 
229 /**
230  * amdgpu_bo_create_reserved - create reserved BO for kernel use
231  *
232  * @adev: amdgpu device object
233  * @size: size for the new BO
234  * @align: alignment for the new BO
235  * @domain: where to place it
236  * @bo_ptr: used to initialize BOs in structures
237  * @gpu_addr: GPU addr of the pinned BO
238  * @cpu_addr: optional CPU address mapping
239  *
240  * Allocates and pins a BO for kernel internal use, and returns it still
241  * reserved.
242  *
243  * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
244  *
245  * Returns:
246  * 0 on success, negative error code otherwise.
247  */
248 int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
249 			      unsigned long size, int align,
250 			      u32 domain, struct amdgpu_bo **bo_ptr,
251 			      u64 *gpu_addr, void **cpu_addr)
252 {
253 	struct amdgpu_bo_param bp;
254 	bool free = false;
255 	int r;
256 
257 	if (!size) {
258 		amdgpu_bo_unref(bo_ptr);
259 		return 0;
260 	}
261 
262 	memset(&bp, 0, sizeof(bp));
263 	bp.size = size;
264 	bp.byte_align = align;
265 	bp.domain = domain;
266 	bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
267 		: AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
268 	bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
269 	bp.type = ttm_bo_type_kernel;
270 	bp.resv = NULL;
271 	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
272 
273 	if (!*bo_ptr) {
274 		r = amdgpu_bo_create(adev, &bp, bo_ptr);
275 		if (r) {
276 			dev_err(adev->dev, "(%d) failed to allocate kernel bo\n",
277 				r);
278 			return r;
279 		}
280 		free = true;
281 	}
282 
283 	r = amdgpu_bo_reserve(*bo_ptr, false);
284 	if (r) {
285 		dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
286 		goto error_free;
287 	}
288 
289 	if (free) {
290 		r = amdgpu_bo_pin(*bo_ptr, domain);
291 		if (r) {
292 			dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
293 			goto error_unreserve;
294 		}
295 	}
296 
297 	r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo);
298 	if (r) {
299 		dev_err(adev->dev, "%p bind failed\n", *bo_ptr);
300 		goto error_unpin;
301 	}
302 
303 	if (gpu_addr)
304 		*gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr);
305 
306 	if (cpu_addr) {
307 		r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
308 		if (r) {
309 			dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
310 			goto error_unpin;
311 		}
312 	}
313 
314 	return 0;
315 
316 error_unpin:
317 	if (free)
318 		amdgpu_bo_unpin(*bo_ptr);
319 error_unreserve:
320 	amdgpu_bo_unreserve(*bo_ptr);
321 
322 error_free:
323 	if (free)
324 		amdgpu_bo_unref(bo_ptr);
325 
326 	return r;
327 }
328 
329 /**
330  * amdgpu_bo_create_kernel - create BO for kernel use
331  *
332  * @adev: amdgpu device object
333  * @size: size for the new BO
334  * @align: alignment for the new BO
335  * @domain: where to place it
336  * @bo_ptr:  used to initialize BOs in structures
337  * @gpu_addr: GPU addr of the pinned BO
338  * @cpu_addr: optional CPU address mapping
339  *
340  * Allocates and pins a BO for kernel internal use.
341  *
342  * This function is exported to allow the V4L2 isp device
343  * external to drm device to create and access the kernel BO.
344  *
345  * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
346  *
347  * Returns:
348  * 0 on success, negative error code otherwise.
349  */
350 int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
351 			    unsigned long size, int align,
352 			    u32 domain, struct amdgpu_bo **bo_ptr,
353 			    u64 *gpu_addr, void **cpu_addr)
354 {
355 	int r;
356 
357 	r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr,
358 				      gpu_addr, cpu_addr);
359 
360 	if (r)
361 		return r;
362 
363 	if (*bo_ptr)
364 		amdgpu_bo_unreserve(*bo_ptr);
365 
366 	return 0;
367 }
368 
369 /**
370  * amdgpu_bo_create_isp_user - create user BO for isp
371  *
372  * @adev: amdgpu device object
373  * @dma_buf: DMABUF handle for isp buffer
374  * @domain: where to place it
375  * @bo:  used to initialize BOs in structures
376  * @gpu_addr: GPU addr of the pinned BO
377  *
378  * Imports isp DMABUF to allocate and pin a user BO for isp internal use. It does
379  * GART alloc to generate gpu_addr for BO to make it accessible through the
380  * GART aperture for ISP HW.
381  *
382  * This function is exported to allow the V4L2 isp device external to drm device
383  * to create and access the isp user BO.
384  *
385  * Returns:
386  * 0 on success, negative error code otherwise.
387  */
388 int amdgpu_bo_create_isp_user(struct amdgpu_device *adev,
389 			   struct dma_buf *dma_buf, u32 domain, struct amdgpu_bo **bo,
390 			   u64 *gpu_addr)
391 
392 {
393 	struct drm_gem_object *gem_obj;
394 	int r;
395 
396 	gem_obj = amdgpu_gem_prime_import(&adev->ddev, dma_buf);
397 	*bo = gem_to_amdgpu_bo(gem_obj);
398 	if (!(*bo)) {
399 		dev_err(adev->dev, "failed to get valid isp user bo\n");
400 		return -EINVAL;
401 	}
402 
403 	r = amdgpu_bo_reserve(*bo, false);
404 	if (r) {
405 		dev_err(adev->dev, "(%d) failed to reserve isp user bo\n", r);
406 		return r;
407 	}
408 
409 	r = amdgpu_bo_pin(*bo, domain);
410 	if (r) {
411 		dev_err(adev->dev, "(%d) isp user bo pin failed\n", r);
412 		goto error_unreserve;
413 	}
414 
415 	r = amdgpu_ttm_alloc_gart(&(*bo)->tbo);
416 	if (r) {
417 		dev_err(adev->dev, "%p bind failed\n", *bo);
418 		goto error_unpin;
419 	}
420 
421 	if (!WARN_ON(!gpu_addr))
422 		*gpu_addr = amdgpu_bo_gpu_offset(*bo);
423 
424 	amdgpu_bo_unreserve(*bo);
425 
426 	return 0;
427 
428 error_unpin:
429 	amdgpu_bo_unpin(*bo);
430 error_unreserve:
431 	amdgpu_bo_unreserve(*bo);
432 	amdgpu_bo_unref(bo);
433 
434 	return r;
435 }
436 
437 /**
438  * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location
439  *
440  * @adev: amdgpu device object
441  * @offset: offset of the BO
442  * @size: size of the BO
443  * @bo_ptr:  used to initialize BOs in structures
444  * @cpu_addr: optional CPU address mapping
445  *
446  * Creates a kernel BO at a specific offset in VRAM.
447  *
448  * Returns:
449  * 0 on success, negative error code otherwise.
450  */
451 int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev,
452 			       uint64_t offset, uint64_t size,
453 			       struct amdgpu_bo **bo_ptr, void **cpu_addr)
454 {
455 	struct ttm_operation_ctx ctx = { false, false };
456 	unsigned int i;
457 	int r;
458 
459 	offset &= PAGE_MASK;
460 	size = ALIGN(size, PAGE_SIZE);
461 
462 	r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE,
463 				      AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL,
464 				      cpu_addr);
465 	if (r)
466 		return r;
467 
468 	if ((*bo_ptr) == NULL)
469 		return 0;
470 
471 	/*
472 	 * Remove the original mem node and create a new one at the request
473 	 * position.
474 	 */
475 	if (cpu_addr)
476 		amdgpu_bo_kunmap(*bo_ptr);
477 
478 	ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource);
479 
480 	for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) {
481 		(*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT;
482 		(*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
483 	}
484 	r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement,
485 			     &(*bo_ptr)->tbo.resource, &ctx);
486 	if (r)
487 		goto error;
488 
489 	if (cpu_addr) {
490 		r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
491 		if (r)
492 			goto error;
493 	}
494 
495 	amdgpu_bo_unreserve(*bo_ptr);
496 	return 0;
497 
498 error:
499 	amdgpu_bo_unreserve(*bo_ptr);
500 	amdgpu_bo_unref(bo_ptr);
501 	return r;
502 }
503 
504 /**
505  * amdgpu_bo_free_kernel - free BO for kernel use
506  *
507  * @bo: amdgpu BO to free
508  * @gpu_addr: pointer to where the BO's GPU memory space address was stored
509  * @cpu_addr: pointer to where the BO's CPU memory space address was stored
510  *
511  * unmaps and unpin a BO for kernel internal use.
512  *
513  * This function is exported to allow the V4L2 isp device
514  * external to drm device to free the kernel BO.
515  */
516 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
517 			   void **cpu_addr)
518 {
519 	if (*bo == NULL)
520 		return;
521 
522 	WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend);
523 
524 	if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {
525 		if (cpu_addr)
526 			amdgpu_bo_kunmap(*bo);
527 
528 		amdgpu_bo_unpin(*bo);
529 		amdgpu_bo_unreserve(*bo);
530 	}
531 	amdgpu_bo_unref(bo);
532 
533 	if (gpu_addr)
534 		*gpu_addr = 0;
535 
536 	if (cpu_addr)
537 		*cpu_addr = NULL;
538 }
539 
540 /**
541  * amdgpu_bo_free_isp_user - free BO for isp use
542  *
543  * @bo: amdgpu isp user BO to free
544  *
545  * unpin and unref BO for isp internal use.
546  *
547  * This function is exported to allow the V4L2 isp device
548  * external to drm device to free the isp user BO.
549  */
550 void amdgpu_bo_free_isp_user(struct amdgpu_bo *bo)
551 {
552 	if (bo == NULL)
553 		return;
554 
555 	if (amdgpu_bo_reserve(bo, true) == 0) {
556 		amdgpu_bo_unpin(bo);
557 		amdgpu_bo_unreserve(bo);
558 	}
559 	amdgpu_bo_unref(&bo);
560 }
561 
562 /* Validate bo size is bit bigger than the request domain */
563 static bool amdgpu_bo_validate_size(struct amdgpu_device *adev,
564 					  unsigned long size, u32 domain)
565 {
566 	struct ttm_resource_manager *man = NULL;
567 
568 	/*
569 	 * If GTT is part of requested domains the check must succeed to
570 	 * allow fall back to GTT.
571 	 */
572 	if (domain & AMDGPU_GEM_DOMAIN_GTT)
573 		man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
574 	else if (domain & AMDGPU_GEM_DOMAIN_VRAM)
575 		man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
576 	else
577 		return true;
578 
579 	if (!man) {
580 		if (domain & AMDGPU_GEM_DOMAIN_GTT)
581 			WARN_ON_ONCE("GTT domain requested but GTT mem manager uninitialized");
582 		return false;
583 	}
584 
585 	/* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU, _DOMAIN_DOORBELL */
586 	if (size < man->size)
587 		return true;
588 
589 	DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, man->size);
590 	return false;
591 }
592 
593 bool amdgpu_bo_support_uswc(u64 bo_flags)
594 {
595 
596 #ifdef CONFIG_X86_32
597 	/* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit
598 	 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627
599 	 */
600 	return false;
601 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
602 	/* Don't try to enable write-combining when it can't work, or things
603 	 * may be slow
604 	 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758
605 	 */
606 
607 #ifndef CONFIG_COMPILE_TEST
608 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
609 	 thanks to write-combining
610 #endif
611 
612 	if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
613 		DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
614 			      "better performance thanks to write-combining\n");
615 	return false;
616 #else
617 	/* For architectures that don't support WC memory,
618 	 * mask out the WC flag from the BO
619 	 */
620 	if (!drm_arch_can_wc_memory())
621 		return false;
622 
623 	return true;
624 #endif
625 }
626 
627 /**
628  * amdgpu_bo_create - create an &amdgpu_bo buffer object
629  * @adev: amdgpu device object
630  * @bp: parameters to be used for the buffer object
631  * @bo_ptr: pointer to the buffer object pointer
632  *
633  * Creates an &amdgpu_bo buffer object.
634  *
635  * Returns:
636  * 0 for success or a negative error code on failure.
637  */
638 int amdgpu_bo_create(struct amdgpu_device *adev,
639 			       struct amdgpu_bo_param *bp,
640 			       struct amdgpu_bo **bo_ptr)
641 {
642 	struct ttm_operation_ctx ctx = {
643 		.interruptible = (bp->type != ttm_bo_type_kernel),
644 		.no_wait_gpu = bp->no_wait_gpu,
645 		/* We opt to avoid OOM on system pages allocations */
646 		.gfp_retry_mayfail = true,
647 		.allow_res_evict = bp->type != ttm_bo_type_kernel,
648 		.resv = bp->resv
649 	};
650 	struct amdgpu_bo *bo;
651 	unsigned long page_align, size = bp->size;
652 	int r;
653 
654 	/* Note that GDS/GWS/OA allocates 1 page per byte/resource. */
655 	if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
656 		/* GWS and OA don't need any alignment. */
657 		page_align = bp->byte_align;
658 		size <<= PAGE_SHIFT;
659 
660 	} else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) {
661 		/* Both size and alignment must be a multiple of 4. */
662 		page_align = ALIGN(bp->byte_align, 4);
663 		size = ALIGN(size, 4) << PAGE_SHIFT;
664 	} else {
665 		/* Memory should be aligned at least to a page size. */
666 		page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT;
667 		size = ALIGN(size, PAGE_SIZE);
668 	}
669 
670 	if (!amdgpu_bo_validate_size(adev, size, bp->domain))
671 		return -ENOMEM;
672 
673 	BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo));
674 
675 	*bo_ptr = NULL;
676 	bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL);
677 	if (bo == NULL)
678 		return -ENOMEM;
679 	drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size);
680 	bo->tbo.base.funcs = &amdgpu_gem_object_funcs;
681 	bo->vm_bo = NULL;
682 	bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain :
683 		bp->domain;
684 	bo->allowed_domains = bo->preferred_domains;
685 	if (bp->type != ttm_bo_type_kernel &&
686 	    !(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE) &&
687 	    bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
688 		bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
689 
690 	bo->flags = bp->flags;
691 
692 	if (adev->gmc.mem_partitions)
693 		/* For GPUs with spatial partitioning, bo->xcp_id=-1 means any partition */
694 		bo->xcp_id = bp->xcp_id_plus1 - 1;
695 	else
696 		/* For GPUs without spatial partitioning */
697 		bo->xcp_id = 0;
698 
699 	if (!amdgpu_bo_support_uswc(bo->flags))
700 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
701 
702 	bo->tbo.bdev = &adev->mman.bdev;
703 	if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA |
704 			  AMDGPU_GEM_DOMAIN_GDS))
705 		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
706 	else
707 		amdgpu_bo_placement_from_domain(bo, bp->domain);
708 	if (bp->type == ttm_bo_type_kernel)
709 		bo->tbo.priority = 2;
710 	else if (!(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE))
711 		bo->tbo.priority = 1;
712 
713 	if (!bp->destroy)
714 		bp->destroy = &amdgpu_bo_destroy;
715 
716 	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type,
717 				 &bo->placement, page_align, &ctx,  NULL,
718 				 bp->resv, bp->destroy);
719 	if (unlikely(r != 0))
720 		return r;
721 
722 	if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
723 	    amdgpu_res_cpu_visible(adev, bo->tbo.resource))
724 		amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved,
725 					     ctx.bytes_moved);
726 	else
727 		amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0);
728 
729 	if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
730 	    bo->tbo.resource->mem_type == TTM_PL_VRAM) {
731 		struct dma_fence *fence;
732 
733 		r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev),
734 					    bo, bo->tbo.base.resv, &fence,
735 					    true, AMDGPU_KERNEL_JOB_ID_TTM_CLEAR_BUFFER);
736 		if (unlikely(r))
737 			goto fail_unreserve;
738 
739 		if (fence) {
740 			dma_resv_add_fence(bo->tbo.base.resv, fence,
741 					   DMA_RESV_USAGE_KERNEL);
742 			dma_fence_put(fence);
743 		}
744 	}
745 	if (!bp->resv)
746 		amdgpu_bo_unreserve(bo);
747 	*bo_ptr = bo;
748 
749 	trace_amdgpu_bo_create(bo);
750 
751 	/* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */
752 	if (bp->type == ttm_bo_type_device)
753 		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
754 
755 	return 0;
756 
757 fail_unreserve:
758 	if (!bp->resv)
759 		dma_resv_unlock(bo->tbo.base.resv);
760 	amdgpu_bo_unref(&bo);
761 	return r;
762 }
763 
764 /**
765  * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object
766  * @adev: amdgpu device object
767  * @bp: parameters to be used for the buffer object
768  * @ubo_ptr: pointer to the buffer object pointer
769  *
770  * Create a BO to be used by user application;
771  *
772  * Returns:
773  * 0 for success or a negative error code on failure.
774  */
775 
776 int amdgpu_bo_create_user(struct amdgpu_device *adev,
777 			  struct amdgpu_bo_param *bp,
778 			  struct amdgpu_bo_user **ubo_ptr)
779 {
780 	struct amdgpu_bo *bo_ptr;
781 	int r;
782 
783 	bp->bo_ptr_size = sizeof(struct amdgpu_bo_user);
784 	bp->destroy = &amdgpu_bo_user_destroy;
785 	r = amdgpu_bo_create(adev, bp, &bo_ptr);
786 	if (r)
787 		return r;
788 
789 	*ubo_ptr = to_amdgpu_bo_user(bo_ptr);
790 	return r;
791 }
792 
793 /**
794  * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object
795  * @adev: amdgpu device object
796  * @bp: parameters to be used for the buffer object
797  * @vmbo_ptr: pointer to the buffer object pointer
798  *
799  * Create a BO to be for GPUVM.
800  *
801  * Returns:
802  * 0 for success or a negative error code on failure.
803  */
804 
805 int amdgpu_bo_create_vm(struct amdgpu_device *adev,
806 			struct amdgpu_bo_param *bp,
807 			struct amdgpu_bo_vm **vmbo_ptr)
808 {
809 	struct amdgpu_bo *bo_ptr;
810 	int r;
811 
812 	/* bo_ptr_size will be determined by the caller and it depends on
813 	 * num of amdgpu_vm_pt entries.
814 	 */
815 	BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm));
816 	r = amdgpu_bo_create(adev, bp, &bo_ptr);
817 	if (r)
818 		return r;
819 
820 	*vmbo_ptr = to_amdgpu_bo_vm(bo_ptr);
821 	return r;
822 }
823 
824 /**
825  * amdgpu_bo_kmap - map an &amdgpu_bo buffer object
826  * @bo: &amdgpu_bo buffer object to be mapped
827  * @ptr: kernel virtual address to be returned
828  *
829  * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls
830  * amdgpu_bo_kptr() to get the kernel virtual address.
831  *
832  * Returns:
833  * 0 for success or a negative error code on failure.
834  */
835 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
836 {
837 	void *kptr;
838 	long r;
839 
840 	if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
841 		return -EPERM;
842 
843 	r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL,
844 				  false, MAX_SCHEDULE_TIMEOUT);
845 	if (r < 0)
846 		return r;
847 
848 	kptr = amdgpu_bo_kptr(bo);
849 	if (kptr) {
850 		if (ptr)
851 			*ptr = kptr;
852 		return 0;
853 	}
854 
855 	r = ttm_bo_kmap(&bo->tbo, 0, PFN_UP(bo->tbo.base.size), &bo->kmap);
856 	if (r)
857 		return r;
858 
859 	if (ptr)
860 		*ptr = amdgpu_bo_kptr(bo);
861 
862 	return 0;
863 }
864 
865 /**
866  * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object
867  * @bo: &amdgpu_bo buffer object
868  *
869  * Calls ttm_kmap_obj_virtual() to get the kernel virtual address
870  *
871  * Returns:
872  * the virtual address of a buffer object area.
873  */
874 void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
875 {
876 	bool is_iomem;
877 
878 	return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
879 }
880 
881 /**
882  * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object
883  * @bo: &amdgpu_bo buffer object to be unmapped
884  *
885  * Unmaps a kernel map set up by amdgpu_bo_kmap().
886  */
887 void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
888 {
889 	if (bo->kmap.bo)
890 		ttm_bo_kunmap(&bo->kmap);
891 }
892 
893 /**
894  * amdgpu_bo_ref - reference an &amdgpu_bo buffer object
895  * @bo: &amdgpu_bo buffer object
896  *
897  * References the contained &ttm_buffer_object.
898  *
899  * Returns:
900  * a refcounted pointer to the &amdgpu_bo buffer object.
901  */
902 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
903 {
904 	if (bo == NULL)
905 		return NULL;
906 
907 	drm_gem_object_get(&bo->tbo.base);
908 	return bo;
909 }
910 
911 /**
912  * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object
913  * @bo: &amdgpu_bo buffer object
914  *
915  * Unreferences the contained &ttm_buffer_object and clear the pointer
916  */
917 void amdgpu_bo_unref(struct amdgpu_bo **bo)
918 {
919 	if ((*bo) == NULL)
920 		return;
921 
922 	drm_gem_object_put(&(*bo)->tbo.base);
923 	*bo = NULL;
924 }
925 
926 /* Map a BO onto vram_pin_size_xcp[]. bo->xcp_id < 0 means "any partition":
927  * such pins are not owned by a single partition and go to
928  * vram_pin_size_common, which every partition subtracts.
929  */
930 static int amdgpu_bo_get_xcp(struct amdgpu_bo *bo)
931 {
932 	s8 xcp = bo->xcp_id;
933 
934 	if (xcp >= 0 && xcp < MAX_XCP)
935 		return xcp;
936 	return -1;
937 }
938 
939 /**
940  * amdgpu_bo_pin - pin an &amdgpu_bo buffer object
941  * @bo: &amdgpu_bo buffer object to be pinned
942  * @domain: domain to be pinned to
943  *
944  * Pins the buffer object according to requested domain. If the memory is
945  * unbound gart memory, binds the pages into gart table. Adjusts pin_count and
946  * pin_size accordingly.
947  *
948  * Pinning means to lock pages in memory along with keeping them at a fixed
949  * offset. It is required when a buffer can not be moved, for example, when
950  * a display buffer is being scanned out.
951  *
952  * Returns:
953  * 0 for success or a negative error code on failure.
954  */
955 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain)
956 {
957 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
958 	struct ttm_operation_ctx ctx = { false, false };
959 	int r, i;
960 
961 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
962 		return -EPERM;
963 
964 	/* Check domain to be pinned to against preferred domains */
965 	if (bo->preferred_domains & domain)
966 		domain = bo->preferred_domains & domain;
967 
968 	/* A shared bo cannot be migrated to VRAM */
969 	if (drm_gem_is_imported(&bo->tbo.base)) {
970 		if (domain & AMDGPU_GEM_DOMAIN_GTT)
971 			domain = AMDGPU_GEM_DOMAIN_GTT;
972 		else
973 			return -EINVAL;
974 	}
975 
976 	if (bo->tbo.pin_count) {
977 		uint32_t mem_type = bo->tbo.resource->mem_type;
978 		uint32_t mem_flags = bo->tbo.resource->placement;
979 
980 		if (!(domain & amdgpu_mem_type_to_domain(mem_type)))
981 			return -EINVAL;
982 
983 		if ((mem_type == TTM_PL_VRAM) &&
984 		    (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) &&
985 		    !(mem_flags & TTM_PL_FLAG_CONTIGUOUS))
986 			return -EINVAL;
987 
988 		ttm_bo_pin(&bo->tbo);
989 		return 0;
990 	}
991 
992 	/* This assumes only APU display buffers are pinned with (VRAM|GTT).
993 	 * See function amdgpu_display_supported_domains()
994 	 */
995 	domain = amdgpu_bo_get_preferred_domain(adev, domain);
996 
997 	if (drm_gem_is_imported(&bo->tbo.base))
998 		dma_buf_pin(bo->tbo.base.import_attach);
999 
1000 	/* force to pin into visible video ram */
1001 	if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))
1002 		bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1003 	amdgpu_bo_placement_from_domain(bo, domain);
1004 	for (i = 0; i < bo->placement.num_placement; i++) {
1005 		if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS &&
1006 		    bo->placements[i].mem_type == TTM_PL_VRAM)
1007 			bo->placements[i].flags |= TTM_PL_FLAG_CONTIGUOUS;
1008 	}
1009 
1010 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1011 	if (unlikely(r)) {
1012 		dev_err(adev->dev, "%p pin failed\n", bo);
1013 		goto error;
1014 	}
1015 
1016 	ttm_bo_pin(&bo->tbo);
1017 
1018 	if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {
1019 		int xcp = amdgpu_bo_get_xcp(bo);
1020 
1021 		atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size);
1022 		if (xcp >= 0) {
1023 			atomic64_add(amdgpu_bo_size(bo),
1024 				     &adev->vram_pin_size_xcp[xcp]);
1025 		} else {
1026 			atomic64_add(amdgpu_bo_size(bo),
1027 				     &adev->vram_pin_size_common);
1028 		}
1029 		amdgpu_amdkfd_account_vram_pin(bo, xcp);
1030 		atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo),
1031 			     &adev->visible_pin_size);
1032 	} else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
1033 		atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size);
1034 	}
1035 
1036 error:
1037 	return r;
1038 }
1039 
1040 /**
1041  * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object
1042  * @bo: &amdgpu_bo buffer object to be unpinned
1043  *
1044  * Decreases the pin_count, and clears the flags if pin_count reaches 0.
1045  * Changes placement and pin size accordingly.
1046  *
1047  * Returns:
1048  * 0 for success or a negative error code on failure.
1049  */
1050 void amdgpu_bo_unpin(struct amdgpu_bo *bo)
1051 {
1052 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1053 
1054 	ttm_bo_unpin(&bo->tbo);
1055 	if (bo->tbo.pin_count)
1056 		return;
1057 
1058 	if (drm_gem_is_imported(&bo->tbo.base))
1059 		dma_buf_unpin(bo->tbo.base.import_attach);
1060 
1061 	if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {
1062 		int xcp = amdgpu_bo_get_xcp(bo);
1063 
1064 		atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size);
1065 		if (xcp >= 0) {
1066 			atomic64_sub(amdgpu_bo_size(bo),
1067 				     &adev->vram_pin_size_xcp[xcp]);
1068 		} else {
1069 			atomic64_sub(amdgpu_bo_size(bo),
1070 				     &adev->vram_pin_size_common);
1071 		}
1072 		amdgpu_amdkfd_account_vram_unpin(bo, xcp);
1073 		atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo),
1074 			     &adev->visible_pin_size);
1075 	} else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
1076 		atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size);
1077 	}
1078 
1079 }
1080 
1081 static const char * const amdgpu_vram_names[] = {
1082 	"UNKNOWN",
1083 	"GDDR1",
1084 	"DDR2",
1085 	"GDDR3",
1086 	"GDDR4",
1087 	"GDDR5",
1088 	"HBM",
1089 	"DDR3",
1090 	"DDR4",
1091 	"GDDR6",
1092 	"DDR5",
1093 	"LPDDR4",
1094 	"LPDDR5",
1095 	"HBM3E",
1096 	"HBM4",
1097 	"GDDR7"
1098 };
1099 
1100 /**
1101  * amdgpu_bo_init - initialize memory manager
1102  * @adev: amdgpu device object
1103  *
1104  * Calls amdgpu_ttm_init() to initialize amdgpu memory manager.
1105  *
1106  * Returns:
1107  * 0 for success or a negative error code on failure.
1108  */
1109 int amdgpu_bo_init(struct amdgpu_device *adev)
1110 {
1111 	/* On A+A platform, VRAM can be mapped as WB */
1112 	if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
1113 		/* reserve PAT memory space to WC for VRAM */
1114 		int r = arch_io_reserve_memtype_wc(adev->gmc.aper_base,
1115 				adev->gmc.aper_size);
1116 
1117 		if (r) {
1118 			DRM_ERROR("Unable to set WC memtype for the aperture base\n");
1119 			return r;
1120 		}
1121 
1122 		/* Add an MTRR for the VRAM */
1123 		adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base,
1124 				adev->gmc.aper_size);
1125 	}
1126 
1127 	drm_info(adev_to_drm(adev), "Detected VRAM RAM=%lluM, BAR=%lluM\n",
1128 		 adev->gmc.mc_vram_size >> 20,
1129 		 (unsigned long long)adev->gmc.aper_size >> 20);
1130 	drm_info(adev_to_drm(adev), "RAM width %dbits %s\n",
1131 		 adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]);
1132 	return amdgpu_ttm_init(adev);
1133 }
1134 
1135 /**
1136  * amdgpu_bo_fini - tear down memory manager
1137  * @adev: amdgpu device object
1138  *
1139  * Reverses amdgpu_bo_init() to tear down memory manager.
1140  */
1141 void amdgpu_bo_fini(struct amdgpu_device *adev)
1142 {
1143 	int idx;
1144 
1145 	amdgpu_ttm_fini(adev);
1146 
1147 	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
1148 		if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
1149 			arch_phys_wc_del(adev->gmc.vram_mtrr);
1150 			arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
1151 		}
1152 		drm_dev_exit(idx);
1153 	}
1154 }
1155 
1156 /**
1157  * amdgpu_bo_set_tiling_flags - set tiling flags
1158  * @bo: &amdgpu_bo buffer object
1159  * @tiling_flags: new flags
1160  *
1161  * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or
1162  * kernel driver to set the tiling flags on a buffer.
1163  *
1164  * Returns:
1165  * 0 for success or a negative error code on failure.
1166  */
1167 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
1168 {
1169 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1170 	struct amdgpu_bo_user *ubo;
1171 
1172 	/* MMIO_REMAP is BAR I/O space; tiling should never be used here. */
1173 	WARN_ON_ONCE(bo->tbo.resource &&
1174 		     bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP);
1175 
1176 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1177 	if (adev->family <= AMDGPU_FAMILY_CZ &&
1178 	    AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
1179 		return -EINVAL;
1180 
1181 	ubo = to_amdgpu_bo_user(bo);
1182 	ubo->tiling_flags = tiling_flags;
1183 	return 0;
1184 }
1185 
1186 /**
1187  * amdgpu_bo_get_tiling_flags - get tiling flags
1188  * @bo: &amdgpu_bo buffer object
1189  * @tiling_flags: returned flags
1190  *
1191  * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to
1192  * set the tiling flags on a buffer.
1193  */
1194 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
1195 {
1196 	struct amdgpu_bo_user *ubo;
1197 
1198 	/*
1199 	 * MMIO_REMAP BOs are not real VRAM/GTT memory but a fixed BAR I/O window.
1200 	 * They should never go through GEM tiling helpers.
1201 	 */
1202 	WARN_ON_ONCE(bo->tbo.resource &&
1203 		     bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP);
1204 
1205 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1206 	dma_resv_assert_held(bo->tbo.base.resv);
1207 	ubo = to_amdgpu_bo_user(bo);
1208 
1209 	if (tiling_flags)
1210 		*tiling_flags = ubo->tiling_flags;
1211 }
1212 
1213 /**
1214  * amdgpu_bo_set_metadata - set metadata
1215  * @bo: &amdgpu_bo buffer object
1216  * @metadata: new metadata
1217  * @metadata_size: size of the new metadata
1218  * @flags: flags of the new metadata
1219  *
1220  * Sets buffer object's metadata, its size and flags.
1221  * Used via GEM ioctl.
1222  *
1223  * Returns:
1224  * 0 for success or a negative error code on failure.
1225  */
1226 int amdgpu_bo_set_metadata(struct amdgpu_bo *bo, void *metadata,
1227 			   u32 metadata_size, uint64_t flags)
1228 {
1229 	struct amdgpu_bo_user *ubo;
1230 	void *buffer;
1231 
1232 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1233 	ubo = to_amdgpu_bo_user(bo);
1234 	if (!metadata_size) {
1235 		if (ubo->metadata_size) {
1236 			kfree(ubo->metadata);
1237 			ubo->metadata = NULL;
1238 			ubo->metadata_size = 0;
1239 		}
1240 		return 0;
1241 	}
1242 
1243 	if (metadata == NULL)
1244 		return -EINVAL;
1245 
1246 	buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
1247 	if (buffer == NULL)
1248 		return -ENOMEM;
1249 
1250 	kfree(ubo->metadata);
1251 	ubo->metadata_flags = flags;
1252 	ubo->metadata = buffer;
1253 	ubo->metadata_size = metadata_size;
1254 
1255 	return 0;
1256 }
1257 
1258 /**
1259  * amdgpu_bo_get_metadata - get metadata
1260  * @bo: &amdgpu_bo buffer object
1261  * @buffer: returned metadata
1262  * @buffer_size: size of the buffer
1263  * @metadata_size: size of the returned metadata
1264  * @flags: flags of the returned metadata
1265  *
1266  * Gets buffer object's metadata, its size and flags. buffer_size shall not be
1267  * less than metadata_size.
1268  * Used via GEM ioctl.
1269  *
1270  * Returns:
1271  * 0 for success or a negative error code on failure.
1272  */
1273 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
1274 			   size_t buffer_size, uint32_t *metadata_size,
1275 			   uint64_t *flags)
1276 {
1277 	struct amdgpu_bo_user *ubo;
1278 
1279 	if (!buffer && !metadata_size)
1280 		return -EINVAL;
1281 
1282 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1283 	ubo = to_amdgpu_bo_user(bo);
1284 	if (metadata_size)
1285 		*metadata_size = ubo->metadata_size;
1286 
1287 	if (buffer) {
1288 		if (buffer_size < ubo->metadata_size)
1289 			return -EINVAL;
1290 
1291 		if (ubo->metadata_size)
1292 			memcpy(buffer, ubo->metadata, ubo->metadata_size);
1293 	}
1294 
1295 	if (flags)
1296 		*flags = ubo->metadata_flags;
1297 
1298 	return 0;
1299 }
1300 
1301 /**
1302  * amdgpu_bo_move_notify - notification about a memory move
1303  * @bo: pointer to a buffer object
1304  * @evict: if this move is evicting the buffer from the graphics address space
1305  * @new_mem: new resource for backing the BO
1306  *
1307  * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs
1308  * bookkeeping.
1309  * TTM driver callback which is called when ttm moves a buffer.
1310  */
1311 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
1312 			   bool evict,
1313 			   struct ttm_resource *new_mem)
1314 {
1315 	struct ttm_resource *old_mem = bo->resource;
1316 	struct amdgpu_bo *abo;
1317 
1318 	if (!amdgpu_bo_is_amdgpu_bo(bo))
1319 		return;
1320 
1321 	abo = ttm_to_amdgpu_bo(bo);
1322 	amdgpu_vm_bo_move(abo, new_mem, evict);
1323 
1324 	amdgpu_bo_kunmap(abo);
1325 
1326 	if (abo->tbo.base.dma_buf && !drm_gem_is_imported(&abo->tbo.base) &&
1327 	    old_mem && old_mem->mem_type != TTM_PL_SYSTEM)
1328 		dma_buf_invalidate_mappings(abo->tbo.base.dma_buf);
1329 
1330 	/* move_notify is called before move happens */
1331 	trace_amdgpu_bo_move(abo, new_mem ? new_mem->mem_type : -1,
1332 			     old_mem ? old_mem->mem_type : -1);
1333 }
1334 
1335 /**
1336  * amdgpu_bo_release_notify - notification about a BO being released
1337  * @bo: pointer to a buffer object
1338  *
1339  * Wipes VRAM buffers whose contents should not be leaked before the
1340  * memory is released.
1341  */
1342 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)
1343 {
1344 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1345 	struct dma_fence *fence = NULL;
1346 	struct amdgpu_bo *abo;
1347 	int r;
1348 
1349 	if (!amdgpu_bo_is_amdgpu_bo(bo))
1350 		return;
1351 
1352 	abo = ttm_to_amdgpu_bo(bo);
1353 
1354 	WARN_ON(abo->vm_bo);
1355 
1356 	if (abo->kfd_bo)
1357 		amdgpu_amdkfd_release_notify(abo);
1358 
1359 	/*
1360 	 * We lock the private dma_resv object here and since the BO is about to
1361 	 * be released nobody else should have a pointer to it.
1362 	 * So when this locking here fails something is wrong with the reference
1363 	 * counting.
1364 	 */
1365 	if (WARN_ON_ONCE(!dma_resv_trylock(&bo->base._resv)))
1366 		return;
1367 
1368 	amdgpu_amdkfd_remove_all_eviction_fences(abo);
1369 
1370 	if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM ||
1371 	    !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) ||
1372 	    adev->in_suspend || drm_dev_is_unplugged(adev_to_drm(adev)))
1373 		goto out;
1374 
1375 	r = dma_resv_reserve_fences(&bo->base._resv, 1);
1376 	if (r)
1377 		goto out;
1378 
1379 	r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev),
1380 				    abo, &bo->base._resv, &fence,
1381 				    false, AMDGPU_KERNEL_JOB_ID_CLEAR_ON_RELEASE);
1382 	if (WARN_ON(r))
1383 		goto out;
1384 
1385 	amdgpu_vram_mgr_set_cleared(bo->resource);
1386 	dma_resv_add_fence(&bo->base._resv, fence, DMA_RESV_USAGE_KERNEL);
1387 	dma_fence_put(fence);
1388 
1389 out:
1390 	dma_resv_unlock(&bo->base._resv);
1391 }
1392 
1393 /**
1394  * amdgpu_bo_fault_reserve_notify - notification about a memory fault
1395  * @bo: pointer to a buffer object
1396  *
1397  * Notifies the driver we are taking a fault on this BO and have reserved it,
1398  * also performs bookkeeping.
1399  * TTM driver callback for dealing with vm faults.
1400  *
1401  * Returns:
1402  * 0 for success or a negative error code on failure.
1403  */
1404 vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
1405 {
1406 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1407 	struct ttm_operation_ctx ctx = { false, false };
1408 	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1409 	int r;
1410 
1411 	/* Remember that this BO was accessed by the CPU */
1412 	abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1413 
1414 	if (amdgpu_res_cpu_visible(adev, bo->resource))
1415 		return 0;
1416 
1417 	/* Can't move a pinned BO to visible VRAM */
1418 	if (abo->tbo.pin_count > 0)
1419 		return VM_FAULT_SIGBUS;
1420 
1421 	/* hurrah the memory is not visible ! */
1422 	atomic64_inc(&adev->num_vram_cpu_page_faults);
1423 	amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
1424 					AMDGPU_GEM_DOMAIN_GTT);
1425 
1426 	/* Avoid costly evictions; only set GTT as a busy placement */
1427 	abo->placements[0].flags |= TTM_PL_FLAG_DESIRED;
1428 
1429 	r = ttm_bo_validate(bo, &abo->placement, &ctx);
1430 	if (unlikely(r == -EBUSY || r == -ERESTARTSYS))
1431 		return VM_FAULT_NOPAGE;
1432 	else if (unlikely(r))
1433 		return VM_FAULT_SIGBUS;
1434 
1435 	/* this should never happen */
1436 	if (bo->resource->mem_type == TTM_PL_VRAM &&
1437 	    !amdgpu_res_cpu_visible(adev, bo->resource))
1438 		return VM_FAULT_SIGBUS;
1439 
1440 	ttm_bo_move_to_lru_tail_unlocked(bo);
1441 	return 0;
1442 }
1443 
1444 /**
1445  * amdgpu_bo_fence - add fence to buffer object
1446  *
1447  * @bo: buffer object in question
1448  * @fence: fence to add
1449  * @shared: true if fence should be added shared
1450  *
1451  */
1452 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
1453 		     bool shared)
1454 {
1455 	struct dma_resv *resv = bo->tbo.base.resv;
1456 	int r;
1457 
1458 	r = dma_resv_reserve_fences(resv, 1);
1459 	if (r) {
1460 		/* As last resort on OOM we block for the fence */
1461 		dma_fence_wait(fence, false);
1462 		return;
1463 	}
1464 
1465 	dma_resv_add_fence(resv, fence, shared ? DMA_RESV_USAGE_READ :
1466 			   DMA_RESV_USAGE_WRITE);
1467 }
1468 
1469 /**
1470  * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences
1471  *
1472  * @adev: amdgpu device pointer
1473  * @resv: reservation object to sync to
1474  * @sync_mode: synchronization mode
1475  * @owner: fence owner
1476  * @intr: Whether the wait is interruptible
1477  *
1478  * Extract the fences from the reservation object and waits for them to finish.
1479  *
1480  * Returns:
1481  * 0 on success, errno otherwise.
1482  */
1483 int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv,
1484 			     enum amdgpu_sync_mode sync_mode, void *owner,
1485 			     bool intr)
1486 {
1487 	struct amdgpu_sync sync;
1488 	int r;
1489 
1490 	amdgpu_sync_create(&sync);
1491 	amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner);
1492 	r = amdgpu_sync_wait(&sync, intr);
1493 	amdgpu_sync_free(&sync);
1494 	return r;
1495 }
1496 
1497 /**
1498  * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv
1499  * @bo: buffer object to wait for
1500  * @owner: fence owner
1501  * @intr: Whether the wait is interruptible
1502  *
1503  * Wrapper to wait for fences in a BO.
1504  * Returns:
1505  * 0 on success, errno otherwise.
1506  */
1507 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr)
1508 {
1509 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1510 
1511 	return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv,
1512 					AMDGPU_SYNC_NE_OWNER, owner, intr);
1513 }
1514 
1515 /**
1516  * amdgpu_bo_gpu_offset - return GPU offset of bo
1517  * @bo:	amdgpu object for which we query the offset
1518  *
1519  * Note: object should either be pinned or reserved when calling this
1520  * function, it might be useful to add check for this for debugging.
1521  *
1522  * Returns:
1523  * current GPU offset of the object.
1524  */
1525 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
1526 {
1527 	WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM);
1528 	WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) &&
1529 		     !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel);
1530 	WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET);
1531 	WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM &&
1532 		     !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));
1533 
1534 	return amdgpu_bo_gpu_offset_no_check(bo);
1535 }
1536 
1537 /**
1538  * amdgpu_bo_fb_aper_addr - return FB aperture GPU offset of the VRAM bo
1539  * @bo:	amdgpu VRAM buffer object for which we query the offset
1540  *
1541  * Returns:
1542  * current FB aperture GPU offset of the object.
1543  */
1544 u64 amdgpu_bo_fb_aper_addr(struct amdgpu_bo *bo)
1545 {
1546 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1547 	uint64_t offset, fb_base;
1548 
1549 	WARN_ON_ONCE(bo->tbo.resource->mem_type != TTM_PL_VRAM);
1550 
1551 	fb_base = adev->gmc.fb_start;
1552 	fb_base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1553 	offset = (bo->tbo.resource->start << PAGE_SHIFT) + fb_base;
1554 	return amdgpu_gmc_sign_extend(offset);
1555 }
1556 
1557 /**
1558  * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo
1559  * @bo:	amdgpu object for which we query the offset
1560  *
1561  * Returns:
1562  * current GPU offset of the object without raising warnings.
1563  */
1564 u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo)
1565 {
1566 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1567 	uint64_t offset = AMDGPU_BO_INVALID_OFFSET;
1568 
1569 	if (bo->tbo.resource->mem_type == TTM_PL_TT)
1570 		offset = amdgpu_gmc_agp_addr(&bo->tbo);
1571 
1572 	if (offset == AMDGPU_BO_INVALID_OFFSET)
1573 		offset = (bo->tbo.resource->start << PAGE_SHIFT) +
1574 			amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type);
1575 
1576 	return amdgpu_gmc_sign_extend(offset);
1577 }
1578 
1579 /**
1580  * amdgpu_bo_mem_stats_placement - bo placement for memory accounting
1581  * @bo:	the buffer object we should look at
1582  *
1583  * BO can have multiple preferred placements, to avoid double counting we want
1584  * to file it under a single placement for memory stats.
1585  * Luckily, if we take the highest set bit in preferred_domains the result is
1586  * quite sensible.
1587  *
1588  * Returns:
1589  * Which of the placements should the BO be accounted under.
1590  */
1591 uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo)
1592 {
1593 	u32 domain;
1594 
1595 	/*
1596 	 * MMIO_REMAP is internal now, so it no longer maps from a userspace
1597 	 * domain bit. Keep fdinfo/mem-stats visibility by checking the actual
1598 	 * TTM placement.
1599 	 */
1600 	if (bo->tbo.resource && bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP)
1601 		return AMDGPU_PL_MMIO_REMAP;
1602 
1603 	domain = bo->preferred_domains & AMDGPU_GEM_DOMAIN_MASK;
1604 	if (!domain)
1605 		return TTM_PL_SYSTEM;
1606 
1607 	switch (rounddown_pow_of_two(domain)) {
1608 	case AMDGPU_GEM_DOMAIN_CPU:
1609 		return TTM_PL_SYSTEM;
1610 	case AMDGPU_GEM_DOMAIN_GTT:
1611 		return TTM_PL_TT;
1612 	case AMDGPU_GEM_DOMAIN_VRAM:
1613 		return TTM_PL_VRAM;
1614 	case AMDGPU_GEM_DOMAIN_GDS:
1615 		return AMDGPU_PL_GDS;
1616 	case AMDGPU_GEM_DOMAIN_GWS:
1617 		return AMDGPU_PL_GWS;
1618 	case AMDGPU_GEM_DOMAIN_OA:
1619 		return AMDGPU_PL_OA;
1620 	case AMDGPU_GEM_DOMAIN_DOORBELL:
1621 		return AMDGPU_PL_DOORBELL;
1622 	default:
1623 		return TTM_PL_SYSTEM;
1624 	}
1625 }
1626 
1627 /**
1628  * amdgpu_bo_get_preferred_domain - get preferred domain
1629  * @adev: amdgpu device object
1630  * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>`
1631  *
1632  * Returns:
1633  * Which of the allowed domains is preferred for allocating the BO.
1634  */
1635 uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,
1636 					    uint32_t domain)
1637 {
1638 	if ((domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) &&
1639 	    ((adev->asic_type == CHIP_CARRIZO) || (adev->asic_type == CHIP_STONEY))) {
1640 		domain = AMDGPU_GEM_DOMAIN_VRAM;
1641 		if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD)
1642 			domain = AMDGPU_GEM_DOMAIN_GTT;
1643 	}
1644 	return domain;
1645 }
1646 
1647 #if defined(CONFIG_DEBUG_FS)
1648 #define amdgpu_bo_print_flag(m, bo, flag)		        \
1649 	do {							\
1650 		if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) {	\
1651 			seq_printf((m), " " #flag);		\
1652 		}						\
1653 	} while (0)
1654 
1655 /**
1656  * amdgpu_bo_print_info - print BO info in debugfs file
1657  *
1658  * @id: Index or Id of the BO
1659  * @bo: Requested BO for printing info
1660  * @m: debugfs file
1661  *
1662  * Print BO information in debugfs file
1663  *
1664  * Returns:
1665  * Size of the BO in bytes.
1666  */
1667 u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m)
1668 {
1669 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1670 	struct dma_buf_attachment *attachment;
1671 	struct dma_buf *dma_buf;
1672 	const char *placement;
1673 	unsigned int pin_count;
1674 	u64 size;
1675 
1676 	if (dma_resv_trylock(bo->tbo.base.resv)) {
1677 		if (!bo->tbo.resource) {
1678 			placement = "NONE";
1679 		} else {
1680 			switch (bo->tbo.resource->mem_type) {
1681 			case TTM_PL_VRAM:
1682 				if (amdgpu_res_cpu_visible(adev, bo->tbo.resource))
1683 					placement = "VRAM VISIBLE";
1684 				else
1685 					placement = "VRAM";
1686 				break;
1687 			case TTM_PL_TT:
1688 				placement = "GTT";
1689 				break;
1690 			case AMDGPU_PL_GDS:
1691 				placement = "GDS";
1692 				break;
1693 			case AMDGPU_PL_GWS:
1694 				placement = "GWS";
1695 				break;
1696 			case AMDGPU_PL_OA:
1697 				placement = "OA";
1698 				break;
1699 			case AMDGPU_PL_PREEMPT:
1700 				placement = "PREEMPTIBLE";
1701 				break;
1702 			case AMDGPU_PL_DOORBELL:
1703 				placement = "DOORBELL";
1704 				break;
1705 			case AMDGPU_PL_MMIO_REMAP:
1706 				placement = "MMIO REMAP";
1707 				break;
1708 			case AMDGPU_PL_NPA:
1709 				placement = "NPA";
1710 				break;
1711 			case TTM_PL_SYSTEM:
1712 			default:
1713 				placement = "CPU";
1714 				break;
1715 			}
1716 		}
1717 		dma_resv_unlock(bo->tbo.base.resv);
1718 	} else {
1719 		placement = "UNKNOWN";
1720 	}
1721 
1722 	size = amdgpu_bo_size(bo);
1723 	seq_printf(m, "\t\t0x%08x: %12lld byte %s",
1724 			id, size, placement);
1725 
1726 	pin_count = READ_ONCE(bo->tbo.pin_count);
1727 	if (pin_count)
1728 		seq_printf(m, " pin count %d", pin_count);
1729 
1730 	dma_buf = READ_ONCE(bo->tbo.base.dma_buf);
1731 	attachment = READ_ONCE(bo->tbo.base.import_attach);
1732 
1733 	if (attachment)
1734 		seq_printf(m, " imported from ino:%llu", file_inode(dma_buf->file)->i_ino);
1735 	else if (dma_buf)
1736 		seq_printf(m, " exported as ino:%llu", file_inode(dma_buf->file)->i_ino);
1737 
1738 	amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED);
1739 	amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS);
1740 	amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC);
1741 	amdgpu_bo_print_flag(m, bo, VRAM_CLEARED);
1742 	amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS);
1743 	amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID);
1744 	amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC);
1745 	/* Add the gem obj resv fence dump*/
1746 	if (dma_resv_trylock(bo->tbo.base.resv)) {
1747 		dma_resv_describe(bo->tbo.base.resv, m);
1748 		dma_resv_unlock(bo->tbo.base.resv);
1749 	}
1750 	seq_puts(m, "\n");
1751 
1752 	return size;
1753 }
1754 #endif
1755