1 /*
2 * Copyright 2009 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26 /*
27 * Authors:
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
30 * Dave Airlie
31 */
32 #include <linux/list.h>
33 #include <linux/slab.h>
34 #include <linux/dma-buf.h>
35
36 #include <drm/drm_drv.h>
37 #include <drm/amdgpu_drm.h>
38 #include <drm/drm_cache.h>
39 #include "amdgpu.h"
40 #include "amdgpu_trace.h"
41 #include "amdgpu_amdkfd.h"
42 #include "amdgpu_vram_mgr.h"
43 #include "amdgpu_vm.h"
44 #include "amdgpu_dma_buf.h"
45
46 /**
47 * DOC: amdgpu_object
48 *
49 * This defines the interfaces to operate on an &amdgpu_bo buffer object which
50 * represents memory used by driver (VRAM, system memory, etc.). The driver
51 * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces
52 * to create/destroy/set buffer object which are then managed by the kernel TTM
53 * memory manager.
54 * The interfaces are also used internally by kernel clients, including gfx,
55 * uvd, etc. for kernel managed allocations used by the GPU.
56 *
57 */
58
amdgpu_bo_destroy(struct ttm_buffer_object * tbo)59 static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo)
60 {
61 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
62
63 amdgpu_bo_kunmap(bo);
64
65 if (bo->tbo.base.import_attach)
66 drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg);
67 drm_gem_object_release(&bo->tbo.base);
68 amdgpu_bo_unref(&bo->parent);
69 kvfree(bo);
70 }
71
amdgpu_bo_user_destroy(struct ttm_buffer_object * tbo)72 static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo)
73 {
74 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
75 struct amdgpu_bo_user *ubo;
76
77 ubo = to_amdgpu_bo_user(bo);
78 kfree(ubo->metadata);
79 amdgpu_bo_destroy(tbo);
80 }
81
82 /**
83 * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo
84 * @bo: buffer object to be checked
85 *
86 * Uses destroy function associated with the object to determine if this is
87 * an &amdgpu_bo.
88 *
89 * Returns:
90 * true if the object belongs to &amdgpu_bo, false if not.
91 */
amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object * bo)92 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
93 {
94 if (bo->destroy == &amdgpu_bo_destroy ||
95 bo->destroy == &amdgpu_bo_user_destroy)
96 return true;
97
98 return false;
99 }
100
101 /**
102 * amdgpu_bo_placement_from_domain - set buffer's placement
103 * @abo: &amdgpu_bo buffer object whose placement is to be set
104 * @domain: requested domain
105 *
106 * Sets buffer's placement according to requested domain and the buffer's
107 * flags.
108 */
amdgpu_bo_placement_from_domain(struct amdgpu_bo * abo,u32 domain)109 void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
110 {
111 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
112 struct ttm_placement *placement = &abo->placement;
113 struct ttm_place *places = abo->placements;
114 u64 flags = abo->flags;
115 u32 c = 0;
116
117 if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
118 unsigned int visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
119 int8_t mem_id = KFD_XCP_MEM_ID(adev, abo->xcp_id);
120
121 if (adev->gmc.mem_partitions && mem_id >= 0) {
122 places[c].fpfn = adev->gmc.mem_partitions[mem_id].range.fpfn;
123 /*
124 * memory partition range lpfn is inclusive start + size - 1
125 * TTM place lpfn is exclusive start + size
126 */
127 places[c].lpfn = adev->gmc.mem_partitions[mem_id].range.lpfn + 1;
128 } else {
129 places[c].fpfn = 0;
130 places[c].lpfn = 0;
131 }
132 places[c].mem_type = TTM_PL_VRAM;
133 places[c].flags = 0;
134
135 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
136 places[c].lpfn = min_not_zero(places[c].lpfn, visible_pfn);
137 else
138 places[c].flags |= TTM_PL_FLAG_TOPDOWN;
139
140 if (abo->tbo.type == ttm_bo_type_kernel &&
141 flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
142 places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;
143
144 c++;
145 }
146
147 if (domain & AMDGPU_GEM_DOMAIN_DOORBELL) {
148 places[c].fpfn = 0;
149 places[c].lpfn = 0;
150 places[c].mem_type = AMDGPU_PL_DOORBELL;
151 places[c].flags = 0;
152 c++;
153 }
154
155 if (domain & AMDGPU_GEM_DOMAIN_GTT) {
156 places[c].fpfn = 0;
157 places[c].lpfn = 0;
158 places[c].mem_type =
159 abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ?
160 AMDGPU_PL_PREEMPT : TTM_PL_TT;
161 places[c].flags = 0;
162 /*
163 * When GTT is just an alternative to VRAM make sure that we
164 * only use it as fallback and still try to fill up VRAM first.
165 */
166 if (abo->tbo.resource && !(adev->flags & AMD_IS_APU) &&
167 domain & abo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)
168 places[c].flags |= TTM_PL_FLAG_FALLBACK;
169 c++;
170 }
171
172 if (domain & AMDGPU_GEM_DOMAIN_CPU) {
173 places[c].fpfn = 0;
174 places[c].lpfn = 0;
175 places[c].mem_type = TTM_PL_SYSTEM;
176 places[c].flags = 0;
177 c++;
178 }
179
180 if (domain & AMDGPU_GEM_DOMAIN_GDS) {
181 places[c].fpfn = 0;
182 places[c].lpfn = 0;
183 places[c].mem_type = AMDGPU_PL_GDS;
184 places[c].flags = 0;
185 c++;
186 }
187
188 if (domain & AMDGPU_GEM_DOMAIN_GWS) {
189 places[c].fpfn = 0;
190 places[c].lpfn = 0;
191 places[c].mem_type = AMDGPU_PL_GWS;
192 places[c].flags = 0;
193 c++;
194 }
195
196 if (domain & AMDGPU_GEM_DOMAIN_OA) {
197 places[c].fpfn = 0;
198 places[c].lpfn = 0;
199 places[c].mem_type = AMDGPU_PL_OA;
200 places[c].flags = 0;
201 c++;
202 }
203
204 if (!c) {
205 places[c].fpfn = 0;
206 places[c].lpfn = 0;
207 places[c].mem_type = TTM_PL_SYSTEM;
208 places[c].flags = 0;
209 c++;
210 }
211
212 BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS);
213
214 placement->num_placement = c;
215 placement->placement = places;
216 }
217
218 /**
219 * amdgpu_bo_create_reserved - create reserved BO for kernel use
220 *
221 * @adev: amdgpu device object
222 * @size: size for the new BO
223 * @align: alignment for the new BO
224 * @domain: where to place it
225 * @bo_ptr: used to initialize BOs in structures
226 * @gpu_addr: GPU addr of the pinned BO
227 * @cpu_addr: optional CPU address mapping
228 *
229 * Allocates and pins a BO for kernel internal use, and returns it still
230 * reserved.
231 *
232 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
233 *
234 * Returns:
235 * 0 on success, negative error code otherwise.
236 */
amdgpu_bo_create_reserved(struct amdgpu_device * adev,unsigned long size,int align,u32 domain,struct amdgpu_bo ** bo_ptr,u64 * gpu_addr,void ** cpu_addr)237 int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
238 unsigned long size, int align,
239 u32 domain, struct amdgpu_bo **bo_ptr,
240 u64 *gpu_addr, void **cpu_addr)
241 {
242 struct amdgpu_bo_param bp;
243 bool free = false;
244 int r;
245
246 if (!size) {
247 amdgpu_bo_unref(bo_ptr);
248 return 0;
249 }
250
251 memset(&bp, 0, sizeof(bp));
252 bp.size = size;
253 bp.byte_align = align;
254 bp.domain = domain;
255 bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
256 : AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
257 bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
258 bp.type = ttm_bo_type_kernel;
259 bp.resv = NULL;
260 bp.bo_ptr_size = sizeof(struct amdgpu_bo);
261
262 if (!*bo_ptr) {
263 r = amdgpu_bo_create(adev, &bp, bo_ptr);
264 if (r) {
265 dev_err(adev->dev, "(%d) failed to allocate kernel bo\n",
266 r);
267 return r;
268 }
269 free = true;
270 }
271
272 r = amdgpu_bo_reserve(*bo_ptr, false);
273 if (r) {
274 dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
275 goto error_free;
276 }
277
278 r = amdgpu_bo_pin(*bo_ptr, domain);
279 if (r) {
280 dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
281 goto error_unreserve;
282 }
283
284 r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo);
285 if (r) {
286 dev_err(adev->dev, "%p bind failed\n", *bo_ptr);
287 goto error_unpin;
288 }
289
290 if (gpu_addr)
291 *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr);
292
293 if (cpu_addr) {
294 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
295 if (r) {
296 dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
297 goto error_unpin;
298 }
299 }
300
301 return 0;
302
303 error_unpin:
304 amdgpu_bo_unpin(*bo_ptr);
305 error_unreserve:
306 amdgpu_bo_unreserve(*bo_ptr);
307
308 error_free:
309 if (free)
310 amdgpu_bo_unref(bo_ptr);
311
312 return r;
313 }
314
315 /**
316 * amdgpu_bo_create_kernel - create BO for kernel use
317 *
318 * @adev: amdgpu device object
319 * @size: size for the new BO
320 * @align: alignment for the new BO
321 * @domain: where to place it
322 * @bo_ptr: used to initialize BOs in structures
323 * @gpu_addr: GPU addr of the pinned BO
324 * @cpu_addr: optional CPU address mapping
325 *
326 * Allocates and pins a BO for kernel internal use.
327 *
328 * This function is exported to allow the V4L2 isp device
329 * external to drm device to create and access the kernel BO.
330 *
331 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
332 *
333 * Returns:
334 * 0 on success, negative error code otherwise.
335 */
amdgpu_bo_create_kernel(struct amdgpu_device * adev,unsigned long size,int align,u32 domain,struct amdgpu_bo ** bo_ptr,u64 * gpu_addr,void ** cpu_addr)336 int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
337 unsigned long size, int align,
338 u32 domain, struct amdgpu_bo **bo_ptr,
339 u64 *gpu_addr, void **cpu_addr)
340 {
341 int r;
342
343 r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr,
344 gpu_addr, cpu_addr);
345
346 if (r)
347 return r;
348
349 if (*bo_ptr)
350 amdgpu_bo_unreserve(*bo_ptr);
351
352 return 0;
353 }
354 EXPORT_SYMBOL(amdgpu_bo_create_kernel);
355
356 /**
357 * amdgpu_bo_create_isp_user - create user BO for isp
358 *
359 * @adev: amdgpu device object
360 * @dma_buf: DMABUF handle for isp buffer
361 * @domain: where to place it
362 * @bo: used to initialize BOs in structures
363 * @gpu_addr: GPU addr of the pinned BO
364 *
365 * Imports isp DMABUF to allocate and pin a user BO for isp internal use. It does
366 * GART alloc to generate gpu_addr for BO to make it accessible through the
367 * GART aperture for ISP HW.
368 *
369 * This function is exported to allow the V4L2 isp device external to drm device
370 * to create and access the isp user BO.
371 *
372 * Returns:
373 * 0 on success, negative error code otherwise.
374 */
amdgpu_bo_create_isp_user(struct amdgpu_device * adev,struct dma_buf * dma_buf,u32 domain,struct amdgpu_bo ** bo,u64 * gpu_addr)375 int amdgpu_bo_create_isp_user(struct amdgpu_device *adev,
376 struct dma_buf *dma_buf, u32 domain, struct amdgpu_bo **bo,
377 u64 *gpu_addr)
378
379 {
380 struct drm_gem_object *gem_obj;
381 int r;
382
383 gem_obj = amdgpu_gem_prime_import(&adev->ddev, dma_buf);
384 *bo = gem_to_amdgpu_bo(gem_obj);
385 if (!(*bo)) {
386 dev_err(adev->dev, "failed to get valid isp user bo\n");
387 return -EINVAL;
388 }
389
390 r = amdgpu_bo_reserve(*bo, false);
391 if (r) {
392 dev_err(adev->dev, "(%d) failed to reserve isp user bo\n", r);
393 return r;
394 }
395
396 r = amdgpu_bo_pin(*bo, domain);
397 if (r) {
398 dev_err(adev->dev, "(%d) isp user bo pin failed\n", r);
399 goto error_unreserve;
400 }
401
402 r = amdgpu_ttm_alloc_gart(&(*bo)->tbo);
403 if (r) {
404 dev_err(adev->dev, "%p bind failed\n", *bo);
405 goto error_unpin;
406 }
407
408 if (!WARN_ON(!gpu_addr))
409 *gpu_addr = amdgpu_bo_gpu_offset(*bo);
410
411 amdgpu_bo_unreserve(*bo);
412
413 return 0;
414
415 error_unpin:
416 amdgpu_bo_unpin(*bo);
417 error_unreserve:
418 amdgpu_bo_unreserve(*bo);
419 amdgpu_bo_unref(bo);
420
421 return r;
422 }
423 EXPORT_SYMBOL(amdgpu_bo_create_isp_user);
424
425 /**
426 * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location
427 *
428 * @adev: amdgpu device object
429 * @offset: offset of the BO
430 * @size: size of the BO
431 * @bo_ptr: used to initialize BOs in structures
432 * @cpu_addr: optional CPU address mapping
433 *
434 * Creates a kernel BO at a specific offset in VRAM.
435 *
436 * Returns:
437 * 0 on success, negative error code otherwise.
438 */
amdgpu_bo_create_kernel_at(struct amdgpu_device * adev,uint64_t offset,uint64_t size,struct amdgpu_bo ** bo_ptr,void ** cpu_addr)439 int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev,
440 uint64_t offset, uint64_t size,
441 struct amdgpu_bo **bo_ptr, void **cpu_addr)
442 {
443 struct ttm_operation_ctx ctx = { false, false };
444 unsigned int i;
445 int r;
446
447 offset &= PAGE_MASK;
448 size = ALIGN(size, PAGE_SIZE);
449
450 r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE,
451 AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL,
452 cpu_addr);
453 if (r)
454 return r;
455
456 if ((*bo_ptr) == NULL)
457 return 0;
458
459 /*
460 * Remove the original mem node and create a new one at the request
461 * position.
462 */
463 if (cpu_addr)
464 amdgpu_bo_kunmap(*bo_ptr);
465
466 ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource);
467
468 for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) {
469 (*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT;
470 (*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
471 }
472 r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement,
473 &(*bo_ptr)->tbo.resource, &ctx);
474 if (r)
475 goto error;
476
477 if (cpu_addr) {
478 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
479 if (r)
480 goto error;
481 }
482
483 amdgpu_bo_unreserve(*bo_ptr);
484 return 0;
485
486 error:
487 amdgpu_bo_unreserve(*bo_ptr);
488 amdgpu_bo_unref(bo_ptr);
489 return r;
490 }
491
492 /**
493 * amdgpu_bo_free_kernel - free BO for kernel use
494 *
495 * @bo: amdgpu BO to free
496 * @gpu_addr: pointer to where the BO's GPU memory space address was stored
497 * @cpu_addr: pointer to where the BO's CPU memory space address was stored
498 *
499 * unmaps and unpin a BO for kernel internal use.
500 *
501 * This function is exported to allow the V4L2 isp device
502 * external to drm device to free the kernel BO.
503 */
amdgpu_bo_free_kernel(struct amdgpu_bo ** bo,u64 * gpu_addr,void ** cpu_addr)504 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
505 void **cpu_addr)
506 {
507 if (*bo == NULL)
508 return;
509
510 WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend);
511
512 if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {
513 if (cpu_addr)
514 amdgpu_bo_kunmap(*bo);
515
516 amdgpu_bo_unpin(*bo);
517 amdgpu_bo_unreserve(*bo);
518 }
519 amdgpu_bo_unref(bo);
520
521 if (gpu_addr)
522 *gpu_addr = 0;
523
524 if (cpu_addr)
525 *cpu_addr = NULL;
526 }
527 EXPORT_SYMBOL(amdgpu_bo_free_kernel);
528
529 /**
530 * amdgpu_bo_free_isp_user - free BO for isp use
531 *
532 * @bo: amdgpu isp user BO to free
533 *
534 * unpin and unref BO for isp internal use.
535 *
536 * This function is exported to allow the V4L2 isp device
537 * external to drm device to free the isp user BO.
538 */
amdgpu_bo_free_isp_user(struct amdgpu_bo * bo)539 void amdgpu_bo_free_isp_user(struct amdgpu_bo *bo)
540 {
541 if (bo == NULL)
542 return;
543
544 if (amdgpu_bo_reserve(bo, true) == 0) {
545 amdgpu_bo_unpin(bo);
546 amdgpu_bo_unreserve(bo);
547 }
548 amdgpu_bo_unref(&bo);
549 }
550 EXPORT_SYMBOL(amdgpu_bo_free_isp_user);
551
552 /* Validate bo size is bit bigger than the request domain */
amdgpu_bo_validate_size(struct amdgpu_device * adev,unsigned long size,u32 domain)553 static bool amdgpu_bo_validate_size(struct amdgpu_device *adev,
554 unsigned long size, u32 domain)
555 {
556 struct ttm_resource_manager *man = NULL;
557
558 /*
559 * If GTT is part of requested domains the check must succeed to
560 * allow fall back to GTT.
561 */
562 if (domain & AMDGPU_GEM_DOMAIN_GTT)
563 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
564 else if (domain & AMDGPU_GEM_DOMAIN_VRAM)
565 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
566 else
567 return true;
568
569 if (!man) {
570 if (domain & AMDGPU_GEM_DOMAIN_GTT)
571 WARN_ON_ONCE("GTT domain requested but GTT mem manager uninitialized");
572 return false;
573 }
574
575 /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU, _DOMAIN_DOORBELL */
576 if (size < man->size)
577 return true;
578
579 DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, man->size);
580 return false;
581 }
582
amdgpu_bo_support_uswc(u64 bo_flags)583 bool amdgpu_bo_support_uswc(u64 bo_flags)
584 {
585
586 #ifdef CONFIG_X86_32
587 /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit
588 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627
589 */
590 return false;
591 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
592 /* Don't try to enable write-combining when it can't work, or things
593 * may be slow
594 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758
595 */
596
597 #ifndef CONFIG_COMPILE_TEST
598 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
599 thanks to write-combining
600 #endif
601
602 if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
603 DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
604 "better performance thanks to write-combining\n");
605 return false;
606 #else
607 /* For architectures that don't support WC memory,
608 * mask out the WC flag from the BO
609 */
610 if (!drm_arch_can_wc_memory())
611 return false;
612
613 return true;
614 #endif
615 }
616
617 /**
618 * amdgpu_bo_create - create an &amdgpu_bo buffer object
619 * @adev: amdgpu device object
620 * @bp: parameters to be used for the buffer object
621 * @bo_ptr: pointer to the buffer object pointer
622 *
623 * Creates an &amdgpu_bo buffer object.
624 *
625 * Returns:
626 * 0 for success or a negative error code on failure.
627 */
amdgpu_bo_create(struct amdgpu_device * adev,struct amdgpu_bo_param * bp,struct amdgpu_bo ** bo_ptr)628 int amdgpu_bo_create(struct amdgpu_device *adev,
629 struct amdgpu_bo_param *bp,
630 struct amdgpu_bo **bo_ptr)
631 {
632 struct ttm_operation_ctx ctx = {
633 .interruptible = (bp->type != ttm_bo_type_kernel),
634 .no_wait_gpu = bp->no_wait_gpu,
635 /* We opt to avoid OOM on system pages allocations */
636 .gfp_retry_mayfail = true,
637 .allow_res_evict = bp->type != ttm_bo_type_kernel,
638 .resv = bp->resv
639 };
640 struct amdgpu_bo *bo;
641 unsigned long page_align, size = bp->size;
642 int r;
643
644 /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */
645 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
646 /* GWS and OA don't need any alignment. */
647 page_align = bp->byte_align;
648 size <<= PAGE_SHIFT;
649
650 } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) {
651 /* Both size and alignment must be a multiple of 4. */
652 page_align = ALIGN(bp->byte_align, 4);
653 size = ALIGN(size, 4) << PAGE_SHIFT;
654 } else {
655 /* Memory should be aligned at least to a page size. */
656 page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT;
657 size = ALIGN(size, PAGE_SIZE);
658 }
659
660 if (!amdgpu_bo_validate_size(adev, size, bp->domain))
661 return -ENOMEM;
662
663 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo));
664
665 *bo_ptr = NULL;
666 bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL);
667 if (bo == NULL)
668 return -ENOMEM;
669 drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size);
670 bo->tbo.base.funcs = &amdgpu_gem_object_funcs;
671 bo->vm_bo = NULL;
672 bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain :
673 bp->domain;
674 bo->allowed_domains = bo->preferred_domains;
675 if (bp->type != ttm_bo_type_kernel &&
676 !(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE) &&
677 bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
678 bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
679
680 bo->flags = bp->flags;
681
682 if (adev->gmc.mem_partitions)
683 /* For GPUs with spatial partitioning, bo->xcp_id=-1 means any partition */
684 bo->xcp_id = bp->xcp_id_plus1 - 1;
685 else
686 /* For GPUs without spatial partitioning */
687 bo->xcp_id = 0;
688
689 if (!amdgpu_bo_support_uswc(bo->flags))
690 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
691
692 bo->tbo.bdev = &adev->mman.bdev;
693 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA |
694 AMDGPU_GEM_DOMAIN_GDS))
695 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
696 else
697 amdgpu_bo_placement_from_domain(bo, bp->domain);
698 if (bp->type == ttm_bo_type_kernel)
699 bo->tbo.priority = 2;
700 else if (!(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE))
701 bo->tbo.priority = 1;
702
703 if (!bp->destroy)
704 bp->destroy = &amdgpu_bo_destroy;
705
706 r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type,
707 &bo->placement, page_align, &ctx, NULL,
708 bp->resv, bp->destroy);
709 if (unlikely(r != 0))
710 return r;
711
712 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
713 amdgpu_res_cpu_visible(adev, bo->tbo.resource))
714 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved,
715 ctx.bytes_moved);
716 else
717 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0);
718
719 if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
720 bo->tbo.resource->mem_type == TTM_PL_VRAM) {
721 struct dma_fence *fence;
722
723 r = amdgpu_ttm_clear_buffer(bo, bo->tbo.base.resv, &fence);
724 if (unlikely(r))
725 goto fail_unreserve;
726
727 dma_resv_add_fence(bo->tbo.base.resv, fence,
728 DMA_RESV_USAGE_KERNEL);
729 dma_fence_put(fence);
730 }
731 if (!bp->resv)
732 amdgpu_bo_unreserve(bo);
733 *bo_ptr = bo;
734
735 trace_amdgpu_bo_create(bo);
736
737 /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */
738 if (bp->type == ttm_bo_type_device)
739 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
740
741 return 0;
742
743 fail_unreserve:
744 if (!bp->resv)
745 dma_resv_unlock(bo->tbo.base.resv);
746 amdgpu_bo_unref(&bo);
747 return r;
748 }
749
750 /**
751 * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object
752 * @adev: amdgpu device object
753 * @bp: parameters to be used for the buffer object
754 * @ubo_ptr: pointer to the buffer object pointer
755 *
756 * Create a BO to be used by user application;
757 *
758 * Returns:
759 * 0 for success or a negative error code on failure.
760 */
761
amdgpu_bo_create_user(struct amdgpu_device * adev,struct amdgpu_bo_param * bp,struct amdgpu_bo_user ** ubo_ptr)762 int amdgpu_bo_create_user(struct amdgpu_device *adev,
763 struct amdgpu_bo_param *bp,
764 struct amdgpu_bo_user **ubo_ptr)
765 {
766 struct amdgpu_bo *bo_ptr;
767 int r;
768
769 bp->bo_ptr_size = sizeof(struct amdgpu_bo_user);
770 bp->destroy = &amdgpu_bo_user_destroy;
771 r = amdgpu_bo_create(adev, bp, &bo_ptr);
772 if (r)
773 return r;
774
775 *ubo_ptr = to_amdgpu_bo_user(bo_ptr);
776 return r;
777 }
778
779 /**
780 * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object
781 * @adev: amdgpu device object
782 * @bp: parameters to be used for the buffer object
783 * @vmbo_ptr: pointer to the buffer object pointer
784 *
785 * Create a BO to be for GPUVM.
786 *
787 * Returns:
788 * 0 for success or a negative error code on failure.
789 */
790
amdgpu_bo_create_vm(struct amdgpu_device * adev,struct amdgpu_bo_param * bp,struct amdgpu_bo_vm ** vmbo_ptr)791 int amdgpu_bo_create_vm(struct amdgpu_device *adev,
792 struct amdgpu_bo_param *bp,
793 struct amdgpu_bo_vm **vmbo_ptr)
794 {
795 struct amdgpu_bo *bo_ptr;
796 int r;
797
798 /* bo_ptr_size will be determined by the caller and it depends on
799 * num of amdgpu_vm_pt entries.
800 */
801 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm));
802 r = amdgpu_bo_create(adev, bp, &bo_ptr);
803 if (r)
804 return r;
805
806 *vmbo_ptr = to_amdgpu_bo_vm(bo_ptr);
807 return r;
808 }
809
810 /**
811 * amdgpu_bo_kmap - map an &amdgpu_bo buffer object
812 * @bo: &amdgpu_bo buffer object to be mapped
813 * @ptr: kernel virtual address to be returned
814 *
815 * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls
816 * amdgpu_bo_kptr() to get the kernel virtual address.
817 *
818 * Returns:
819 * 0 for success or a negative error code on failure.
820 */
amdgpu_bo_kmap(struct amdgpu_bo * bo,void ** ptr)821 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
822 {
823 void *kptr;
824 long r;
825
826 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
827 return -EPERM;
828
829 r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL,
830 false, MAX_SCHEDULE_TIMEOUT);
831 if (r < 0)
832 return r;
833
834 kptr = amdgpu_bo_kptr(bo);
835 if (kptr) {
836 if (ptr)
837 *ptr = kptr;
838 return 0;
839 }
840
841 r = ttm_bo_kmap(&bo->tbo, 0, PFN_UP(bo->tbo.base.size), &bo->kmap);
842 if (r)
843 return r;
844
845 if (ptr)
846 *ptr = amdgpu_bo_kptr(bo);
847
848 return 0;
849 }
850
851 /**
852 * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object
853 * @bo: &amdgpu_bo buffer object
854 *
855 * Calls ttm_kmap_obj_virtual() to get the kernel virtual address
856 *
857 * Returns:
858 * the virtual address of a buffer object area.
859 */
amdgpu_bo_kptr(struct amdgpu_bo * bo)860 void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
861 {
862 bool is_iomem;
863
864 return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
865 }
866
867 /**
868 * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object
869 * @bo: &amdgpu_bo buffer object to be unmapped
870 *
871 * Unmaps a kernel map set up by amdgpu_bo_kmap().
872 */
amdgpu_bo_kunmap(struct amdgpu_bo * bo)873 void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
874 {
875 if (bo->kmap.bo)
876 ttm_bo_kunmap(&bo->kmap);
877 }
878
879 /**
880 * amdgpu_bo_ref - reference an &amdgpu_bo buffer object
881 * @bo: &amdgpu_bo buffer object
882 *
883 * References the contained &ttm_buffer_object.
884 *
885 * Returns:
886 * a refcounted pointer to the &amdgpu_bo buffer object.
887 */
amdgpu_bo_ref(struct amdgpu_bo * bo)888 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
889 {
890 if (bo == NULL)
891 return NULL;
892
893 drm_gem_object_get(&bo->tbo.base);
894 return bo;
895 }
896
897 /**
898 * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object
899 * @bo: &amdgpu_bo buffer object
900 *
901 * Unreferences the contained &ttm_buffer_object and clear the pointer
902 */
amdgpu_bo_unref(struct amdgpu_bo ** bo)903 void amdgpu_bo_unref(struct amdgpu_bo **bo)
904 {
905 if ((*bo) == NULL)
906 return;
907
908 drm_gem_object_put(&(*bo)->tbo.base);
909 *bo = NULL;
910 }
911
912 /**
913 * amdgpu_bo_pin - pin an &amdgpu_bo buffer object
914 * @bo: &amdgpu_bo buffer object to be pinned
915 * @domain: domain to be pinned to
916 *
917 * Pins the buffer object according to requested domain. If the memory is
918 * unbound gart memory, binds the pages into gart table. Adjusts pin_count and
919 * pin_size accordingly.
920 *
921 * Pinning means to lock pages in memory along with keeping them at a fixed
922 * offset. It is required when a buffer can not be moved, for example, when
923 * a display buffer is being scanned out.
924 *
925 * Returns:
926 * 0 for success or a negative error code on failure.
927 */
amdgpu_bo_pin(struct amdgpu_bo * bo,u32 domain)928 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain)
929 {
930 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
931 struct ttm_operation_ctx ctx = { false, false };
932 int r, i;
933
934 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
935 return -EPERM;
936
937 /* Check domain to be pinned to against preferred domains */
938 if (bo->preferred_domains & domain)
939 domain = bo->preferred_domains & domain;
940
941 /* A shared bo cannot be migrated to VRAM */
942 if (bo->tbo.base.import_attach) {
943 if (domain & AMDGPU_GEM_DOMAIN_GTT)
944 domain = AMDGPU_GEM_DOMAIN_GTT;
945 else
946 return -EINVAL;
947 }
948
949 if (bo->tbo.pin_count) {
950 uint32_t mem_type = bo->tbo.resource->mem_type;
951 uint32_t mem_flags = bo->tbo.resource->placement;
952
953 if (!(domain & amdgpu_mem_type_to_domain(mem_type)))
954 return -EINVAL;
955
956 if ((mem_type == TTM_PL_VRAM) &&
957 (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) &&
958 !(mem_flags & TTM_PL_FLAG_CONTIGUOUS))
959 return -EINVAL;
960
961 ttm_bo_pin(&bo->tbo);
962 return 0;
963 }
964
965 /* This assumes only APU display buffers are pinned with (VRAM|GTT).
966 * See function amdgpu_display_supported_domains()
967 */
968 domain = amdgpu_bo_get_preferred_domain(adev, domain);
969
970 if (bo->tbo.base.import_attach)
971 dma_buf_pin(bo->tbo.base.import_attach);
972
973 /* force to pin into visible video ram */
974 if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))
975 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
976 amdgpu_bo_placement_from_domain(bo, domain);
977 for (i = 0; i < bo->placement.num_placement; i++) {
978 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS &&
979 bo->placements[i].mem_type == TTM_PL_VRAM)
980 bo->placements[i].flags |= TTM_PL_FLAG_CONTIGUOUS;
981 }
982
983 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
984 if (unlikely(r)) {
985 dev_err(adev->dev, "%p pin failed\n", bo);
986 goto error;
987 }
988
989 ttm_bo_pin(&bo->tbo);
990
991 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {
992 atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size);
993 atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo),
994 &adev->visible_pin_size);
995 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
996 atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size);
997 }
998
999 error:
1000 return r;
1001 }
1002
1003 /**
1004 * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object
1005 * @bo: &amdgpu_bo buffer object to be unpinned
1006 *
1007 * Decreases the pin_count, and clears the flags if pin_count reaches 0.
1008 * Changes placement and pin size accordingly.
1009 *
1010 * Returns:
1011 * 0 for success or a negative error code on failure.
1012 */
amdgpu_bo_unpin(struct amdgpu_bo * bo)1013 void amdgpu_bo_unpin(struct amdgpu_bo *bo)
1014 {
1015 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1016
1017 ttm_bo_unpin(&bo->tbo);
1018 if (bo->tbo.pin_count)
1019 return;
1020
1021 if (bo->tbo.base.import_attach)
1022 dma_buf_unpin(bo->tbo.base.import_attach);
1023
1024 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {
1025 atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size);
1026 atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo),
1027 &adev->visible_pin_size);
1028 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
1029 atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size);
1030 }
1031
1032 }
1033
1034 static const char * const amdgpu_vram_names[] = {
1035 "UNKNOWN",
1036 "GDDR1",
1037 "DDR2",
1038 "GDDR3",
1039 "GDDR4",
1040 "GDDR5",
1041 "HBM",
1042 "DDR3",
1043 "DDR4",
1044 "GDDR6",
1045 "DDR5",
1046 "LPDDR4",
1047 "LPDDR5",
1048 "HBM3E"
1049 };
1050
1051 /**
1052 * amdgpu_bo_init - initialize memory manager
1053 * @adev: amdgpu device object
1054 *
1055 * Calls amdgpu_ttm_init() to initialize amdgpu memory manager.
1056 *
1057 * Returns:
1058 * 0 for success or a negative error code on failure.
1059 */
amdgpu_bo_init(struct amdgpu_device * adev)1060 int amdgpu_bo_init(struct amdgpu_device *adev)
1061 {
1062 /* On A+A platform, VRAM can be mapped as WB */
1063 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
1064 /* reserve PAT memory space to WC for VRAM */
1065 int r = arch_io_reserve_memtype_wc(adev->gmc.aper_base,
1066 adev->gmc.aper_size);
1067
1068 if (r) {
1069 DRM_ERROR("Unable to set WC memtype for the aperture base\n");
1070 return r;
1071 }
1072
1073 /* Add an MTRR for the VRAM */
1074 adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base,
1075 adev->gmc.aper_size);
1076 }
1077
1078 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
1079 adev->gmc.mc_vram_size >> 20,
1080 (unsigned long long)adev->gmc.aper_size >> 20);
1081 DRM_INFO("RAM width %dbits %s\n",
1082 adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]);
1083 return amdgpu_ttm_init(adev);
1084 }
1085
1086 /**
1087 * amdgpu_bo_fini - tear down memory manager
1088 * @adev: amdgpu device object
1089 *
1090 * Reverses amdgpu_bo_init() to tear down memory manager.
1091 */
amdgpu_bo_fini(struct amdgpu_device * adev)1092 void amdgpu_bo_fini(struct amdgpu_device *adev)
1093 {
1094 int idx;
1095
1096 amdgpu_ttm_fini(adev);
1097
1098 if (drm_dev_enter(adev_to_drm(adev), &idx)) {
1099 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
1100 arch_phys_wc_del(adev->gmc.vram_mtrr);
1101 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
1102 }
1103 drm_dev_exit(idx);
1104 }
1105 }
1106
1107 /**
1108 * amdgpu_bo_set_tiling_flags - set tiling flags
1109 * @bo: &amdgpu_bo buffer object
1110 * @tiling_flags: new flags
1111 *
1112 * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or
1113 * kernel driver to set the tiling flags on a buffer.
1114 *
1115 * Returns:
1116 * 0 for success or a negative error code on failure.
1117 */
amdgpu_bo_set_tiling_flags(struct amdgpu_bo * bo,u64 tiling_flags)1118 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
1119 {
1120 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1121 struct amdgpu_bo_user *ubo;
1122
1123 BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1124 if (adev->family <= AMDGPU_FAMILY_CZ &&
1125 AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
1126 return -EINVAL;
1127
1128 ubo = to_amdgpu_bo_user(bo);
1129 ubo->tiling_flags = tiling_flags;
1130 return 0;
1131 }
1132
1133 /**
1134 * amdgpu_bo_get_tiling_flags - get tiling flags
1135 * @bo: &amdgpu_bo buffer object
1136 * @tiling_flags: returned flags
1137 *
1138 * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to
1139 * set the tiling flags on a buffer.
1140 */
amdgpu_bo_get_tiling_flags(struct amdgpu_bo * bo,u64 * tiling_flags)1141 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
1142 {
1143 struct amdgpu_bo_user *ubo;
1144
1145 BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1146 dma_resv_assert_held(bo->tbo.base.resv);
1147 ubo = to_amdgpu_bo_user(bo);
1148
1149 if (tiling_flags)
1150 *tiling_flags = ubo->tiling_flags;
1151 }
1152
1153 /**
1154 * amdgpu_bo_set_metadata - set metadata
1155 * @bo: &amdgpu_bo buffer object
1156 * @metadata: new metadata
1157 * @metadata_size: size of the new metadata
1158 * @flags: flags of the new metadata
1159 *
1160 * Sets buffer object's metadata, its size and flags.
1161 * Used via GEM ioctl.
1162 *
1163 * Returns:
1164 * 0 for success or a negative error code on failure.
1165 */
amdgpu_bo_set_metadata(struct amdgpu_bo * bo,void * metadata,u32 metadata_size,uint64_t flags)1166 int amdgpu_bo_set_metadata(struct amdgpu_bo *bo, void *metadata,
1167 u32 metadata_size, uint64_t flags)
1168 {
1169 struct amdgpu_bo_user *ubo;
1170 void *buffer;
1171
1172 BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1173 ubo = to_amdgpu_bo_user(bo);
1174 if (!metadata_size) {
1175 if (ubo->metadata_size) {
1176 kfree(ubo->metadata);
1177 ubo->metadata = NULL;
1178 ubo->metadata_size = 0;
1179 }
1180 return 0;
1181 }
1182
1183 if (metadata == NULL)
1184 return -EINVAL;
1185
1186 buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
1187 if (buffer == NULL)
1188 return -ENOMEM;
1189
1190 kfree(ubo->metadata);
1191 ubo->metadata_flags = flags;
1192 ubo->metadata = buffer;
1193 ubo->metadata_size = metadata_size;
1194
1195 return 0;
1196 }
1197
1198 /**
1199 * amdgpu_bo_get_metadata - get metadata
1200 * @bo: &amdgpu_bo buffer object
1201 * @buffer: returned metadata
1202 * @buffer_size: size of the buffer
1203 * @metadata_size: size of the returned metadata
1204 * @flags: flags of the returned metadata
1205 *
1206 * Gets buffer object's metadata, its size and flags. buffer_size shall not be
1207 * less than metadata_size.
1208 * Used via GEM ioctl.
1209 *
1210 * Returns:
1211 * 0 for success or a negative error code on failure.
1212 */
amdgpu_bo_get_metadata(struct amdgpu_bo * bo,void * buffer,size_t buffer_size,uint32_t * metadata_size,uint64_t * flags)1213 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
1214 size_t buffer_size, uint32_t *metadata_size,
1215 uint64_t *flags)
1216 {
1217 struct amdgpu_bo_user *ubo;
1218
1219 if (!buffer && !metadata_size)
1220 return -EINVAL;
1221
1222 BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1223 ubo = to_amdgpu_bo_user(bo);
1224 if (metadata_size)
1225 *metadata_size = ubo->metadata_size;
1226
1227 if (buffer) {
1228 if (buffer_size < ubo->metadata_size)
1229 return -EINVAL;
1230
1231 if (ubo->metadata_size)
1232 memcpy(buffer, ubo->metadata, ubo->metadata_size);
1233 }
1234
1235 if (flags)
1236 *flags = ubo->metadata_flags;
1237
1238 return 0;
1239 }
1240
1241 /**
1242 * amdgpu_bo_move_notify - notification about a memory move
1243 * @bo: pointer to a buffer object
1244 * @evict: if this move is evicting the buffer from the graphics address space
1245 * @new_mem: new resource for backing the BO
1246 *
1247 * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs
1248 * bookkeeping.
1249 * TTM driver callback which is called when ttm moves a buffer.
1250 */
amdgpu_bo_move_notify(struct ttm_buffer_object * bo,bool evict,struct ttm_resource * new_mem)1251 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
1252 bool evict,
1253 struct ttm_resource *new_mem)
1254 {
1255 struct ttm_resource *old_mem = bo->resource;
1256 struct amdgpu_bo *abo;
1257
1258 if (!amdgpu_bo_is_amdgpu_bo(bo))
1259 return;
1260
1261 abo = ttm_to_amdgpu_bo(bo);
1262 amdgpu_vm_bo_move(abo, new_mem, evict);
1263
1264 amdgpu_bo_kunmap(abo);
1265
1266 if (abo->tbo.base.dma_buf && !abo->tbo.base.import_attach &&
1267 old_mem && old_mem->mem_type != TTM_PL_SYSTEM)
1268 dma_buf_move_notify(abo->tbo.base.dma_buf);
1269
1270 /* move_notify is called before move happens */
1271 trace_amdgpu_bo_move(abo, new_mem ? new_mem->mem_type : -1,
1272 old_mem ? old_mem->mem_type : -1);
1273 }
1274
1275 /**
1276 * amdgpu_bo_release_notify - notification about a BO being released
1277 * @bo: pointer to a buffer object
1278 *
1279 * Wipes VRAM buffers whose contents should not be leaked before the
1280 * memory is released.
1281 */
amdgpu_bo_release_notify(struct ttm_buffer_object * bo)1282 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)
1283 {
1284 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1285 struct dma_fence *fence = NULL;
1286 struct amdgpu_bo *abo;
1287 int r;
1288
1289 if (!amdgpu_bo_is_amdgpu_bo(bo))
1290 return;
1291
1292 abo = ttm_to_amdgpu_bo(bo);
1293
1294 WARN_ON(abo->vm_bo);
1295
1296 if (abo->kfd_bo)
1297 amdgpu_amdkfd_release_notify(abo);
1298
1299 /*
1300 * We lock the private dma_resv object here and since the BO is about to
1301 * be released nobody else should have a pointer to it.
1302 * So when this locking here fails something is wrong with the reference
1303 * counting.
1304 */
1305 if (WARN_ON_ONCE(!dma_resv_trylock(&bo->base._resv)))
1306 return;
1307
1308 amdgpu_amdkfd_remove_all_eviction_fences(abo);
1309
1310 if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM ||
1311 !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) ||
1312 adev->in_suspend || drm_dev_is_unplugged(adev_to_drm(adev)))
1313 goto out;
1314
1315 r = dma_resv_reserve_fences(&bo->base._resv, 1);
1316 if (r)
1317 goto out;
1318
1319 r = amdgpu_fill_buffer(abo, 0, &bo->base._resv, &fence, true);
1320 if (WARN_ON(r))
1321 goto out;
1322
1323 amdgpu_vram_mgr_set_cleared(bo->resource);
1324 dma_resv_add_fence(&bo->base._resv, fence, DMA_RESV_USAGE_KERNEL);
1325 dma_fence_put(fence);
1326
1327 out:
1328 dma_resv_unlock(&bo->base._resv);
1329 }
1330
1331 /**
1332 * amdgpu_bo_fault_reserve_notify - notification about a memory fault
1333 * @bo: pointer to a buffer object
1334 *
1335 * Notifies the driver we are taking a fault on this BO and have reserved it,
1336 * also performs bookkeeping.
1337 * TTM driver callback for dealing with vm faults.
1338 *
1339 * Returns:
1340 * 0 for success or a negative error code on failure.
1341 */
amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object * bo)1342 vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
1343 {
1344 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1345 struct ttm_operation_ctx ctx = { false, false };
1346 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1347 int r;
1348
1349 /* Remember that this BO was accessed by the CPU */
1350 abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1351
1352 if (amdgpu_res_cpu_visible(adev, bo->resource))
1353 return 0;
1354
1355 /* Can't move a pinned BO to visible VRAM */
1356 if (abo->tbo.pin_count > 0)
1357 return VM_FAULT_SIGBUS;
1358
1359 /* hurrah the memory is not visible ! */
1360 atomic64_inc(&adev->num_vram_cpu_page_faults);
1361 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
1362 AMDGPU_GEM_DOMAIN_GTT);
1363
1364 /* Avoid costly evictions; only set GTT as a busy placement */
1365 abo->placements[0].flags |= TTM_PL_FLAG_DESIRED;
1366
1367 r = ttm_bo_validate(bo, &abo->placement, &ctx);
1368 if (unlikely(r == -EBUSY || r == -ERESTARTSYS))
1369 return VM_FAULT_NOPAGE;
1370 else if (unlikely(r))
1371 return VM_FAULT_SIGBUS;
1372
1373 /* this should never happen */
1374 if (bo->resource->mem_type == TTM_PL_VRAM &&
1375 !amdgpu_res_cpu_visible(adev, bo->resource))
1376 return VM_FAULT_SIGBUS;
1377
1378 ttm_bo_move_to_lru_tail_unlocked(bo);
1379 return 0;
1380 }
1381
1382 /**
1383 * amdgpu_bo_fence - add fence to buffer object
1384 *
1385 * @bo: buffer object in question
1386 * @fence: fence to add
1387 * @shared: true if fence should be added shared
1388 *
1389 */
amdgpu_bo_fence(struct amdgpu_bo * bo,struct dma_fence * fence,bool shared)1390 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
1391 bool shared)
1392 {
1393 struct dma_resv *resv = bo->tbo.base.resv;
1394 int r;
1395
1396 r = dma_resv_reserve_fences(resv, 1);
1397 if (r) {
1398 /* As last resort on OOM we block for the fence */
1399 dma_fence_wait(fence, false);
1400 return;
1401 }
1402
1403 dma_resv_add_fence(resv, fence, shared ? DMA_RESV_USAGE_READ :
1404 DMA_RESV_USAGE_WRITE);
1405 }
1406
1407 /**
1408 * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences
1409 *
1410 * @adev: amdgpu device pointer
1411 * @resv: reservation object to sync to
1412 * @sync_mode: synchronization mode
1413 * @owner: fence owner
1414 * @intr: Whether the wait is interruptible
1415 *
1416 * Extract the fences from the reservation object and waits for them to finish.
1417 *
1418 * Returns:
1419 * 0 on success, errno otherwise.
1420 */
amdgpu_bo_sync_wait_resv(struct amdgpu_device * adev,struct dma_resv * resv,enum amdgpu_sync_mode sync_mode,void * owner,bool intr)1421 int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv,
1422 enum amdgpu_sync_mode sync_mode, void *owner,
1423 bool intr)
1424 {
1425 struct amdgpu_sync sync;
1426 int r;
1427
1428 amdgpu_sync_create(&sync);
1429 amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner);
1430 r = amdgpu_sync_wait(&sync, intr);
1431 amdgpu_sync_free(&sync);
1432 return r;
1433 }
1434
1435 /**
1436 * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv
1437 * @bo: buffer object to wait for
1438 * @owner: fence owner
1439 * @intr: Whether the wait is interruptible
1440 *
1441 * Wrapper to wait for fences in a BO.
1442 * Returns:
1443 * 0 on success, errno otherwise.
1444 */
amdgpu_bo_sync_wait(struct amdgpu_bo * bo,void * owner,bool intr)1445 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr)
1446 {
1447 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1448
1449 return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv,
1450 AMDGPU_SYNC_NE_OWNER, owner, intr);
1451 }
1452
1453 /**
1454 * amdgpu_bo_gpu_offset - return GPU offset of bo
1455 * @bo: amdgpu object for which we query the offset
1456 *
1457 * Note: object should either be pinned or reserved when calling this
1458 * function, it might be useful to add check for this for debugging.
1459 *
1460 * Returns:
1461 * current GPU offset of the object.
1462 */
amdgpu_bo_gpu_offset(struct amdgpu_bo * bo)1463 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
1464 {
1465 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM);
1466 WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) &&
1467 !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel);
1468 WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET);
1469 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM &&
1470 !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));
1471
1472 return amdgpu_bo_gpu_offset_no_check(bo);
1473 }
1474
1475 /**
1476 * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo
1477 * @bo: amdgpu object for which we query the offset
1478 *
1479 * Returns:
1480 * current GPU offset of the object without raising warnings.
1481 */
amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo * bo)1482 u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo)
1483 {
1484 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1485 uint64_t offset = AMDGPU_BO_INVALID_OFFSET;
1486
1487 if (bo->tbo.resource->mem_type == TTM_PL_TT)
1488 offset = amdgpu_gmc_agp_addr(&bo->tbo);
1489
1490 if (offset == AMDGPU_BO_INVALID_OFFSET)
1491 offset = (bo->tbo.resource->start << PAGE_SHIFT) +
1492 amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type);
1493
1494 return amdgpu_gmc_sign_extend(offset);
1495 }
1496
1497 /**
1498 * amdgpu_bo_mem_stats_placement - bo placement for memory accounting
1499 * @bo: the buffer object we should look at
1500 *
1501 * BO can have multiple preferred placements, to avoid double counting we want
1502 * to file it under a single placement for memory stats.
1503 * Luckily, if we take the highest set bit in preferred_domains the result is
1504 * quite sensible.
1505 *
1506 * Returns:
1507 * Which of the placements should the BO be accounted under.
1508 */
amdgpu_bo_mem_stats_placement(struct amdgpu_bo * bo)1509 uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo)
1510 {
1511 uint32_t domain = bo->preferred_domains & AMDGPU_GEM_DOMAIN_MASK;
1512
1513 if (!domain)
1514 return TTM_PL_SYSTEM;
1515
1516 switch (rounddown_pow_of_two(domain)) {
1517 case AMDGPU_GEM_DOMAIN_CPU:
1518 return TTM_PL_SYSTEM;
1519 case AMDGPU_GEM_DOMAIN_GTT:
1520 return TTM_PL_TT;
1521 case AMDGPU_GEM_DOMAIN_VRAM:
1522 return TTM_PL_VRAM;
1523 case AMDGPU_GEM_DOMAIN_GDS:
1524 return AMDGPU_PL_GDS;
1525 case AMDGPU_GEM_DOMAIN_GWS:
1526 return AMDGPU_PL_GWS;
1527 case AMDGPU_GEM_DOMAIN_OA:
1528 return AMDGPU_PL_OA;
1529 case AMDGPU_GEM_DOMAIN_DOORBELL:
1530 return AMDGPU_PL_DOORBELL;
1531 default:
1532 return TTM_PL_SYSTEM;
1533 }
1534 }
1535
1536 /**
1537 * amdgpu_bo_get_preferred_domain - get preferred domain
1538 * @adev: amdgpu device object
1539 * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>`
1540 *
1541 * Returns:
1542 * Which of the allowed domains is preferred for allocating the BO.
1543 */
amdgpu_bo_get_preferred_domain(struct amdgpu_device * adev,uint32_t domain)1544 uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,
1545 uint32_t domain)
1546 {
1547 if ((domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) &&
1548 ((adev->asic_type == CHIP_CARRIZO) || (adev->asic_type == CHIP_STONEY))) {
1549 domain = AMDGPU_GEM_DOMAIN_VRAM;
1550 if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD)
1551 domain = AMDGPU_GEM_DOMAIN_GTT;
1552 }
1553 return domain;
1554 }
1555
1556 #if defined(CONFIG_DEBUG_FS)
1557 #define amdgpu_bo_print_flag(m, bo, flag) \
1558 do { \
1559 if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \
1560 seq_printf((m), " " #flag); \
1561 } \
1562 } while (0)
1563
1564 /**
1565 * amdgpu_bo_print_info - print BO info in debugfs file
1566 *
1567 * @id: Index or Id of the BO
1568 * @bo: Requested BO for printing info
1569 * @m: debugfs file
1570 *
1571 * Print BO information in debugfs file
1572 *
1573 * Returns:
1574 * Size of the BO in bytes.
1575 */
amdgpu_bo_print_info(int id,struct amdgpu_bo * bo,struct seq_file * m)1576 u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m)
1577 {
1578 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1579 struct dma_buf_attachment *attachment;
1580 struct dma_buf *dma_buf;
1581 const char *placement;
1582 unsigned int pin_count;
1583 u64 size;
1584
1585 if (dma_resv_trylock(bo->tbo.base.resv)) {
1586 if (!bo->tbo.resource) {
1587 placement = "NONE";
1588 } else {
1589 switch (bo->tbo.resource->mem_type) {
1590 case TTM_PL_VRAM:
1591 if (amdgpu_res_cpu_visible(adev, bo->tbo.resource))
1592 placement = "VRAM VISIBLE";
1593 else
1594 placement = "VRAM";
1595 break;
1596 case TTM_PL_TT:
1597 placement = "GTT";
1598 break;
1599 case AMDGPU_PL_GDS:
1600 placement = "GDS";
1601 break;
1602 case AMDGPU_PL_GWS:
1603 placement = "GWS";
1604 break;
1605 case AMDGPU_PL_OA:
1606 placement = "OA";
1607 break;
1608 case AMDGPU_PL_PREEMPT:
1609 placement = "PREEMPTIBLE";
1610 break;
1611 case AMDGPU_PL_DOORBELL:
1612 placement = "DOORBELL";
1613 break;
1614 case TTM_PL_SYSTEM:
1615 default:
1616 placement = "CPU";
1617 break;
1618 }
1619 }
1620 dma_resv_unlock(bo->tbo.base.resv);
1621 } else {
1622 placement = "UNKNOWN";
1623 }
1624
1625 size = amdgpu_bo_size(bo);
1626 seq_printf(m, "\t\t0x%08x: %12lld byte %s",
1627 id, size, placement);
1628
1629 pin_count = READ_ONCE(bo->tbo.pin_count);
1630 if (pin_count)
1631 seq_printf(m, " pin count %d", pin_count);
1632
1633 dma_buf = READ_ONCE(bo->tbo.base.dma_buf);
1634 attachment = READ_ONCE(bo->tbo.base.import_attach);
1635
1636 if (attachment)
1637 seq_printf(m, " imported from ino:%lu", file_inode(dma_buf->file)->i_ino);
1638 else if (dma_buf)
1639 seq_printf(m, " exported as ino:%lu", file_inode(dma_buf->file)->i_ino);
1640
1641 amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED);
1642 amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS);
1643 amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC);
1644 amdgpu_bo_print_flag(m, bo, VRAM_CLEARED);
1645 amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS);
1646 amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID);
1647 amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC);
1648 /* Add the gem obj resv fence dump*/
1649 if (dma_resv_trylock(bo->tbo.base.resv)) {
1650 dma_resv_describe(bo->tbo.base.resv, m);
1651 dma_resv_unlock(bo->tbo.base.resv);
1652 }
1653 seq_puts(m, "\n");
1654
1655 return size;
1656 }
1657 #endif
1658