1 /*
2 * Copyright 2009 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26 /*
27 * Authors:
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
30 * Dave Airlie
31 */
32 #include <linux/list.h>
33 #include <linux/slab.h>
34 #include <linux/dma-buf.h>
35
36 #include <drm/drm_drv.h>
37 #include <drm/amdgpu_drm.h>
38 #include <drm/drm_cache.h>
39 #include "amdgpu.h"
40 #include "amdgpu_trace.h"
41 #include "amdgpu_amdkfd.h"
42 #include "amdgpu_vram_mgr.h"
43
44 /**
45 * DOC: amdgpu_object
46 *
47 * This defines the interfaces to operate on an &amdgpu_bo buffer object which
48 * represents memory used by driver (VRAM, system memory, etc.). The driver
49 * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces
50 * to create/destroy/set buffer object which are then managed by the kernel TTM
51 * memory manager.
52 * The interfaces are also used internally by kernel clients, including gfx,
53 * uvd, etc. for kernel managed allocations used by the GPU.
54 *
55 */
56
amdgpu_bo_destroy(struct ttm_buffer_object * tbo)57 static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo)
58 {
59 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
60
61 amdgpu_bo_kunmap(bo);
62
63 if (bo->tbo.base.import_attach)
64 drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg);
65 drm_gem_object_release(&bo->tbo.base);
66 amdgpu_bo_unref(&bo->parent);
67 kvfree(bo);
68 }
69
amdgpu_bo_user_destroy(struct ttm_buffer_object * tbo)70 static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo)
71 {
72 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
73 struct amdgpu_bo_user *ubo;
74
75 ubo = to_amdgpu_bo_user(bo);
76 kfree(ubo->metadata);
77 amdgpu_bo_destroy(tbo);
78 }
79
80 /**
81 * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo
82 * @bo: buffer object to be checked
83 *
84 * Uses destroy function associated with the object to determine if this is
85 * an &amdgpu_bo.
86 *
87 * Returns:
88 * true if the object belongs to &amdgpu_bo, false if not.
89 */
amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object * bo)90 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
91 {
92 if (bo->destroy == &amdgpu_bo_destroy ||
93 bo->destroy == &amdgpu_bo_user_destroy)
94 return true;
95
96 return false;
97 }
98
99 /**
100 * amdgpu_bo_placement_from_domain - set buffer's placement
101 * @abo: &amdgpu_bo buffer object whose placement is to be set
102 * @domain: requested domain
103 *
104 * Sets buffer's placement according to requested domain and the buffer's
105 * flags.
106 */
amdgpu_bo_placement_from_domain(struct amdgpu_bo * abo,u32 domain)107 void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
108 {
109 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
110 struct ttm_placement *placement = &abo->placement;
111 struct ttm_place *places = abo->placements;
112 u64 flags = abo->flags;
113 u32 c = 0;
114
115 if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
116 unsigned int visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
117 int8_t mem_id = KFD_XCP_MEM_ID(adev, abo->xcp_id);
118
119 if (adev->gmc.mem_partitions && mem_id >= 0) {
120 places[c].fpfn = adev->gmc.mem_partitions[mem_id].range.fpfn;
121 /*
122 * memory partition range lpfn is inclusive start + size - 1
123 * TTM place lpfn is exclusive start + size
124 */
125 places[c].lpfn = adev->gmc.mem_partitions[mem_id].range.lpfn + 1;
126 } else {
127 places[c].fpfn = 0;
128 places[c].lpfn = 0;
129 }
130 places[c].mem_type = TTM_PL_VRAM;
131 places[c].flags = 0;
132
133 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
134 places[c].lpfn = min_not_zero(places[c].lpfn, visible_pfn);
135 else
136 places[c].flags |= TTM_PL_FLAG_TOPDOWN;
137
138 if (abo->tbo.type == ttm_bo_type_kernel &&
139 flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
140 places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;
141
142 c++;
143 }
144
145 if (domain & AMDGPU_GEM_DOMAIN_DOORBELL) {
146 places[c].fpfn = 0;
147 places[c].lpfn = 0;
148 places[c].mem_type = AMDGPU_PL_DOORBELL;
149 places[c].flags = 0;
150 c++;
151 }
152
153 if (domain & AMDGPU_GEM_DOMAIN_GTT) {
154 places[c].fpfn = 0;
155 places[c].lpfn = 0;
156 places[c].mem_type =
157 abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ?
158 AMDGPU_PL_PREEMPT : TTM_PL_TT;
159 places[c].flags = 0;
160 /*
161 * When GTT is just an alternative to VRAM make sure that we
162 * only use it as fallback and still try to fill up VRAM first.
163 */
164 if (domain & abo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)
165 places[c].flags |= TTM_PL_FLAG_FALLBACK;
166 c++;
167 }
168
169 if (domain & AMDGPU_GEM_DOMAIN_CPU) {
170 places[c].fpfn = 0;
171 places[c].lpfn = 0;
172 places[c].mem_type = TTM_PL_SYSTEM;
173 places[c].flags = 0;
174 c++;
175 }
176
177 if (domain & AMDGPU_GEM_DOMAIN_GDS) {
178 places[c].fpfn = 0;
179 places[c].lpfn = 0;
180 places[c].mem_type = AMDGPU_PL_GDS;
181 places[c].flags = 0;
182 c++;
183 }
184
185 if (domain & AMDGPU_GEM_DOMAIN_GWS) {
186 places[c].fpfn = 0;
187 places[c].lpfn = 0;
188 places[c].mem_type = AMDGPU_PL_GWS;
189 places[c].flags = 0;
190 c++;
191 }
192
193 if (domain & AMDGPU_GEM_DOMAIN_OA) {
194 places[c].fpfn = 0;
195 places[c].lpfn = 0;
196 places[c].mem_type = AMDGPU_PL_OA;
197 places[c].flags = 0;
198 c++;
199 }
200
201 if (!c) {
202 places[c].fpfn = 0;
203 places[c].lpfn = 0;
204 places[c].mem_type = TTM_PL_SYSTEM;
205 places[c].flags = 0;
206 c++;
207 }
208
209 BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS);
210
211 placement->num_placement = c;
212 placement->placement = places;
213 }
214
215 /**
216 * amdgpu_bo_create_reserved - create reserved BO for kernel use
217 *
218 * @adev: amdgpu device object
219 * @size: size for the new BO
220 * @align: alignment for the new BO
221 * @domain: where to place it
222 * @bo_ptr: used to initialize BOs in structures
223 * @gpu_addr: GPU addr of the pinned BO
224 * @cpu_addr: optional CPU address mapping
225 *
226 * Allocates and pins a BO for kernel internal use, and returns it still
227 * reserved.
228 *
229 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
230 *
231 * Returns:
232 * 0 on success, negative error code otherwise.
233 */
amdgpu_bo_create_reserved(struct amdgpu_device * adev,unsigned long size,int align,u32 domain,struct amdgpu_bo ** bo_ptr,u64 * gpu_addr,void ** cpu_addr)234 int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
235 unsigned long size, int align,
236 u32 domain, struct amdgpu_bo **bo_ptr,
237 u64 *gpu_addr, void **cpu_addr)
238 {
239 struct amdgpu_bo_param bp;
240 bool free = false;
241 int r;
242
243 if (!size) {
244 amdgpu_bo_unref(bo_ptr);
245 return 0;
246 }
247
248 memset(&bp, 0, sizeof(bp));
249 bp.size = size;
250 bp.byte_align = align;
251 bp.domain = domain;
252 bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
253 : AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
254 bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
255 bp.type = ttm_bo_type_kernel;
256 bp.resv = NULL;
257 bp.bo_ptr_size = sizeof(struct amdgpu_bo);
258
259 if (!*bo_ptr) {
260 r = amdgpu_bo_create(adev, &bp, bo_ptr);
261 if (r) {
262 dev_err(adev->dev, "(%d) failed to allocate kernel bo\n",
263 r);
264 return r;
265 }
266 free = true;
267 }
268
269 r = amdgpu_bo_reserve(*bo_ptr, false);
270 if (r) {
271 dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
272 goto error_free;
273 }
274
275 r = amdgpu_bo_pin(*bo_ptr, domain);
276 if (r) {
277 dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
278 goto error_unreserve;
279 }
280
281 r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo);
282 if (r) {
283 dev_err(adev->dev, "%p bind failed\n", *bo_ptr);
284 goto error_unpin;
285 }
286
287 if (gpu_addr)
288 *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr);
289
290 if (cpu_addr) {
291 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
292 if (r) {
293 dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
294 goto error_unpin;
295 }
296 }
297
298 return 0;
299
300 error_unpin:
301 amdgpu_bo_unpin(*bo_ptr);
302 error_unreserve:
303 amdgpu_bo_unreserve(*bo_ptr);
304
305 error_free:
306 if (free)
307 amdgpu_bo_unref(bo_ptr);
308
309 return r;
310 }
311
312 /**
313 * amdgpu_bo_create_kernel - create BO for kernel use
314 *
315 * @adev: amdgpu device object
316 * @size: size for the new BO
317 * @align: alignment for the new BO
318 * @domain: where to place it
319 * @bo_ptr: used to initialize BOs in structures
320 * @gpu_addr: GPU addr of the pinned BO
321 * @cpu_addr: optional CPU address mapping
322 *
323 * Allocates and pins a BO for kernel internal use.
324 *
325 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
326 *
327 * Returns:
328 * 0 on success, negative error code otherwise.
329 */
amdgpu_bo_create_kernel(struct amdgpu_device * adev,unsigned long size,int align,u32 domain,struct amdgpu_bo ** bo_ptr,u64 * gpu_addr,void ** cpu_addr)330 int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
331 unsigned long size, int align,
332 u32 domain, struct amdgpu_bo **bo_ptr,
333 u64 *gpu_addr, void **cpu_addr)
334 {
335 int r;
336
337 r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr,
338 gpu_addr, cpu_addr);
339
340 if (r)
341 return r;
342
343 if (*bo_ptr)
344 amdgpu_bo_unreserve(*bo_ptr);
345
346 return 0;
347 }
348
349 /**
350 * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location
351 *
352 * @adev: amdgpu device object
353 * @offset: offset of the BO
354 * @size: size of the BO
355 * @bo_ptr: used to initialize BOs in structures
356 * @cpu_addr: optional CPU address mapping
357 *
358 * Creates a kernel BO at a specific offset in VRAM.
359 *
360 * Returns:
361 * 0 on success, negative error code otherwise.
362 */
amdgpu_bo_create_kernel_at(struct amdgpu_device * adev,uint64_t offset,uint64_t size,struct amdgpu_bo ** bo_ptr,void ** cpu_addr)363 int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev,
364 uint64_t offset, uint64_t size,
365 struct amdgpu_bo **bo_ptr, void **cpu_addr)
366 {
367 struct ttm_operation_ctx ctx = { false, false };
368 unsigned int i;
369 int r;
370
371 offset &= PAGE_MASK;
372 size = ALIGN(size, PAGE_SIZE);
373
374 r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE,
375 AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL,
376 cpu_addr);
377 if (r)
378 return r;
379
380 if ((*bo_ptr) == NULL)
381 return 0;
382
383 /*
384 * Remove the original mem node and create a new one at the request
385 * position.
386 */
387 if (cpu_addr)
388 amdgpu_bo_kunmap(*bo_ptr);
389
390 ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource);
391
392 for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) {
393 (*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT;
394 (*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
395 }
396 r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement,
397 &(*bo_ptr)->tbo.resource, &ctx);
398 if (r)
399 goto error;
400
401 if (cpu_addr) {
402 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
403 if (r)
404 goto error;
405 }
406
407 amdgpu_bo_unreserve(*bo_ptr);
408 return 0;
409
410 error:
411 amdgpu_bo_unreserve(*bo_ptr);
412 amdgpu_bo_unref(bo_ptr);
413 return r;
414 }
415
416 /**
417 * amdgpu_bo_free_kernel - free BO for kernel use
418 *
419 * @bo: amdgpu BO to free
420 * @gpu_addr: pointer to where the BO's GPU memory space address was stored
421 * @cpu_addr: pointer to where the BO's CPU memory space address was stored
422 *
423 * unmaps and unpin a BO for kernel internal use.
424 */
amdgpu_bo_free_kernel(struct amdgpu_bo ** bo,u64 * gpu_addr,void ** cpu_addr)425 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
426 void **cpu_addr)
427 {
428 if (*bo == NULL)
429 return;
430
431 WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend);
432
433 if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {
434 if (cpu_addr)
435 amdgpu_bo_kunmap(*bo);
436
437 amdgpu_bo_unpin(*bo);
438 amdgpu_bo_unreserve(*bo);
439 }
440 amdgpu_bo_unref(bo);
441
442 if (gpu_addr)
443 *gpu_addr = 0;
444
445 if (cpu_addr)
446 *cpu_addr = NULL;
447 }
448
449 /* Validate bo size is bit bigger than the request domain */
amdgpu_bo_validate_size(struct amdgpu_device * adev,unsigned long size,u32 domain)450 static bool amdgpu_bo_validate_size(struct amdgpu_device *adev,
451 unsigned long size, u32 domain)
452 {
453 struct ttm_resource_manager *man = NULL;
454
455 /*
456 * If GTT is part of requested domains the check must succeed to
457 * allow fall back to GTT.
458 */
459 if (domain & AMDGPU_GEM_DOMAIN_GTT)
460 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
461 else if (domain & AMDGPU_GEM_DOMAIN_VRAM)
462 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
463 else
464 return true;
465
466 if (!man) {
467 if (domain & AMDGPU_GEM_DOMAIN_GTT)
468 WARN_ON_ONCE("GTT domain requested but GTT mem manager uninitialized");
469 return false;
470 }
471
472 /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU, _DOMAIN_DOORBELL */
473 if (size < man->size)
474 return true;
475
476 DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, man->size);
477 return false;
478 }
479
amdgpu_bo_support_uswc(u64 bo_flags)480 bool amdgpu_bo_support_uswc(u64 bo_flags)
481 {
482
483 #ifdef CONFIG_X86_32
484 /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit
485 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627
486 */
487 return false;
488 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
489 /* Don't try to enable write-combining when it can't work, or things
490 * may be slow
491 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758
492 */
493
494 #ifndef CONFIG_COMPILE_TEST
495 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
496 thanks to write-combining
497 #endif
498
499 if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
500 DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
501 "better performance thanks to write-combining\n");
502 return false;
503 #else
504 /* For architectures that don't support WC memory,
505 * mask out the WC flag from the BO
506 */
507 if (!drm_arch_can_wc_memory())
508 return false;
509
510 return true;
511 #endif
512 }
513
514 /**
515 * amdgpu_bo_create - create an &amdgpu_bo buffer object
516 * @adev: amdgpu device object
517 * @bp: parameters to be used for the buffer object
518 * @bo_ptr: pointer to the buffer object pointer
519 *
520 * Creates an &amdgpu_bo buffer object.
521 *
522 * Returns:
523 * 0 for success or a negative error code on failure.
524 */
amdgpu_bo_create(struct amdgpu_device * adev,struct amdgpu_bo_param * bp,struct amdgpu_bo ** bo_ptr)525 int amdgpu_bo_create(struct amdgpu_device *adev,
526 struct amdgpu_bo_param *bp,
527 struct amdgpu_bo **bo_ptr)
528 {
529 struct ttm_operation_ctx ctx = {
530 .interruptible = (bp->type != ttm_bo_type_kernel),
531 .no_wait_gpu = bp->no_wait_gpu,
532 /* We opt to avoid OOM on system pages allocations */
533 .gfp_retry_mayfail = true,
534 .allow_res_evict = bp->type != ttm_bo_type_kernel,
535 .resv = bp->resv
536 };
537 struct amdgpu_bo *bo;
538 unsigned long page_align, size = bp->size;
539 int r;
540
541 /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */
542 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
543 /* GWS and OA don't need any alignment. */
544 page_align = bp->byte_align;
545 size <<= PAGE_SHIFT;
546
547 } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) {
548 /* Both size and alignment must be a multiple of 4. */
549 page_align = ALIGN(bp->byte_align, 4);
550 size = ALIGN(size, 4) << PAGE_SHIFT;
551 } else {
552 /* Memory should be aligned at least to a page size. */
553 page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT;
554 size = ALIGN(size, PAGE_SIZE);
555 }
556
557 if (!amdgpu_bo_validate_size(adev, size, bp->domain))
558 return -ENOMEM;
559
560 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo));
561
562 *bo_ptr = NULL;
563 bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL);
564 if (bo == NULL)
565 return -ENOMEM;
566 drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size);
567 bo->tbo.base.funcs = &amdgpu_gem_object_funcs;
568 bo->vm_bo = NULL;
569 bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain :
570 bp->domain;
571 bo->allowed_domains = bo->preferred_domains;
572 if (bp->type != ttm_bo_type_kernel &&
573 !(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE) &&
574 bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
575 bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
576
577 bo->flags = bp->flags;
578
579 if (adev->gmc.mem_partitions)
580 /* For GPUs with spatial partitioning, bo->xcp_id=-1 means any partition */
581 bo->xcp_id = bp->xcp_id_plus1 - 1;
582 else
583 /* For GPUs without spatial partitioning */
584 bo->xcp_id = 0;
585
586 if (!amdgpu_bo_support_uswc(bo->flags))
587 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
588
589 bo->tbo.bdev = &adev->mman.bdev;
590 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA |
591 AMDGPU_GEM_DOMAIN_GDS))
592 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
593 else
594 amdgpu_bo_placement_from_domain(bo, bp->domain);
595 if (bp->type == ttm_bo_type_kernel)
596 bo->tbo.priority = 2;
597 else if (!(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE))
598 bo->tbo.priority = 1;
599
600 if (!bp->destroy)
601 bp->destroy = &amdgpu_bo_destroy;
602
603 r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type,
604 &bo->placement, page_align, &ctx, NULL,
605 bp->resv, bp->destroy);
606 if (unlikely(r != 0))
607 return r;
608
609 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
610 amdgpu_res_cpu_visible(adev, bo->tbo.resource))
611 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved,
612 ctx.bytes_moved);
613 else
614 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0);
615
616 if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
617 bo->tbo.resource->mem_type == TTM_PL_VRAM) {
618 struct dma_fence *fence;
619
620 r = amdgpu_ttm_clear_buffer(bo, bo->tbo.base.resv, &fence);
621 if (unlikely(r))
622 goto fail_unreserve;
623
624 dma_resv_add_fence(bo->tbo.base.resv, fence,
625 DMA_RESV_USAGE_KERNEL);
626 dma_fence_put(fence);
627 }
628 if (!bp->resv)
629 amdgpu_bo_unreserve(bo);
630 *bo_ptr = bo;
631
632 trace_amdgpu_bo_create(bo);
633
634 /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */
635 if (bp->type == ttm_bo_type_device)
636 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
637
638 return 0;
639
640 fail_unreserve:
641 if (!bp->resv)
642 dma_resv_unlock(bo->tbo.base.resv);
643 amdgpu_bo_unref(&bo);
644 return r;
645 }
646
647 /**
648 * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object
649 * @adev: amdgpu device object
650 * @bp: parameters to be used for the buffer object
651 * @ubo_ptr: pointer to the buffer object pointer
652 *
653 * Create a BO to be used by user application;
654 *
655 * Returns:
656 * 0 for success or a negative error code on failure.
657 */
658
amdgpu_bo_create_user(struct amdgpu_device * adev,struct amdgpu_bo_param * bp,struct amdgpu_bo_user ** ubo_ptr)659 int amdgpu_bo_create_user(struct amdgpu_device *adev,
660 struct amdgpu_bo_param *bp,
661 struct amdgpu_bo_user **ubo_ptr)
662 {
663 struct amdgpu_bo *bo_ptr;
664 int r;
665
666 bp->bo_ptr_size = sizeof(struct amdgpu_bo_user);
667 bp->destroy = &amdgpu_bo_user_destroy;
668 r = amdgpu_bo_create(adev, bp, &bo_ptr);
669 if (r)
670 return r;
671
672 *ubo_ptr = to_amdgpu_bo_user(bo_ptr);
673 return r;
674 }
675
676 /**
677 * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object
678 * @adev: amdgpu device object
679 * @bp: parameters to be used for the buffer object
680 * @vmbo_ptr: pointer to the buffer object pointer
681 *
682 * Create a BO to be for GPUVM.
683 *
684 * Returns:
685 * 0 for success or a negative error code on failure.
686 */
687
amdgpu_bo_create_vm(struct amdgpu_device * adev,struct amdgpu_bo_param * bp,struct amdgpu_bo_vm ** vmbo_ptr)688 int amdgpu_bo_create_vm(struct amdgpu_device *adev,
689 struct amdgpu_bo_param *bp,
690 struct amdgpu_bo_vm **vmbo_ptr)
691 {
692 struct amdgpu_bo *bo_ptr;
693 int r;
694
695 /* bo_ptr_size will be determined by the caller and it depends on
696 * num of amdgpu_vm_pt entries.
697 */
698 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm));
699 r = amdgpu_bo_create(adev, bp, &bo_ptr);
700 if (r)
701 return r;
702
703 *vmbo_ptr = to_amdgpu_bo_vm(bo_ptr);
704 return r;
705 }
706
707 /**
708 * amdgpu_bo_kmap - map an &amdgpu_bo buffer object
709 * @bo: &amdgpu_bo buffer object to be mapped
710 * @ptr: kernel virtual address to be returned
711 *
712 * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls
713 * amdgpu_bo_kptr() to get the kernel virtual address.
714 *
715 * Returns:
716 * 0 for success or a negative error code on failure.
717 */
amdgpu_bo_kmap(struct amdgpu_bo * bo,void ** ptr)718 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
719 {
720 void *kptr;
721 long r;
722
723 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
724 return -EPERM;
725
726 r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL,
727 false, MAX_SCHEDULE_TIMEOUT);
728 if (r < 0)
729 return r;
730
731 kptr = amdgpu_bo_kptr(bo);
732 if (kptr) {
733 if (ptr)
734 *ptr = kptr;
735 return 0;
736 }
737
738 r = ttm_bo_kmap(&bo->tbo, 0, PFN_UP(bo->tbo.base.size), &bo->kmap);
739 if (r)
740 return r;
741
742 if (ptr)
743 *ptr = amdgpu_bo_kptr(bo);
744
745 return 0;
746 }
747
748 /**
749 * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object
750 * @bo: &amdgpu_bo buffer object
751 *
752 * Calls ttm_kmap_obj_virtual() to get the kernel virtual address
753 *
754 * Returns:
755 * the virtual address of a buffer object area.
756 */
amdgpu_bo_kptr(struct amdgpu_bo * bo)757 void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
758 {
759 bool is_iomem;
760
761 return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
762 }
763
764 /**
765 * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object
766 * @bo: &amdgpu_bo buffer object to be unmapped
767 *
768 * Unmaps a kernel map set up by amdgpu_bo_kmap().
769 */
amdgpu_bo_kunmap(struct amdgpu_bo * bo)770 void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
771 {
772 if (bo->kmap.bo)
773 ttm_bo_kunmap(&bo->kmap);
774 }
775
776 /**
777 * amdgpu_bo_ref - reference an &amdgpu_bo buffer object
778 * @bo: &amdgpu_bo buffer object
779 *
780 * References the contained &ttm_buffer_object.
781 *
782 * Returns:
783 * a refcounted pointer to the &amdgpu_bo buffer object.
784 */
amdgpu_bo_ref(struct amdgpu_bo * bo)785 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
786 {
787 if (bo == NULL)
788 return NULL;
789
790 drm_gem_object_get(&bo->tbo.base);
791 return bo;
792 }
793
794 /**
795 * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object
796 * @bo: &amdgpu_bo buffer object
797 *
798 * Unreferences the contained &ttm_buffer_object and clear the pointer
799 */
amdgpu_bo_unref(struct amdgpu_bo ** bo)800 void amdgpu_bo_unref(struct amdgpu_bo **bo)
801 {
802 if ((*bo) == NULL)
803 return;
804
805 drm_gem_object_put(&(*bo)->tbo.base);
806 *bo = NULL;
807 }
808
809 /**
810 * amdgpu_bo_pin - pin an &amdgpu_bo buffer object
811 * @bo: &amdgpu_bo buffer object to be pinned
812 * @domain: domain to be pinned to
813 *
814 * Pins the buffer object according to requested domain. If the memory is
815 * unbound gart memory, binds the pages into gart table. Adjusts pin_count and
816 * pin_size accordingly.
817 *
818 * Pinning means to lock pages in memory along with keeping them at a fixed
819 * offset. It is required when a buffer can not be moved, for example, when
820 * a display buffer is being scanned out.
821 *
822 * Returns:
823 * 0 for success or a negative error code on failure.
824 */
amdgpu_bo_pin(struct amdgpu_bo * bo,u32 domain)825 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain)
826 {
827 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
828 struct ttm_operation_ctx ctx = { false, false };
829 int r, i;
830
831 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
832 return -EPERM;
833
834 /* Check domain to be pinned to against preferred domains */
835 if (bo->preferred_domains & domain)
836 domain = bo->preferred_domains & domain;
837
838 /* A shared bo cannot be migrated to VRAM */
839 if (bo->tbo.base.import_attach) {
840 if (domain & AMDGPU_GEM_DOMAIN_GTT)
841 domain = AMDGPU_GEM_DOMAIN_GTT;
842 else
843 return -EINVAL;
844 }
845
846 if (bo->tbo.pin_count) {
847 uint32_t mem_type = bo->tbo.resource->mem_type;
848 uint32_t mem_flags = bo->tbo.resource->placement;
849
850 if (!(domain & amdgpu_mem_type_to_domain(mem_type)))
851 return -EINVAL;
852
853 if ((mem_type == TTM_PL_VRAM) &&
854 (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) &&
855 !(mem_flags & TTM_PL_FLAG_CONTIGUOUS))
856 return -EINVAL;
857
858 ttm_bo_pin(&bo->tbo);
859 return 0;
860 }
861
862 /* This assumes only APU display buffers are pinned with (VRAM|GTT).
863 * See function amdgpu_display_supported_domains()
864 */
865 domain = amdgpu_bo_get_preferred_domain(adev, domain);
866
867 if (bo->tbo.base.import_attach)
868 dma_buf_pin(bo->tbo.base.import_attach);
869
870 /* force to pin into visible video ram */
871 if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))
872 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
873 amdgpu_bo_placement_from_domain(bo, domain);
874 for (i = 0; i < bo->placement.num_placement; i++) {
875 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS &&
876 bo->placements[i].mem_type == TTM_PL_VRAM)
877 bo->placements[i].flags |= TTM_PL_FLAG_CONTIGUOUS;
878 }
879
880 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
881 if (unlikely(r)) {
882 dev_err(adev->dev, "%p pin failed\n", bo);
883 goto error;
884 }
885
886 ttm_bo_pin(&bo->tbo);
887
888 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {
889 atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size);
890 atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo),
891 &adev->visible_pin_size);
892 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
893 atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size);
894 }
895
896 error:
897 return r;
898 }
899
900 /**
901 * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object
902 * @bo: &amdgpu_bo buffer object to be unpinned
903 *
904 * Decreases the pin_count, and clears the flags if pin_count reaches 0.
905 * Changes placement and pin size accordingly.
906 *
907 * Returns:
908 * 0 for success or a negative error code on failure.
909 */
amdgpu_bo_unpin(struct amdgpu_bo * bo)910 void amdgpu_bo_unpin(struct amdgpu_bo *bo)
911 {
912 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
913
914 ttm_bo_unpin(&bo->tbo);
915 if (bo->tbo.pin_count)
916 return;
917
918 if (bo->tbo.base.import_attach)
919 dma_buf_unpin(bo->tbo.base.import_attach);
920
921 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {
922 atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size);
923 atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo),
924 &adev->visible_pin_size);
925 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
926 atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size);
927 }
928
929 }
930
931 static const char * const amdgpu_vram_names[] = {
932 "UNKNOWN",
933 "GDDR1",
934 "DDR2",
935 "GDDR3",
936 "GDDR4",
937 "GDDR5",
938 "HBM",
939 "DDR3",
940 "DDR4",
941 "GDDR6",
942 "DDR5",
943 "LPDDR4",
944 "LPDDR5"
945 };
946
947 /**
948 * amdgpu_bo_init - initialize memory manager
949 * @adev: amdgpu device object
950 *
951 * Calls amdgpu_ttm_init() to initialize amdgpu memory manager.
952 *
953 * Returns:
954 * 0 for success or a negative error code on failure.
955 */
amdgpu_bo_init(struct amdgpu_device * adev)956 int amdgpu_bo_init(struct amdgpu_device *adev)
957 {
958 /* On A+A platform, VRAM can be mapped as WB */
959 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
960 /* reserve PAT memory space to WC for VRAM */
961 int r = arch_io_reserve_memtype_wc(adev->gmc.aper_base,
962 adev->gmc.aper_size);
963
964 if (r) {
965 DRM_ERROR("Unable to set WC memtype for the aperture base\n");
966 return r;
967 }
968
969 /* Add an MTRR for the VRAM */
970 adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base,
971 adev->gmc.aper_size);
972 }
973
974 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
975 adev->gmc.mc_vram_size >> 20,
976 (unsigned long long)adev->gmc.aper_size >> 20);
977 DRM_INFO("RAM width %dbits %s\n",
978 adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]);
979 return amdgpu_ttm_init(adev);
980 }
981
982 /**
983 * amdgpu_bo_fini - tear down memory manager
984 * @adev: amdgpu device object
985 *
986 * Reverses amdgpu_bo_init() to tear down memory manager.
987 */
amdgpu_bo_fini(struct amdgpu_device * adev)988 void amdgpu_bo_fini(struct amdgpu_device *adev)
989 {
990 int idx;
991
992 amdgpu_ttm_fini(adev);
993
994 if (drm_dev_enter(adev_to_drm(adev), &idx)) {
995 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
996 arch_phys_wc_del(adev->gmc.vram_mtrr);
997 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
998 }
999 drm_dev_exit(idx);
1000 }
1001 }
1002
1003 /**
1004 * amdgpu_bo_set_tiling_flags - set tiling flags
1005 * @bo: &amdgpu_bo buffer object
1006 * @tiling_flags: new flags
1007 *
1008 * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or
1009 * kernel driver to set the tiling flags on a buffer.
1010 *
1011 * Returns:
1012 * 0 for success or a negative error code on failure.
1013 */
amdgpu_bo_set_tiling_flags(struct amdgpu_bo * bo,u64 tiling_flags)1014 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
1015 {
1016 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1017 struct amdgpu_bo_user *ubo;
1018
1019 BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1020 if (adev->family <= AMDGPU_FAMILY_CZ &&
1021 AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
1022 return -EINVAL;
1023
1024 ubo = to_amdgpu_bo_user(bo);
1025 ubo->tiling_flags = tiling_flags;
1026 return 0;
1027 }
1028
1029 /**
1030 * amdgpu_bo_get_tiling_flags - get tiling flags
1031 * @bo: &amdgpu_bo buffer object
1032 * @tiling_flags: returned flags
1033 *
1034 * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to
1035 * set the tiling flags on a buffer.
1036 */
amdgpu_bo_get_tiling_flags(struct amdgpu_bo * bo,u64 * tiling_flags)1037 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
1038 {
1039 struct amdgpu_bo_user *ubo;
1040
1041 BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1042 dma_resv_assert_held(bo->tbo.base.resv);
1043 ubo = to_amdgpu_bo_user(bo);
1044
1045 if (tiling_flags)
1046 *tiling_flags = ubo->tiling_flags;
1047 }
1048
1049 /**
1050 * amdgpu_bo_set_metadata - set metadata
1051 * @bo: &amdgpu_bo buffer object
1052 * @metadata: new metadata
1053 * @metadata_size: size of the new metadata
1054 * @flags: flags of the new metadata
1055 *
1056 * Sets buffer object's metadata, its size and flags.
1057 * Used via GEM ioctl.
1058 *
1059 * Returns:
1060 * 0 for success or a negative error code on failure.
1061 */
amdgpu_bo_set_metadata(struct amdgpu_bo * bo,void * metadata,u32 metadata_size,uint64_t flags)1062 int amdgpu_bo_set_metadata(struct amdgpu_bo *bo, void *metadata,
1063 u32 metadata_size, uint64_t flags)
1064 {
1065 struct amdgpu_bo_user *ubo;
1066 void *buffer;
1067
1068 BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1069 ubo = to_amdgpu_bo_user(bo);
1070 if (!metadata_size) {
1071 if (ubo->metadata_size) {
1072 kfree(ubo->metadata);
1073 ubo->metadata = NULL;
1074 ubo->metadata_size = 0;
1075 }
1076 return 0;
1077 }
1078
1079 if (metadata == NULL)
1080 return -EINVAL;
1081
1082 buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
1083 if (buffer == NULL)
1084 return -ENOMEM;
1085
1086 kfree(ubo->metadata);
1087 ubo->metadata_flags = flags;
1088 ubo->metadata = buffer;
1089 ubo->metadata_size = metadata_size;
1090
1091 return 0;
1092 }
1093
1094 /**
1095 * amdgpu_bo_get_metadata - get metadata
1096 * @bo: &amdgpu_bo buffer object
1097 * @buffer: returned metadata
1098 * @buffer_size: size of the buffer
1099 * @metadata_size: size of the returned metadata
1100 * @flags: flags of the returned metadata
1101 *
1102 * Gets buffer object's metadata, its size and flags. buffer_size shall not be
1103 * less than metadata_size.
1104 * Used via GEM ioctl.
1105 *
1106 * Returns:
1107 * 0 for success or a negative error code on failure.
1108 */
amdgpu_bo_get_metadata(struct amdgpu_bo * bo,void * buffer,size_t buffer_size,uint32_t * metadata_size,uint64_t * flags)1109 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
1110 size_t buffer_size, uint32_t *metadata_size,
1111 uint64_t *flags)
1112 {
1113 struct amdgpu_bo_user *ubo;
1114
1115 if (!buffer && !metadata_size)
1116 return -EINVAL;
1117
1118 BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1119 ubo = to_amdgpu_bo_user(bo);
1120 if (metadata_size)
1121 *metadata_size = ubo->metadata_size;
1122
1123 if (buffer) {
1124 if (buffer_size < ubo->metadata_size)
1125 return -EINVAL;
1126
1127 if (ubo->metadata_size)
1128 memcpy(buffer, ubo->metadata, ubo->metadata_size);
1129 }
1130
1131 if (flags)
1132 *flags = ubo->metadata_flags;
1133
1134 return 0;
1135 }
1136
1137 /**
1138 * amdgpu_bo_move_notify - notification about a memory move
1139 * @bo: pointer to a buffer object
1140 * @evict: if this move is evicting the buffer from the graphics address space
1141 * @new_mem: new resource for backing the BO
1142 *
1143 * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs
1144 * bookkeeping.
1145 * TTM driver callback which is called when ttm moves a buffer.
1146 */
amdgpu_bo_move_notify(struct ttm_buffer_object * bo,bool evict,struct ttm_resource * new_mem)1147 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
1148 bool evict,
1149 struct ttm_resource *new_mem)
1150 {
1151 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1152 struct ttm_resource *old_mem = bo->resource;
1153 struct amdgpu_bo *abo;
1154
1155 if (!amdgpu_bo_is_amdgpu_bo(bo))
1156 return;
1157
1158 abo = ttm_to_amdgpu_bo(bo);
1159 amdgpu_vm_bo_invalidate(adev, abo, evict);
1160
1161 amdgpu_bo_kunmap(abo);
1162
1163 if (abo->tbo.base.dma_buf && !abo->tbo.base.import_attach &&
1164 old_mem && old_mem->mem_type != TTM_PL_SYSTEM)
1165 dma_buf_move_notify(abo->tbo.base.dma_buf);
1166
1167 /* move_notify is called before move happens */
1168 trace_amdgpu_bo_move(abo, new_mem ? new_mem->mem_type : -1,
1169 old_mem ? old_mem->mem_type : -1);
1170 }
1171
amdgpu_bo_get_memory(struct amdgpu_bo * bo,struct amdgpu_mem_stats * stats)1172 void amdgpu_bo_get_memory(struct amdgpu_bo *bo,
1173 struct amdgpu_mem_stats *stats)
1174 {
1175 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1176 struct ttm_resource *res = bo->tbo.resource;
1177 uint64_t size = amdgpu_bo_size(bo);
1178 struct drm_gem_object *obj;
1179 bool shared;
1180
1181 /* Abort if the BO doesn't currently have a backing store */
1182 if (!res)
1183 return;
1184
1185 obj = &bo->tbo.base;
1186 shared = drm_gem_object_is_shared_for_memory_stats(obj);
1187
1188 switch (res->mem_type) {
1189 case TTM_PL_VRAM:
1190 stats->vram += size;
1191 if (amdgpu_res_cpu_visible(adev, res))
1192 stats->visible_vram += size;
1193 if (shared)
1194 stats->vram_shared += size;
1195 break;
1196 case TTM_PL_TT:
1197 stats->gtt += size;
1198 if (shared)
1199 stats->gtt_shared += size;
1200 break;
1201 case TTM_PL_SYSTEM:
1202 default:
1203 stats->cpu += size;
1204 if (shared)
1205 stats->cpu_shared += size;
1206 break;
1207 }
1208
1209 if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) {
1210 stats->requested_vram += size;
1211 if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
1212 stats->requested_visible_vram += size;
1213
1214 if (res->mem_type != TTM_PL_VRAM) {
1215 stats->evicted_vram += size;
1216 if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
1217 stats->evicted_visible_vram += size;
1218 }
1219 } else if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_GTT) {
1220 stats->requested_gtt += size;
1221 }
1222 }
1223
1224 /**
1225 * amdgpu_bo_release_notify - notification about a BO being released
1226 * @bo: pointer to a buffer object
1227 *
1228 * Wipes VRAM buffers whose contents should not be leaked before the
1229 * memory is released.
1230 */
amdgpu_bo_release_notify(struct ttm_buffer_object * bo)1231 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)
1232 {
1233 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1234 struct dma_fence *fence = NULL;
1235 struct amdgpu_bo *abo;
1236 int r;
1237
1238 if (!amdgpu_bo_is_amdgpu_bo(bo))
1239 return;
1240
1241 abo = ttm_to_amdgpu_bo(bo);
1242
1243 WARN_ON(abo->vm_bo);
1244
1245 if (abo->kfd_bo)
1246 amdgpu_amdkfd_release_notify(abo);
1247
1248 /* We only remove the fence if the resv has individualized. */
1249 WARN_ON_ONCE(bo->type == ttm_bo_type_kernel
1250 && bo->base.resv != &bo->base._resv);
1251 if (bo->base.resv == &bo->base._resv)
1252 amdgpu_amdkfd_remove_fence_on_pt_pd_bos(abo);
1253
1254 if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM ||
1255 !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) ||
1256 adev->in_suspend || drm_dev_is_unplugged(adev_to_drm(adev)))
1257 return;
1258
1259 if (WARN_ON_ONCE(!dma_resv_trylock(bo->base.resv)))
1260 return;
1261
1262 r = amdgpu_fill_buffer(abo, 0, bo->base.resv, &fence, true);
1263 if (!WARN_ON(r)) {
1264 amdgpu_vram_mgr_set_cleared(bo->resource);
1265 amdgpu_bo_fence(abo, fence, false);
1266 dma_fence_put(fence);
1267 }
1268
1269 dma_resv_unlock(bo->base.resv);
1270 }
1271
1272 /**
1273 * amdgpu_bo_fault_reserve_notify - notification about a memory fault
1274 * @bo: pointer to a buffer object
1275 *
1276 * Notifies the driver we are taking a fault on this BO and have reserved it,
1277 * also performs bookkeeping.
1278 * TTM driver callback for dealing with vm faults.
1279 *
1280 * Returns:
1281 * 0 for success or a negative error code on failure.
1282 */
amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object * bo)1283 vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
1284 {
1285 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1286 struct ttm_operation_ctx ctx = { false, false };
1287 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1288 int r;
1289
1290 /* Remember that this BO was accessed by the CPU */
1291 abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1292
1293 if (amdgpu_res_cpu_visible(adev, bo->resource))
1294 return 0;
1295
1296 /* Can't move a pinned BO to visible VRAM */
1297 if (abo->tbo.pin_count > 0)
1298 return VM_FAULT_SIGBUS;
1299
1300 /* hurrah the memory is not visible ! */
1301 atomic64_inc(&adev->num_vram_cpu_page_faults);
1302 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
1303 AMDGPU_GEM_DOMAIN_GTT);
1304
1305 /* Avoid costly evictions; only set GTT as a busy placement */
1306 abo->placements[0].flags |= TTM_PL_FLAG_DESIRED;
1307
1308 r = ttm_bo_validate(bo, &abo->placement, &ctx);
1309 if (unlikely(r == -EBUSY || r == -ERESTARTSYS))
1310 return VM_FAULT_NOPAGE;
1311 else if (unlikely(r))
1312 return VM_FAULT_SIGBUS;
1313
1314 /* this should never happen */
1315 if (bo->resource->mem_type == TTM_PL_VRAM &&
1316 !amdgpu_res_cpu_visible(adev, bo->resource))
1317 return VM_FAULT_SIGBUS;
1318
1319 ttm_bo_move_to_lru_tail_unlocked(bo);
1320 return 0;
1321 }
1322
1323 /**
1324 * amdgpu_bo_fence - add fence to buffer object
1325 *
1326 * @bo: buffer object in question
1327 * @fence: fence to add
1328 * @shared: true if fence should be added shared
1329 *
1330 */
amdgpu_bo_fence(struct amdgpu_bo * bo,struct dma_fence * fence,bool shared)1331 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
1332 bool shared)
1333 {
1334 struct dma_resv *resv = bo->tbo.base.resv;
1335 int r;
1336
1337 r = dma_resv_reserve_fences(resv, 1);
1338 if (r) {
1339 /* As last resort on OOM we block for the fence */
1340 dma_fence_wait(fence, false);
1341 return;
1342 }
1343
1344 dma_resv_add_fence(resv, fence, shared ? DMA_RESV_USAGE_READ :
1345 DMA_RESV_USAGE_WRITE);
1346 }
1347
1348 /**
1349 * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences
1350 *
1351 * @adev: amdgpu device pointer
1352 * @resv: reservation object to sync to
1353 * @sync_mode: synchronization mode
1354 * @owner: fence owner
1355 * @intr: Whether the wait is interruptible
1356 *
1357 * Extract the fences from the reservation object and waits for them to finish.
1358 *
1359 * Returns:
1360 * 0 on success, errno otherwise.
1361 */
amdgpu_bo_sync_wait_resv(struct amdgpu_device * adev,struct dma_resv * resv,enum amdgpu_sync_mode sync_mode,void * owner,bool intr)1362 int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv,
1363 enum amdgpu_sync_mode sync_mode, void *owner,
1364 bool intr)
1365 {
1366 struct amdgpu_sync sync;
1367 int r;
1368
1369 amdgpu_sync_create(&sync);
1370 amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner);
1371 r = amdgpu_sync_wait(&sync, intr);
1372 amdgpu_sync_free(&sync);
1373 return r;
1374 }
1375
1376 /**
1377 * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv
1378 * @bo: buffer object to wait for
1379 * @owner: fence owner
1380 * @intr: Whether the wait is interruptible
1381 *
1382 * Wrapper to wait for fences in a BO.
1383 * Returns:
1384 * 0 on success, errno otherwise.
1385 */
amdgpu_bo_sync_wait(struct amdgpu_bo * bo,void * owner,bool intr)1386 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr)
1387 {
1388 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1389
1390 return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv,
1391 AMDGPU_SYNC_NE_OWNER, owner, intr);
1392 }
1393
1394 /**
1395 * amdgpu_bo_gpu_offset - return GPU offset of bo
1396 * @bo: amdgpu object for which we query the offset
1397 *
1398 * Note: object should either be pinned or reserved when calling this
1399 * function, it might be useful to add check for this for debugging.
1400 *
1401 * Returns:
1402 * current GPU offset of the object.
1403 */
amdgpu_bo_gpu_offset(struct amdgpu_bo * bo)1404 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
1405 {
1406 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM);
1407 WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) &&
1408 !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel);
1409 WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET);
1410 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM &&
1411 !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));
1412
1413 return amdgpu_bo_gpu_offset_no_check(bo);
1414 }
1415
1416 /**
1417 * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo
1418 * @bo: amdgpu object for which we query the offset
1419 *
1420 * Returns:
1421 * current GPU offset of the object without raising warnings.
1422 */
amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo * bo)1423 u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo)
1424 {
1425 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1426 uint64_t offset = AMDGPU_BO_INVALID_OFFSET;
1427
1428 if (bo->tbo.resource->mem_type == TTM_PL_TT)
1429 offset = amdgpu_gmc_agp_addr(&bo->tbo);
1430
1431 if (offset == AMDGPU_BO_INVALID_OFFSET)
1432 offset = (bo->tbo.resource->start << PAGE_SHIFT) +
1433 amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type);
1434
1435 return amdgpu_gmc_sign_extend(offset);
1436 }
1437
1438 /**
1439 * amdgpu_bo_get_preferred_domain - get preferred domain
1440 * @adev: amdgpu device object
1441 * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>`
1442 *
1443 * Returns:
1444 * Which of the allowed domains is preferred for allocating the BO.
1445 */
amdgpu_bo_get_preferred_domain(struct amdgpu_device * adev,uint32_t domain)1446 uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,
1447 uint32_t domain)
1448 {
1449 if ((domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) &&
1450 ((adev->asic_type == CHIP_CARRIZO) || (adev->asic_type == CHIP_STONEY))) {
1451 domain = AMDGPU_GEM_DOMAIN_VRAM;
1452 if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD)
1453 domain = AMDGPU_GEM_DOMAIN_GTT;
1454 }
1455 return domain;
1456 }
1457
1458 #if defined(CONFIG_DEBUG_FS)
1459 #define amdgpu_bo_print_flag(m, bo, flag) \
1460 do { \
1461 if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \
1462 seq_printf((m), " " #flag); \
1463 } \
1464 } while (0)
1465
1466 /**
1467 * amdgpu_bo_print_info - print BO info in debugfs file
1468 *
1469 * @id: Index or Id of the BO
1470 * @bo: Requested BO for printing info
1471 * @m: debugfs file
1472 *
1473 * Print BO information in debugfs file
1474 *
1475 * Returns:
1476 * Size of the BO in bytes.
1477 */
amdgpu_bo_print_info(int id,struct amdgpu_bo * bo,struct seq_file * m)1478 u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m)
1479 {
1480 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1481 struct dma_buf_attachment *attachment;
1482 struct dma_buf *dma_buf;
1483 const char *placement;
1484 unsigned int pin_count;
1485 u64 size;
1486
1487 if (dma_resv_trylock(bo->tbo.base.resv)) {
1488 if (!bo->tbo.resource) {
1489 placement = "NONE";
1490 } else {
1491 switch (bo->tbo.resource->mem_type) {
1492 case TTM_PL_VRAM:
1493 if (amdgpu_res_cpu_visible(adev, bo->tbo.resource))
1494 placement = "VRAM VISIBLE";
1495 else
1496 placement = "VRAM";
1497 break;
1498 case TTM_PL_TT:
1499 placement = "GTT";
1500 break;
1501 case AMDGPU_PL_GDS:
1502 placement = "GDS";
1503 break;
1504 case AMDGPU_PL_GWS:
1505 placement = "GWS";
1506 break;
1507 case AMDGPU_PL_OA:
1508 placement = "OA";
1509 break;
1510 case AMDGPU_PL_PREEMPT:
1511 placement = "PREEMPTIBLE";
1512 break;
1513 case AMDGPU_PL_DOORBELL:
1514 placement = "DOORBELL";
1515 break;
1516 case TTM_PL_SYSTEM:
1517 default:
1518 placement = "CPU";
1519 break;
1520 }
1521 }
1522 dma_resv_unlock(bo->tbo.base.resv);
1523 } else {
1524 placement = "UNKNOWN";
1525 }
1526
1527 size = amdgpu_bo_size(bo);
1528 seq_printf(m, "\t\t0x%08x: %12lld byte %s",
1529 id, size, placement);
1530
1531 pin_count = READ_ONCE(bo->tbo.pin_count);
1532 if (pin_count)
1533 seq_printf(m, " pin count %d", pin_count);
1534
1535 dma_buf = READ_ONCE(bo->tbo.base.dma_buf);
1536 attachment = READ_ONCE(bo->tbo.base.import_attach);
1537
1538 if (attachment)
1539 seq_printf(m, " imported from ino:%lu", file_inode(dma_buf->file)->i_ino);
1540 else if (dma_buf)
1541 seq_printf(m, " exported as ino:%lu", file_inode(dma_buf->file)->i_ino);
1542
1543 amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED);
1544 amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS);
1545 amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC);
1546 amdgpu_bo_print_flag(m, bo, VRAM_CLEARED);
1547 amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS);
1548 amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID);
1549 amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC);
1550
1551 seq_puts(m, "\n");
1552
1553 return size;
1554 }
1555 #endif
1556