1 /*
2 * Copyright 2009 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26 /*
27 * Authors:
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
30 * Dave Airlie
31 */
32
33 #include <linux/dma-mapping.h>
34 #include <linux/iommu.h>
35 #include <linux/pagemap.h>
36 #include <linux/sched/task.h>
37 #include <linux/sched/mm.h>
38 #include <linux/seq_file.h>
39 #include <linux/slab.h>
40 #include <linux/swap.h>
41 #include <linux/dma-buf.h>
42 #include <linux/sizes.h>
43 #include <linux/module.h>
44
45 #include <drm/drm_drv.h>
46 #include <drm/ttm/ttm_bo.h>
47 #include <drm/ttm/ttm_placement.h>
48 #include <drm/ttm/ttm_range_manager.h>
49 #include <drm/ttm/ttm_tt.h>
50
51 #include <drm/amdgpu_drm.h>
52
53 #include "amdgpu.h"
54 #include "amdgpu_object.h"
55 #include "amdgpu_trace.h"
56 #include "amdgpu_amdkfd.h"
57 #include "amdgpu_sdma.h"
58 #include "amdgpu_ras.h"
59 #include "amdgpu_hmm.h"
60 #include "amdgpu_atomfirmware.h"
61 #include "amdgpu_res_cursor.h"
62 #include "bif/bif_4_1_d.h"
63
64 MODULE_IMPORT_NS("DMA_BUF");
65
66 #define AMDGPU_TTM_VRAM_MAX_DW_READ ((size_t)128)
67
68 static int amdgpu_ttm_backend_bind(struct ttm_device *bdev,
69 struct ttm_tt *ttm,
70 struct ttm_resource *bo_mem);
71 static void amdgpu_ttm_backend_unbind(struct ttm_device *bdev,
72 struct ttm_tt *ttm);
73
amdgpu_ttm_init_on_chip(struct amdgpu_device * adev,unsigned int type,uint64_t size_in_page)74 static int amdgpu_ttm_init_on_chip(struct amdgpu_device *adev,
75 unsigned int type,
76 uint64_t size_in_page)
77 {
78 return ttm_range_man_init(&adev->mman.bdev, type,
79 false, size_in_page);
80 }
81
82 /**
83 * amdgpu_evict_flags - Compute placement flags
84 *
85 * @bo: The buffer object to evict
86 * @placement: Possible destination(s) for evicted BO
87 *
88 * Fill in placement data when ttm_bo_evict() is called
89 */
amdgpu_evict_flags(struct ttm_buffer_object * bo,struct ttm_placement * placement)90 static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
91 struct ttm_placement *placement)
92 {
93 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
94 struct amdgpu_bo *abo;
95 static const struct ttm_place placements = {
96 .fpfn = 0,
97 .lpfn = 0,
98 .mem_type = TTM_PL_SYSTEM,
99 .flags = 0
100 };
101
102 /* Don't handle scatter gather BOs */
103 if (bo->type == ttm_bo_type_sg) {
104 placement->num_placement = 0;
105 return;
106 }
107
108 /* Object isn't an AMDGPU object so ignore */
109 if (!amdgpu_bo_is_amdgpu_bo(bo)) {
110 placement->placement = &placements;
111 placement->num_placement = 1;
112 return;
113 }
114
115 abo = ttm_to_amdgpu_bo(bo);
116 if (abo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) {
117 placement->num_placement = 0;
118 return;
119 }
120
121 switch (bo->resource->mem_type) {
122 case AMDGPU_PL_GDS:
123 case AMDGPU_PL_GWS:
124 case AMDGPU_PL_OA:
125 case AMDGPU_PL_DOORBELL:
126 case AMDGPU_PL_MMIO_REMAP:
127 placement->num_placement = 0;
128 return;
129
130 case TTM_PL_VRAM:
131 if (!adev->mman.buffer_funcs_enabled) {
132 /* Move to system memory */
133 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
134
135 } else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
136 !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
137 amdgpu_res_cpu_visible(adev, bo->resource)) {
138
139 /* Try evicting to the CPU inaccessible part of VRAM
140 * first, but only set GTT as busy placement, so this
141 * BO will be evicted to GTT rather than causing other
142 * BOs to be evicted from VRAM
143 */
144 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
145 AMDGPU_GEM_DOMAIN_GTT |
146 AMDGPU_GEM_DOMAIN_CPU);
147 abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
148 abo->placements[0].lpfn = 0;
149 abo->placements[0].flags |= TTM_PL_FLAG_DESIRED;
150 } else {
151 /* Move to GTT memory */
152 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT |
153 AMDGPU_GEM_DOMAIN_CPU);
154 }
155 break;
156 case TTM_PL_TT:
157 case AMDGPU_PL_PREEMPT:
158 default:
159 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
160 break;
161 }
162 *placement = abo->placement;
163 }
164
165 static struct dma_fence *
amdgpu_ttm_job_submit(struct amdgpu_device * adev,struct amdgpu_job * job,u32 num_dw)166 amdgpu_ttm_job_submit(struct amdgpu_device *adev, struct amdgpu_job *job, u32 num_dw)
167 {
168 struct amdgpu_ring *ring;
169
170 ring = adev->mman.buffer_funcs_ring;
171 amdgpu_ring_pad_ib(ring, &job->ibs[0]);
172 WARN_ON(job->ibs[0].length_dw > num_dw);
173
174 return amdgpu_job_submit(job);
175 }
176
177 /**
178 * amdgpu_ttm_map_buffer - Map memory into the GART windows
179 * @entity: entity to run the window setup job
180 * @bo: buffer object to map
181 * @mem: memory object to map
182 * @mm_cur: range to map
183 * @window: which GART window to use
184 * @tmz: if we should setup a TMZ enabled mapping
185 * @size: in number of bytes to map, out number of bytes mapped
186 * @addr: resulting address inside the MC address space
187 *
188 * Setup one of the GART windows to access a specific piece of memory or return
189 * the physical address for local memory.
190 */
amdgpu_ttm_map_buffer(struct amdgpu_ttm_buffer_entity * entity,struct ttm_buffer_object * bo,struct ttm_resource * mem,struct amdgpu_res_cursor * mm_cur,unsigned int window,bool tmz,uint64_t * size,uint64_t * addr)191 static int amdgpu_ttm_map_buffer(struct amdgpu_ttm_buffer_entity *entity,
192 struct ttm_buffer_object *bo,
193 struct ttm_resource *mem,
194 struct amdgpu_res_cursor *mm_cur,
195 unsigned int window,
196 bool tmz, uint64_t *size, uint64_t *addr)
197 {
198 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
199 unsigned int offset, num_pages, num_dw, num_bytes;
200 uint64_t src_addr, dst_addr;
201 struct amdgpu_job *job;
202 void *cpu_addr;
203 uint64_t flags;
204 int r;
205
206 BUG_ON(adev->mman.buffer_funcs->copy_max_bytes <
207 AMDGPU_GTT_MAX_TRANSFER_SIZE * 8);
208
209 if (WARN_ON(mem->mem_type == AMDGPU_PL_PREEMPT))
210 return -EINVAL;
211
212 /* Map only what can't be accessed directly */
213 if (!tmz && mem->start != AMDGPU_BO_INVALID_OFFSET) {
214 *addr = amdgpu_ttm_domain_start(adev, mem->mem_type) +
215 mm_cur->start;
216 return 0;
217 }
218
219
220 /*
221 * If start begins at an offset inside the page, then adjust the size
222 * and addr accordingly
223 */
224 offset = mm_cur->start & ~PAGE_MASK;
225
226 num_pages = PFN_UP(*size + offset);
227 num_pages = min_t(uint32_t, num_pages, AMDGPU_GTT_MAX_TRANSFER_SIZE);
228
229 *size = min(*size, (uint64_t)num_pages * PAGE_SIZE - offset);
230
231 *addr = adev->gmc.gart_start;
232 *addr += (u64)window * AMDGPU_GTT_MAX_TRANSFER_SIZE *
233 AMDGPU_GPU_PAGE_SIZE;
234 *addr += offset;
235
236 num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
237 num_bytes = num_pages * 8 * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
238
239 r = amdgpu_job_alloc_with_ib(adev, &entity->base,
240 AMDGPU_FENCE_OWNER_UNDEFINED,
241 num_dw * 4 + num_bytes,
242 AMDGPU_IB_POOL_DELAYED, &job,
243 AMDGPU_KERNEL_JOB_ID_TTM_MAP_BUFFER);
244 if (r)
245 return r;
246
247 src_addr = num_dw * 4;
248 src_addr += job->ibs[0].gpu_addr;
249
250 dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
251 dst_addr += window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 8;
252 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
253 dst_addr, num_bytes, 0);
254
255 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, mem);
256 if (tmz)
257 flags |= AMDGPU_PTE_TMZ;
258
259 cpu_addr = &job->ibs[0].ptr[num_dw];
260
261 if (mem->mem_type == TTM_PL_TT) {
262 dma_addr_t *dma_addr;
263
264 dma_addr = &bo->ttm->dma_address[mm_cur->start >> PAGE_SHIFT];
265 amdgpu_gart_map(adev, 0, num_pages, dma_addr, flags, cpu_addr);
266 } else {
267 u64 pa = mm_cur->start + adev->vm_manager.vram_base_offset;
268
269 amdgpu_gart_map_vram_range(adev, pa, 0, num_pages, flags, cpu_addr);
270 }
271
272 dma_fence_put(amdgpu_ttm_job_submit(adev, job, num_dw));
273 return 0;
274 }
275
276 /**
277 * amdgpu_ttm_copy_mem_to_mem - Helper function for copy
278 * @adev: amdgpu device
279 * @entity: entity to run the jobs
280 * @src: buffer/address where to read from
281 * @dst: buffer/address where to write to
282 * @size: number of bytes to copy
283 * @tmz: if a secure copy should be used
284 * @resv: resv object to sync to
285 * @f: Returns the last fence if multiple jobs are submitted.
286 *
287 * The function copies @size bytes from {src->mem + src->offset} to
288 * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a
289 * move and different for a BO to BO copy.
290 *
291 */
292 __attribute__((nonnull))
amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device * adev,struct amdgpu_ttm_buffer_entity * entity,const struct amdgpu_copy_mem * src,const struct amdgpu_copy_mem * dst,uint64_t size,bool tmz,struct dma_resv * resv,struct dma_fence ** f)293 static int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
294 struct amdgpu_ttm_buffer_entity *entity,
295 const struct amdgpu_copy_mem *src,
296 const struct amdgpu_copy_mem *dst,
297 uint64_t size, bool tmz,
298 struct dma_resv *resv,
299 struct dma_fence **f)
300 {
301 struct amdgpu_res_cursor src_mm, dst_mm;
302 struct dma_fence *fence = NULL;
303 int r = 0;
304 uint32_t copy_flags = 0;
305 struct amdgpu_bo *abo_src, *abo_dst;
306
307 if (!adev->mman.buffer_funcs_enabled) {
308 dev_err(adev->dev,
309 "Trying to move memory with ring turned off.\n");
310 return -EINVAL;
311 }
312
313 amdgpu_res_first(src->mem, src->offset, size, &src_mm);
314 amdgpu_res_first(dst->mem, dst->offset, size, &dst_mm);
315
316 mutex_lock(&adev->mman.gtt_window_lock);
317 while (src_mm.remaining) {
318 uint64_t from, to, cur_size, tiling_flags;
319 uint32_t num_type, data_format, max_com, write_compress_disable;
320 struct dma_fence *next;
321
322 /* Never copy more than 256MiB at once to avoid a timeout */
323 cur_size = min3(src_mm.size, dst_mm.size, 256ULL << 20);
324
325 /* Map src to window 0 and dst to window 1. */
326 r = amdgpu_ttm_map_buffer(entity, src->bo, src->mem, &src_mm,
327 0, tmz, &cur_size, &from);
328 if (r)
329 goto error;
330
331 r = amdgpu_ttm_map_buffer(entity, dst->bo, dst->mem, &dst_mm,
332 1, tmz, &cur_size, &to);
333 if (r)
334 goto error;
335
336 abo_src = ttm_to_amdgpu_bo(src->bo);
337 abo_dst = ttm_to_amdgpu_bo(dst->bo);
338 if (tmz)
339 copy_flags |= AMDGPU_COPY_FLAGS_TMZ;
340 if ((abo_src->flags & AMDGPU_GEM_CREATE_GFX12_DCC) &&
341 (abo_src->tbo.resource->mem_type == TTM_PL_VRAM))
342 copy_flags |= AMDGPU_COPY_FLAGS_READ_DECOMPRESSED;
343 if ((abo_dst->flags & AMDGPU_GEM_CREATE_GFX12_DCC) &&
344 (dst->mem->mem_type == TTM_PL_VRAM)) {
345 copy_flags |= AMDGPU_COPY_FLAGS_WRITE_COMPRESSED;
346 amdgpu_bo_get_tiling_flags(abo_dst, &tiling_flags);
347 max_com = AMDGPU_TILING_GET(tiling_flags, GFX12_DCC_MAX_COMPRESSED_BLOCK);
348 num_type = AMDGPU_TILING_GET(tiling_flags, GFX12_DCC_NUMBER_TYPE);
349 data_format = AMDGPU_TILING_GET(tiling_flags, GFX12_DCC_DATA_FORMAT);
350 write_compress_disable =
351 AMDGPU_TILING_GET(tiling_flags, GFX12_DCC_WRITE_COMPRESS_DISABLE);
352 copy_flags |= (AMDGPU_COPY_FLAGS_SET(MAX_COMPRESSED, max_com) |
353 AMDGPU_COPY_FLAGS_SET(NUMBER_TYPE, num_type) |
354 AMDGPU_COPY_FLAGS_SET(DATA_FORMAT, data_format) |
355 AMDGPU_COPY_FLAGS_SET(WRITE_COMPRESS_DISABLE,
356 write_compress_disable));
357 }
358
359 r = amdgpu_copy_buffer(adev, entity, from, to, cur_size, resv,
360 &next, true, copy_flags);
361 if (r)
362 goto error;
363
364 dma_fence_put(fence);
365 fence = next;
366
367 amdgpu_res_next(&src_mm, cur_size);
368 amdgpu_res_next(&dst_mm, cur_size);
369 }
370 error:
371 mutex_unlock(&adev->mman.gtt_window_lock);
372 *f = fence;
373 return r;
374 }
375
376 /*
377 * amdgpu_move_blit - Copy an entire buffer to another buffer
378 *
379 * This is a helper called by amdgpu_bo_move() and amdgpu_move_vram_ram() to
380 * help move buffers to and from VRAM.
381 */
amdgpu_move_blit(struct ttm_buffer_object * bo,bool evict,struct ttm_resource * new_mem,struct ttm_resource * old_mem)382 static int amdgpu_move_blit(struct ttm_buffer_object *bo,
383 bool evict,
384 struct ttm_resource *new_mem,
385 struct ttm_resource *old_mem)
386 {
387 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
388 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
389 struct amdgpu_copy_mem src, dst;
390 struct dma_fence *fence = NULL;
391 int r;
392
393 src.bo = bo;
394 dst.bo = bo;
395 src.mem = old_mem;
396 dst.mem = new_mem;
397 src.offset = 0;
398 dst.offset = 0;
399
400 r = amdgpu_ttm_copy_mem_to_mem(adev,
401 &adev->mman.move_entity,
402 &src, &dst,
403 new_mem->size,
404 amdgpu_bo_encrypted(abo),
405 bo->base.resv, &fence);
406 if (r)
407 goto error;
408
409 /* clear the space being freed */
410 if (old_mem->mem_type == TTM_PL_VRAM &&
411 (abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) {
412 struct dma_fence *wipe_fence = NULL;
413
414 r = amdgpu_fill_buffer(&adev->mman.move_entity,
415 abo, 0, NULL, &wipe_fence,
416 AMDGPU_KERNEL_JOB_ID_MOVE_BLIT);
417 if (r) {
418 goto error;
419 } else if (wipe_fence) {
420 amdgpu_vram_mgr_set_cleared(bo->resource);
421 dma_fence_put(fence);
422 fence = wipe_fence;
423 }
424 }
425
426 /* Always block for VM page tables before committing the new location */
427 if (bo->type == ttm_bo_type_kernel)
428 r = ttm_bo_move_accel_cleanup(bo, fence, true, false, new_mem);
429 else
430 r = ttm_bo_move_accel_cleanup(bo, fence, evict, true, new_mem);
431 dma_fence_put(fence);
432 return r;
433
434 error:
435 if (fence)
436 dma_fence_wait(fence, false);
437 dma_fence_put(fence);
438 return r;
439 }
440
441 /**
442 * amdgpu_res_cpu_visible - Check that resource can be accessed by CPU
443 * @adev: amdgpu device
444 * @res: the resource to check
445 *
446 * Returns: true if the full resource is CPU visible, false otherwise.
447 */
amdgpu_res_cpu_visible(struct amdgpu_device * adev,struct ttm_resource * res)448 bool amdgpu_res_cpu_visible(struct amdgpu_device *adev,
449 struct ttm_resource *res)
450 {
451 struct amdgpu_res_cursor cursor;
452
453 if (!res)
454 return false;
455
456 if (res->mem_type == TTM_PL_SYSTEM || res->mem_type == TTM_PL_TT ||
457 res->mem_type == AMDGPU_PL_PREEMPT || res->mem_type == AMDGPU_PL_DOORBELL ||
458 res->mem_type == AMDGPU_PL_MMIO_REMAP)
459 return true;
460
461 if (res->mem_type != TTM_PL_VRAM)
462 return false;
463
464 amdgpu_res_first(res, 0, res->size, &cursor);
465 while (cursor.remaining) {
466 if ((cursor.start + cursor.size) > adev->gmc.visible_vram_size)
467 return false;
468 amdgpu_res_next(&cursor, cursor.size);
469 }
470
471 return true;
472 }
473
474 /*
475 * amdgpu_res_copyable - Check that memory can be accessed by ttm_bo_move_memcpy
476 *
477 * Called by amdgpu_bo_move()
478 */
amdgpu_res_copyable(struct amdgpu_device * adev,struct ttm_resource * mem)479 static bool amdgpu_res_copyable(struct amdgpu_device *adev,
480 struct ttm_resource *mem)
481 {
482 if (!amdgpu_res_cpu_visible(adev, mem))
483 return false;
484
485 /* ttm_resource_ioremap only supports contiguous memory */
486 if (mem->mem_type == TTM_PL_VRAM &&
487 !(mem->placement & TTM_PL_FLAG_CONTIGUOUS))
488 return false;
489
490 return true;
491 }
492
493 /*
494 * amdgpu_bo_move - Move a buffer object to a new memory location
495 *
496 * Called by ttm_bo_handle_move_mem()
497 */
amdgpu_bo_move(struct ttm_buffer_object * bo,bool evict,struct ttm_operation_ctx * ctx,struct ttm_resource * new_mem,struct ttm_place * hop)498 static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
499 struct ttm_operation_ctx *ctx,
500 struct ttm_resource *new_mem,
501 struct ttm_place *hop)
502 {
503 struct amdgpu_device *adev;
504 struct amdgpu_bo *abo;
505 struct ttm_resource *old_mem = bo->resource;
506 int r;
507
508 if (new_mem->mem_type == TTM_PL_TT ||
509 new_mem->mem_type == AMDGPU_PL_PREEMPT) {
510 r = amdgpu_ttm_backend_bind(bo->bdev, bo->ttm, new_mem);
511 if (r)
512 return r;
513 }
514
515 abo = ttm_to_amdgpu_bo(bo);
516 adev = amdgpu_ttm_adev(bo->bdev);
517
518 if (!old_mem || (old_mem->mem_type == TTM_PL_SYSTEM &&
519 bo->ttm == NULL)) {
520 amdgpu_bo_move_notify(bo, evict, new_mem);
521 ttm_bo_move_null(bo, new_mem);
522 return 0;
523 }
524 if (old_mem->mem_type == TTM_PL_SYSTEM &&
525 (new_mem->mem_type == TTM_PL_TT ||
526 new_mem->mem_type == AMDGPU_PL_PREEMPT)) {
527 amdgpu_bo_move_notify(bo, evict, new_mem);
528 ttm_bo_move_null(bo, new_mem);
529 return 0;
530 }
531 if ((old_mem->mem_type == TTM_PL_TT ||
532 old_mem->mem_type == AMDGPU_PL_PREEMPT) &&
533 new_mem->mem_type == TTM_PL_SYSTEM) {
534 r = ttm_bo_wait_ctx(bo, ctx);
535 if (r)
536 return r;
537
538 amdgpu_ttm_backend_unbind(bo->bdev, bo->ttm);
539 amdgpu_bo_move_notify(bo, evict, new_mem);
540 ttm_resource_free(bo, &bo->resource);
541 ttm_bo_assign_mem(bo, new_mem);
542 return 0;
543 }
544
545 if (old_mem->mem_type == AMDGPU_PL_GDS ||
546 old_mem->mem_type == AMDGPU_PL_GWS ||
547 old_mem->mem_type == AMDGPU_PL_OA ||
548 old_mem->mem_type == AMDGPU_PL_DOORBELL ||
549 old_mem->mem_type == AMDGPU_PL_MMIO_REMAP ||
550 new_mem->mem_type == AMDGPU_PL_GDS ||
551 new_mem->mem_type == AMDGPU_PL_GWS ||
552 new_mem->mem_type == AMDGPU_PL_OA ||
553 new_mem->mem_type == AMDGPU_PL_DOORBELL ||
554 new_mem->mem_type == AMDGPU_PL_MMIO_REMAP) {
555 /* Nothing to save here */
556 amdgpu_bo_move_notify(bo, evict, new_mem);
557 ttm_bo_move_null(bo, new_mem);
558 return 0;
559 }
560
561 if (bo->type == ttm_bo_type_device &&
562 new_mem->mem_type == TTM_PL_VRAM &&
563 old_mem->mem_type != TTM_PL_VRAM) {
564 /* amdgpu_bo_fault_reserve_notify will re-set this if the CPU
565 * accesses the BO after it's moved.
566 */
567 abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
568 }
569
570 if (adev->mman.buffer_funcs_enabled &&
571 ((old_mem->mem_type == TTM_PL_SYSTEM &&
572 new_mem->mem_type == TTM_PL_VRAM) ||
573 (old_mem->mem_type == TTM_PL_VRAM &&
574 new_mem->mem_type == TTM_PL_SYSTEM))) {
575 hop->fpfn = 0;
576 hop->lpfn = 0;
577 hop->mem_type = TTM_PL_TT;
578 hop->flags = TTM_PL_FLAG_TEMPORARY;
579 return -EMULTIHOP;
580 }
581
582 amdgpu_bo_move_notify(bo, evict, new_mem);
583 if (adev->mman.buffer_funcs_enabled)
584 r = amdgpu_move_blit(bo, evict, new_mem, old_mem);
585 else
586 r = -ENODEV;
587
588 if (r) {
589 /* Check that all memory is CPU accessible */
590 if (!amdgpu_res_copyable(adev, old_mem) ||
591 !amdgpu_res_copyable(adev, new_mem)) {
592 pr_err("Move buffer fallback to memcpy unavailable\n");
593 return r;
594 }
595
596 r = ttm_bo_move_memcpy(bo, ctx, new_mem);
597 if (r)
598 return r;
599 }
600
601 /* update statistics after the move */
602 if (evict)
603 atomic64_inc(&adev->num_evictions);
604 atomic64_add(bo->base.size, &adev->num_bytes_moved);
605 return 0;
606 }
607
608 /*
609 * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault
610 *
611 * Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault()
612 */
amdgpu_ttm_io_mem_reserve(struct ttm_device * bdev,struct ttm_resource * mem)613 static int amdgpu_ttm_io_mem_reserve(struct ttm_device *bdev,
614 struct ttm_resource *mem)
615 {
616 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
617
618 switch (mem->mem_type) {
619 case TTM_PL_SYSTEM:
620 /* system memory */
621 return 0;
622 case TTM_PL_TT:
623 case AMDGPU_PL_PREEMPT:
624 break;
625 case TTM_PL_VRAM:
626 mem->bus.offset = mem->start << PAGE_SHIFT;
627
628 if (adev->mman.aper_base_kaddr &&
629 mem->placement & TTM_PL_FLAG_CONTIGUOUS)
630 mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr +
631 mem->bus.offset;
632
633 mem->bus.offset += adev->gmc.aper_base;
634 mem->bus.is_iomem = true;
635 break;
636 case AMDGPU_PL_DOORBELL:
637 mem->bus.offset = mem->start << PAGE_SHIFT;
638 mem->bus.offset += adev->doorbell.base;
639 mem->bus.is_iomem = true;
640 mem->bus.caching = ttm_uncached;
641 break;
642 case AMDGPU_PL_MMIO_REMAP:
643 mem->bus.offset = mem->start << PAGE_SHIFT;
644 mem->bus.offset += adev->rmmio_remap.bus_addr;
645 mem->bus.is_iomem = true;
646 mem->bus.caching = ttm_uncached;
647 break;
648 default:
649 return -EINVAL;
650 }
651 return 0;
652 }
653
amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object * bo,unsigned long page_offset)654 static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
655 unsigned long page_offset)
656 {
657 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
658 struct amdgpu_res_cursor cursor;
659
660 amdgpu_res_first(bo->resource, (u64)page_offset << PAGE_SHIFT, 0,
661 &cursor);
662
663 if (bo->resource->mem_type == AMDGPU_PL_DOORBELL)
664 return ((uint64_t)(adev->doorbell.base + cursor.start)) >> PAGE_SHIFT;
665 else if (bo->resource->mem_type == AMDGPU_PL_MMIO_REMAP)
666 return ((uint64_t)(adev->rmmio_remap.bus_addr + cursor.start)) >> PAGE_SHIFT;
667
668 return (adev->gmc.aper_base + cursor.start) >> PAGE_SHIFT;
669 }
670
671 /**
672 * amdgpu_ttm_domain_start - Returns GPU start address
673 * @adev: amdgpu device object
674 * @type: type of the memory
675 *
676 * Returns:
677 * GPU start address of a memory domain
678 */
679
amdgpu_ttm_domain_start(struct amdgpu_device * adev,uint32_t type)680 uint64_t amdgpu_ttm_domain_start(struct amdgpu_device *adev, uint32_t type)
681 {
682 switch (type) {
683 case TTM_PL_TT:
684 return adev->gmc.gart_start;
685 case TTM_PL_VRAM:
686 return adev->gmc.vram_start;
687 }
688
689 return 0;
690 }
691
692 /*
693 * TTM backend functions.
694 */
695 struct amdgpu_ttm_tt {
696 struct ttm_tt ttm;
697 struct drm_gem_object *gobj;
698 u64 offset;
699 uint64_t userptr;
700 struct task_struct *usertask;
701 uint32_t userflags;
702 bool bound;
703 int32_t pool_id;
704 };
705
706 #define ttm_to_amdgpu_ttm_tt(ptr) container_of(ptr, struct amdgpu_ttm_tt, ttm)
707
708 #ifdef CONFIG_DRM_AMDGPU_USERPTR
709 /*
710 * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user
711 * memory and start HMM tracking CPU page table update
712 *
713 * Calling function must call amdgpu_ttm_tt_userptr_range_done() once and only
714 * once afterwards to stop HMM tracking. Its the caller responsibility to ensure
715 * that range is a valid memory and it is freed too.
716 */
amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo * bo,struct amdgpu_hmm_range * range)717 int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo,
718 struct amdgpu_hmm_range *range)
719 {
720 struct ttm_tt *ttm = bo->tbo.ttm;
721 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
722 unsigned long start = gtt->userptr;
723 struct vm_area_struct *vma;
724 struct mm_struct *mm;
725 bool readonly;
726 int r = 0;
727
728 mm = bo->notifier.mm;
729 if (unlikely(!mm)) {
730 DRM_DEBUG_DRIVER("BO is not registered?\n");
731 return -EFAULT;
732 }
733
734 if (!mmget_not_zero(mm)) /* Happens during process shutdown */
735 return -ESRCH;
736
737 mmap_read_lock(mm);
738 vma = vma_lookup(mm, start);
739 if (unlikely(!vma)) {
740 r = -EFAULT;
741 goto out_unlock;
742 }
743 if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) &&
744 vma->vm_file)) {
745 r = -EPERM;
746 goto out_unlock;
747 }
748
749 readonly = amdgpu_ttm_tt_is_readonly(ttm);
750 r = amdgpu_hmm_range_get_pages(&bo->notifier, start, ttm->num_pages,
751 readonly, NULL, range);
752 out_unlock:
753 mmap_read_unlock(mm);
754 if (r)
755 pr_debug("failed %d to get user pages 0x%lx\n", r, start);
756
757 mmput(mm);
758
759 return r;
760 }
761
762 #endif
763
764 /*
765 * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
766 *
767 * Called by amdgpu_cs_list_validate(). This creates the page list
768 * that backs user memory and will ultimately be mapped into the device
769 * address space.
770 */
amdgpu_ttm_tt_set_user_pages(struct ttm_tt * ttm,struct amdgpu_hmm_range * range)771 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct amdgpu_hmm_range *range)
772 {
773 unsigned long i;
774
775 for (i = 0; i < ttm->num_pages; ++i)
776 ttm->pages[i] = range ? hmm_pfn_to_page(range->hmm_range.hmm_pfns[i]) : NULL;
777 }
778
779 /*
780 * amdgpu_ttm_tt_pin_userptr - prepare the sg table with the user pages
781 *
782 * Called by amdgpu_ttm_backend_bind()
783 **/
amdgpu_ttm_tt_pin_userptr(struct ttm_device * bdev,struct ttm_tt * ttm)784 static int amdgpu_ttm_tt_pin_userptr(struct ttm_device *bdev,
785 struct ttm_tt *ttm)
786 {
787 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
788 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
789 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
790 enum dma_data_direction direction = write ?
791 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
792 int r;
793
794 /* Allocate an SG array and squash pages into it */
795 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,
796 (u64)ttm->num_pages << PAGE_SHIFT,
797 GFP_KERNEL);
798 if (r)
799 goto release_sg;
800
801 /* Map SG to device */
802 r = dma_map_sgtable(adev->dev, ttm->sg, direction, 0);
803 if (r)
804 goto release_sg_table;
805
806 /* convert SG to linear array of pages and dma addresses */
807 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address,
808 ttm->num_pages);
809
810 return 0;
811
812 release_sg_table:
813 sg_free_table(ttm->sg);
814 release_sg:
815 kfree(ttm->sg);
816 ttm->sg = NULL;
817 return r;
818 }
819
820 /*
821 * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages
822 */
amdgpu_ttm_tt_unpin_userptr(struct ttm_device * bdev,struct ttm_tt * ttm)823 static void amdgpu_ttm_tt_unpin_userptr(struct ttm_device *bdev,
824 struct ttm_tt *ttm)
825 {
826 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
827 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
828 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
829 enum dma_data_direction direction = write ?
830 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
831
832 /* double check that we don't free the table twice */
833 if (!ttm->sg || !ttm->sg->sgl)
834 return;
835
836 /* unmap the pages mapped to the device */
837 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0);
838 sg_free_table(ttm->sg);
839 }
840
841 /*
842 * total_pages is constructed as MQD0+CtrlStack0 + MQD1+CtrlStack1 + ...
843 * MQDn+CtrlStackn where n is the number of XCCs per partition.
844 * pages_per_xcc is the size of one MQD+CtrlStack. The first page is MQD
845 * and uses memory type default, UC. The rest of pages_per_xcc are
846 * Ctrl stack and modify their memory type to NC.
847 */
amdgpu_ttm_gart_bind_gfx9_mqd(struct amdgpu_device * adev,struct ttm_tt * ttm,uint64_t flags)848 static void amdgpu_ttm_gart_bind_gfx9_mqd(struct amdgpu_device *adev,
849 struct ttm_tt *ttm, uint64_t flags)
850 {
851 struct amdgpu_ttm_tt *gtt = (void *)ttm;
852 uint64_t total_pages = ttm->num_pages;
853 int num_xcc = max(1U, adev->gfx.num_xcc_per_xcp);
854 uint64_t page_idx, pages_per_xcc;
855 int i;
856 uint64_t ctrl_flags = AMDGPU_PTE_MTYPE_VG10(flags, AMDGPU_MTYPE_NC);
857
858 pages_per_xcc = total_pages;
859 do_div(pages_per_xcc, num_xcc);
860
861 for (i = 0, page_idx = 0; i < num_xcc; i++, page_idx += pages_per_xcc) {
862 /* MQD page: use default flags */
863 amdgpu_gart_bind(adev,
864 gtt->offset + (page_idx << PAGE_SHIFT),
865 1, >t->ttm.dma_address[page_idx], flags);
866 /*
867 * Ctrl pages - modify the memory type to NC (ctrl_flags) from
868 * the second page of the BO onward.
869 */
870 amdgpu_gart_bind(adev,
871 gtt->offset + ((page_idx + 1) << PAGE_SHIFT),
872 pages_per_xcc - 1,
873 >t->ttm.dma_address[page_idx + 1],
874 ctrl_flags);
875 }
876 }
877
amdgpu_ttm_gart_bind(struct amdgpu_device * adev,struct ttm_buffer_object * tbo,uint64_t flags)878 static void amdgpu_ttm_gart_bind(struct amdgpu_device *adev,
879 struct ttm_buffer_object *tbo,
880 uint64_t flags)
881 {
882 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo);
883 struct ttm_tt *ttm = tbo->ttm;
884 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
885
886 if (amdgpu_bo_encrypted(abo))
887 flags |= AMDGPU_PTE_TMZ;
888
889 if (abo->flags & AMDGPU_GEM_CREATE_CP_MQD_GFX9) {
890 amdgpu_ttm_gart_bind_gfx9_mqd(adev, ttm, flags);
891 } else {
892 amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
893 gtt->ttm.dma_address, flags);
894 }
895 gtt->bound = true;
896 }
897
898 /*
899 * amdgpu_ttm_backend_bind - Bind GTT memory
900 *
901 * Called by ttm_tt_bind() on behalf of ttm_bo_handle_move_mem().
902 * This handles binding GTT memory to the device address space.
903 */
amdgpu_ttm_backend_bind(struct ttm_device * bdev,struct ttm_tt * ttm,struct ttm_resource * bo_mem)904 static int amdgpu_ttm_backend_bind(struct ttm_device *bdev,
905 struct ttm_tt *ttm,
906 struct ttm_resource *bo_mem)
907 {
908 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
909 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
910 uint64_t flags;
911 int r;
912
913 if (!bo_mem)
914 return -EINVAL;
915
916 if (gtt->bound)
917 return 0;
918
919 if (gtt->userptr) {
920 r = amdgpu_ttm_tt_pin_userptr(bdev, ttm);
921 if (r) {
922 dev_err(adev->dev, "failed to pin userptr\n");
923 return r;
924 }
925 } else if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) {
926 if (!ttm->sg) {
927 struct dma_buf_attachment *attach;
928 struct sg_table *sgt;
929
930 attach = gtt->gobj->import_attach;
931 sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
932 if (IS_ERR(sgt))
933 return PTR_ERR(sgt);
934
935 ttm->sg = sgt;
936 }
937
938 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address,
939 ttm->num_pages);
940 }
941
942 if (!ttm->num_pages) {
943 WARN(1, "nothing to bind %u pages for mreg %p back %p!\n",
944 ttm->num_pages, bo_mem, ttm);
945 }
946
947 if (bo_mem->mem_type != TTM_PL_TT ||
948 !amdgpu_gtt_mgr_has_gart_addr(bo_mem)) {
949 gtt->offset = AMDGPU_BO_INVALID_OFFSET;
950 return 0;
951 }
952
953 /* compute PTE flags relevant to this BO memory */
954 flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem);
955
956 /* bind pages into GART page tables */
957 gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
958 amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
959 gtt->ttm.dma_address, flags);
960 gtt->bound = true;
961 return 0;
962 }
963
964 /*
965 * amdgpu_ttm_alloc_gart - Make sure buffer object is accessible either
966 * through AGP or GART aperture.
967 *
968 * If bo is accessible through AGP aperture, then use AGP aperture
969 * to access bo; otherwise allocate logical space in GART aperture
970 * and map bo to GART aperture.
971 */
amdgpu_ttm_alloc_gart(struct ttm_buffer_object * bo)972 int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
973 {
974 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
975 struct ttm_operation_ctx ctx = { false, false };
976 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(bo->ttm);
977 struct ttm_placement placement;
978 struct ttm_place placements;
979 struct ttm_resource *tmp;
980 uint64_t addr, flags;
981 int r;
982
983 if (bo->resource->start != AMDGPU_BO_INVALID_OFFSET)
984 return 0;
985
986 addr = amdgpu_gmc_agp_addr(bo);
987 if (addr != AMDGPU_BO_INVALID_OFFSET)
988 return 0;
989
990 /* allocate GART space */
991 placement.num_placement = 1;
992 placement.placement = &placements;
993 placements.fpfn = 0;
994 placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT;
995 placements.mem_type = TTM_PL_TT;
996 placements.flags = bo->resource->placement;
997
998 r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx);
999 if (unlikely(r))
1000 return r;
1001
1002 /* compute PTE flags for this buffer object */
1003 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, tmp);
1004
1005 /* Bind pages */
1006 gtt->offset = (u64)tmp->start << PAGE_SHIFT;
1007 amdgpu_ttm_gart_bind(adev, bo, flags);
1008 amdgpu_gart_invalidate_tlb(adev);
1009 ttm_resource_free(bo, &bo->resource);
1010 ttm_bo_assign_mem(bo, tmp);
1011
1012 return 0;
1013 }
1014
1015 /*
1016 * amdgpu_ttm_recover_gart - Rebind GTT pages
1017 *
1018 * Called by amdgpu_gtt_mgr_recover() from amdgpu_device_reset() to
1019 * rebind GTT pages during a GPU reset.
1020 */
amdgpu_ttm_recover_gart(struct ttm_buffer_object * tbo)1021 void amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo)
1022 {
1023 struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
1024 uint64_t flags;
1025
1026 if (!tbo->ttm)
1027 return;
1028
1029 flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, tbo->resource);
1030 amdgpu_ttm_gart_bind(adev, tbo, flags);
1031 }
1032
1033 /*
1034 * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages
1035 *
1036 * Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and
1037 * ttm_tt_destroy().
1038 */
amdgpu_ttm_backend_unbind(struct ttm_device * bdev,struct ttm_tt * ttm)1039 static void amdgpu_ttm_backend_unbind(struct ttm_device *bdev,
1040 struct ttm_tt *ttm)
1041 {
1042 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
1043 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
1044
1045 /* if the pages have userptr pinning then clear that first */
1046 if (gtt->userptr) {
1047 amdgpu_ttm_tt_unpin_userptr(bdev, ttm);
1048 } else if (ttm->sg && drm_gem_is_imported(gtt->gobj)) {
1049 struct dma_buf_attachment *attach;
1050
1051 attach = gtt->gobj->import_attach;
1052 dma_buf_unmap_attachment(attach, ttm->sg, DMA_BIDIRECTIONAL);
1053 ttm->sg = NULL;
1054 }
1055
1056 if (!gtt->bound)
1057 return;
1058
1059 if (gtt->offset == AMDGPU_BO_INVALID_OFFSET)
1060 return;
1061
1062 /* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */
1063 amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages);
1064 gtt->bound = false;
1065 }
1066
amdgpu_ttm_backend_destroy(struct ttm_device * bdev,struct ttm_tt * ttm)1067 static void amdgpu_ttm_backend_destroy(struct ttm_device *bdev,
1068 struct ttm_tt *ttm)
1069 {
1070 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
1071
1072 if (gtt->usertask)
1073 put_task_struct(gtt->usertask);
1074
1075 ttm_tt_fini(>t->ttm);
1076 kfree(gtt);
1077 }
1078
1079 /**
1080 * amdgpu_ttm_mmio_remap_alloc_sgt - build an sg_table for MMIO_REMAP I/O aperture
1081 * @adev: amdgpu device providing the remap BAR base (adev->rmmio_remap.bus_addr)
1082 * @res: TTM resource of the BO to export; expected to live in AMDGPU_PL_MMIO_REMAP
1083 * @dev: importing device to map for (typically @attach->dev in dma-buf paths)
1084 * @dir: DMA data direction for the importer (passed to dma_map_resource())
1085 * @sgt: output; on success, set to a newly allocated sg_table describing the I/O span
1086 *
1087 * The HDP flush page (AMDGPU_PL_MMIO_REMAP) is a fixed hardware I/O window in a PCI
1088 * BAR—there are no struct pages to back it. Importers still need a DMA address list,
1089 * so we synthesize a minimal sg_table and populate it from dma_map_resource(), not
1090 * from pages. Using the common amdgpu_res_cursor walker keeps the offset/size math
1091 * consistent with other TTM/manager users.
1092 *
1093 * - @res is assumed to be a small, contiguous I/O region (typically a single 4 KiB
1094 * page) in AMDGPU_PL_MMIO_REMAP. Callers should validate placement before calling.
1095 * - The sg entry is created with sg_set_page(sg, NULL, …) to reflect I/O space.
1096 * - The mapping uses DMA_ATTR_SKIP_CPU_SYNC because this is MMIO, not cacheable RAM.
1097 * - Peer reachability / p2pdma policy checks must be done by the caller.
1098 *
1099 * Return:
1100 * * 0 on success, with *@sgt set to a valid table that must be freed via
1101 * amdgpu_ttm_mmio_remap_free_sgt().
1102 * * -ENOMEM if allocation of the sg_table fails.
1103 * * -EIO if dma_map_resource() fails.
1104 *
1105 */
amdgpu_ttm_mmio_remap_alloc_sgt(struct amdgpu_device * adev,struct ttm_resource * res,struct device * dev,enum dma_data_direction dir,struct sg_table ** sgt)1106 int amdgpu_ttm_mmio_remap_alloc_sgt(struct amdgpu_device *adev,
1107 struct ttm_resource *res,
1108 struct device *dev,
1109 enum dma_data_direction dir,
1110 struct sg_table **sgt)
1111 {
1112 struct amdgpu_res_cursor cur;
1113 dma_addr_t dma;
1114 resource_size_t phys;
1115 struct scatterlist *sg;
1116 int r;
1117
1118 /* Walk the resource once; MMIO_REMAP is expected to be contiguous+small. */
1119 amdgpu_res_first(res, 0, res->size, &cur);
1120
1121 /* Translate byte offset in the remap window into a host physical BAR address. */
1122 phys = adev->rmmio_remap.bus_addr + cur.start;
1123
1124 /* Build a single-entry sg_table mapped as I/O (no struct page backing). */
1125 *sgt = kzalloc_obj(**sgt);
1126 if (!*sgt)
1127 return -ENOMEM;
1128 r = sg_alloc_table(*sgt, 1, GFP_KERNEL);
1129 if (r) {
1130 kfree(*sgt);
1131 return r;
1132 }
1133 sg = (*sgt)->sgl;
1134 sg_set_page(sg, NULL, cur.size, 0); /* WHY: I/O space → no pages */
1135
1136 dma = dma_map_resource(dev, phys, cur.size, dir, DMA_ATTR_SKIP_CPU_SYNC);
1137 if (dma_mapping_error(dev, dma)) {
1138 sg_free_table(*sgt);
1139 kfree(*sgt);
1140 return -EIO;
1141 }
1142 sg_dma_address(sg) = dma;
1143 sg_dma_len(sg) = cur.size;
1144 return 0;
1145 }
1146
amdgpu_ttm_mmio_remap_free_sgt(struct device * dev,enum dma_data_direction dir,struct sg_table * sgt)1147 void amdgpu_ttm_mmio_remap_free_sgt(struct device *dev,
1148 enum dma_data_direction dir,
1149 struct sg_table *sgt)
1150 {
1151 struct scatterlist *sg = sgt->sgl;
1152
1153 dma_unmap_resource(dev, sg_dma_address(sg), sg_dma_len(sg),
1154 dir, DMA_ATTR_SKIP_CPU_SYNC);
1155 sg_free_table(sgt);
1156 kfree(sgt);
1157 }
1158
1159 /**
1160 * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
1161 *
1162 * @bo: The buffer object to create a GTT ttm_tt object around
1163 * @page_flags: Page flags to be added to the ttm_tt object
1164 *
1165 * Called by ttm_tt_create().
1166 */
amdgpu_ttm_tt_create(struct ttm_buffer_object * bo,uint32_t page_flags)1167 static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
1168 uint32_t page_flags)
1169 {
1170 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1171 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1172 struct amdgpu_ttm_tt *gtt;
1173 enum ttm_caching caching;
1174
1175 gtt = kzalloc_obj(struct amdgpu_ttm_tt);
1176 if (!gtt)
1177 return NULL;
1178
1179 gtt->gobj = &bo->base;
1180 if (adev->gmc.mem_partitions && abo->xcp_id >= 0)
1181 gtt->pool_id = KFD_XCP_MEM_ID(adev, abo->xcp_id);
1182 else
1183 gtt->pool_id = abo->xcp_id;
1184
1185 if (abo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
1186 caching = ttm_write_combined;
1187 else
1188 caching = ttm_cached;
1189
1190 /* allocate space for the uninitialized page entries */
1191 if (ttm_sg_tt_init(>t->ttm, bo, page_flags, caching)) {
1192 kfree(gtt);
1193 return NULL;
1194 }
1195 return >t->ttm;
1196 }
1197
1198 /*
1199 * amdgpu_ttm_tt_populate - Map GTT pages visible to the device
1200 *
1201 * Map the pages of a ttm_tt object to an address space visible
1202 * to the underlying device.
1203 */
amdgpu_ttm_tt_populate(struct ttm_device * bdev,struct ttm_tt * ttm,struct ttm_operation_ctx * ctx)1204 static int amdgpu_ttm_tt_populate(struct ttm_device *bdev,
1205 struct ttm_tt *ttm,
1206 struct ttm_operation_ctx *ctx)
1207 {
1208 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
1209 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
1210 struct ttm_pool *pool;
1211 pgoff_t i;
1212 int ret;
1213
1214 /* user pages are bound by amdgpu_ttm_tt_pin_userptr() */
1215 if (gtt->userptr) {
1216 ttm->sg = kzalloc_obj(struct sg_table);
1217 if (!ttm->sg)
1218 return -ENOMEM;
1219 return 0;
1220 }
1221
1222 if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL)
1223 return 0;
1224
1225 if (adev->mman.ttm_pools && gtt->pool_id >= 0)
1226 pool = &adev->mman.ttm_pools[gtt->pool_id];
1227 else
1228 pool = &adev->mman.bdev.pool;
1229 ret = ttm_pool_alloc(pool, ttm, ctx);
1230 if (ret)
1231 return ret;
1232
1233 for (i = 0; i < ttm->num_pages; ++i)
1234 ttm->pages[i]->mapping = bdev->dev_mapping;
1235
1236 return 0;
1237 }
1238
1239 /*
1240 * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays
1241 *
1242 * Unmaps pages of a ttm_tt object from the device address space and
1243 * unpopulates the page array backing it.
1244 */
amdgpu_ttm_tt_unpopulate(struct ttm_device * bdev,struct ttm_tt * ttm)1245 static void amdgpu_ttm_tt_unpopulate(struct ttm_device *bdev,
1246 struct ttm_tt *ttm)
1247 {
1248 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
1249 struct amdgpu_device *adev;
1250 struct ttm_pool *pool;
1251 pgoff_t i;
1252
1253 amdgpu_ttm_backend_unbind(bdev, ttm);
1254
1255 if (gtt->userptr) {
1256 amdgpu_ttm_tt_set_user_pages(ttm, NULL);
1257 kfree(ttm->sg);
1258 ttm->sg = NULL;
1259 return;
1260 }
1261
1262 if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL)
1263 return;
1264
1265 for (i = 0; i < ttm->num_pages; ++i)
1266 ttm->pages[i]->mapping = NULL;
1267
1268 adev = amdgpu_ttm_adev(bdev);
1269
1270 if (adev->mman.ttm_pools && gtt->pool_id >= 0)
1271 pool = &adev->mman.ttm_pools[gtt->pool_id];
1272 else
1273 pool = &adev->mman.bdev.pool;
1274
1275 return ttm_pool_free(pool, ttm);
1276 }
1277
1278 /**
1279 * amdgpu_ttm_tt_get_userptr - Return the userptr GTT ttm_tt for the current
1280 * task
1281 *
1282 * @tbo: The ttm_buffer_object that contains the userptr
1283 * @user_addr: The returned value
1284 */
amdgpu_ttm_tt_get_userptr(const struct ttm_buffer_object * tbo,uint64_t * user_addr)1285 int amdgpu_ttm_tt_get_userptr(const struct ttm_buffer_object *tbo,
1286 uint64_t *user_addr)
1287 {
1288 struct amdgpu_ttm_tt *gtt;
1289
1290 if (!tbo->ttm)
1291 return -EINVAL;
1292
1293 gtt = (void *)tbo->ttm;
1294 *user_addr = gtt->userptr;
1295 return 0;
1296 }
1297
1298 /**
1299 * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current
1300 * task
1301 *
1302 * @bo: The ttm_buffer_object to bind this userptr to
1303 * @addr: The address in the current tasks VM space to use
1304 * @flags: Requirements of userptr object.
1305 *
1306 * Called by amdgpu_gem_userptr_ioctl() and kfd_ioctl_alloc_memory_of_gpu() to
1307 * bind userptr pages to current task and by kfd_ioctl_acquire_vm() to
1308 * initialize GPU VM for a KFD process.
1309 */
amdgpu_ttm_tt_set_userptr(struct ttm_buffer_object * bo,uint64_t addr,uint32_t flags)1310 int amdgpu_ttm_tt_set_userptr(struct ttm_buffer_object *bo,
1311 uint64_t addr, uint32_t flags)
1312 {
1313 struct amdgpu_ttm_tt *gtt;
1314
1315 if (!bo->ttm) {
1316 /* TODO: We want a separate TTM object type for userptrs */
1317 bo->ttm = amdgpu_ttm_tt_create(bo, 0);
1318 if (bo->ttm == NULL)
1319 return -ENOMEM;
1320 }
1321
1322 /* Set TTM_TT_FLAG_EXTERNAL before populate but after create. */
1323 bo->ttm->page_flags |= TTM_TT_FLAG_EXTERNAL;
1324
1325 gtt = ttm_to_amdgpu_ttm_tt(bo->ttm);
1326 gtt->userptr = addr;
1327 gtt->userflags = flags;
1328
1329 if (gtt->usertask)
1330 put_task_struct(gtt->usertask);
1331 gtt->usertask = current->group_leader;
1332 get_task_struct(gtt->usertask);
1333
1334 return 0;
1335 }
1336
1337 /*
1338 * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object
1339 */
amdgpu_ttm_tt_get_usermm(struct ttm_tt * ttm)1340 struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)
1341 {
1342 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
1343
1344 if (gtt == NULL)
1345 return NULL;
1346
1347 if (gtt->usertask == NULL)
1348 return NULL;
1349
1350 return gtt->usertask->mm;
1351 }
1352
1353 /*
1354 * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an
1355 * address range for the current task.
1356 *
1357 */
amdgpu_ttm_tt_affect_userptr(struct ttm_tt * ttm,unsigned long start,unsigned long end,unsigned long * userptr)1358 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
1359 unsigned long end, unsigned long *userptr)
1360 {
1361 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
1362 unsigned long size;
1363
1364 if (gtt == NULL || !gtt->userptr)
1365 return false;
1366
1367 /* Return false if no part of the ttm_tt object lies within
1368 * the range
1369 */
1370 size = (unsigned long)gtt->ttm.num_pages * PAGE_SIZE;
1371 if (gtt->userptr > end || gtt->userptr + size <= start)
1372 return false;
1373
1374 if (userptr)
1375 *userptr = gtt->userptr;
1376 return true;
1377 }
1378
1379 /*
1380 * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr?
1381 */
amdgpu_ttm_tt_is_userptr(struct ttm_tt * ttm)1382 bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm)
1383 {
1384 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
1385
1386 if (gtt == NULL || !gtt->userptr)
1387 return false;
1388
1389 return true;
1390 }
1391
1392 /*
1393 * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only?
1394 */
amdgpu_ttm_tt_is_readonly(struct ttm_tt * ttm)1395 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
1396 {
1397 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
1398
1399 if (gtt == NULL)
1400 return false;
1401
1402 return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
1403 }
1404
1405 /**
1406 * amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object
1407 *
1408 * @ttm: The ttm_tt object to compute the flags for
1409 * @mem: The memory registry backing this ttm_tt object
1410 *
1411 * Figure out the flags to use for a VM PDE (Page Directory Entry).
1412 */
amdgpu_ttm_tt_pde_flags(struct ttm_tt * ttm,struct ttm_resource * mem)1413 uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_resource *mem)
1414 {
1415 uint64_t flags = 0;
1416
1417 if (mem && mem->mem_type != TTM_PL_SYSTEM)
1418 flags |= AMDGPU_PTE_VALID;
1419
1420 if (mem && (mem->mem_type == TTM_PL_TT ||
1421 mem->mem_type == AMDGPU_PL_DOORBELL ||
1422 mem->mem_type == AMDGPU_PL_PREEMPT ||
1423 mem->mem_type == AMDGPU_PL_MMIO_REMAP)) {
1424 flags |= AMDGPU_PTE_SYSTEM;
1425
1426 if (ttm && ttm->caching == ttm_cached)
1427 flags |= AMDGPU_PTE_SNOOPED;
1428 }
1429
1430 if (mem && mem->mem_type == TTM_PL_VRAM &&
1431 mem->bus.caching == ttm_cached)
1432 flags |= AMDGPU_PTE_SNOOPED;
1433
1434 return flags;
1435 }
1436
1437 /**
1438 * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object
1439 *
1440 * @adev: amdgpu_device pointer
1441 * @ttm: The ttm_tt object to compute the flags for
1442 * @mem: The memory registry backing this ttm_tt object
1443 *
1444 * Figure out the flags to use for a VM PTE (Page Table Entry).
1445 */
amdgpu_ttm_tt_pte_flags(struct amdgpu_device * adev,struct ttm_tt * ttm,struct ttm_resource * mem)1446 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
1447 struct ttm_resource *mem)
1448 {
1449 uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem);
1450
1451 flags |= adev->gart.gart_pte_flags;
1452 flags |= AMDGPU_PTE_READABLE;
1453
1454 if (!amdgpu_ttm_tt_is_readonly(ttm))
1455 flags |= AMDGPU_PTE_WRITEABLE;
1456
1457 return flags;
1458 }
1459
1460 /*
1461 * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer
1462 * object.
1463 *
1464 * Return true if eviction is sensible. Called by ttm_mem_evict_first() on
1465 * behalf of ttm_bo_mem_force_space() which tries to evict buffer objects until
1466 * it can find space for a new object and by ttm_bo_force_list_clean() which is
1467 * used to clean out a memory space.
1468 */
amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object * bo,const struct ttm_place * place)1469 static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
1470 const struct ttm_place *place)
1471 {
1472 struct dma_resv_iter resv_cursor;
1473 struct dma_fence *f;
1474
1475 if (!amdgpu_bo_is_amdgpu_bo(bo))
1476 return ttm_bo_eviction_valuable(bo, place);
1477
1478 /* Swapout? */
1479 if (bo->resource->mem_type == TTM_PL_SYSTEM)
1480 return true;
1481
1482 if (bo->type == ttm_bo_type_kernel &&
1483 !amdgpu_vm_evictable(ttm_to_amdgpu_bo(bo)))
1484 return false;
1485
1486 /* If bo is a KFD BO, check if the bo belongs to the current process.
1487 * If true, then return false as any KFD process needs all its BOs to
1488 * be resident to run successfully
1489 */
1490 dma_resv_for_each_fence(&resv_cursor, bo->base.resv,
1491 DMA_RESV_USAGE_BOOKKEEP, f) {
1492 if (amdkfd_fence_check_mm(f, current->mm) &&
1493 !(place->flags & TTM_PL_FLAG_CONTIGUOUS))
1494 return false;
1495 }
1496
1497 /* Preemptible BOs don't own system resources managed by the
1498 * driver (pages, VRAM, GART space). They point to resources
1499 * owned by someone else (e.g. pageable memory in user mode
1500 * or a DMABuf). They are used in a preemptible context so we
1501 * can guarantee no deadlocks and good QoS in case of MMU
1502 * notifiers or DMABuf move notifiers from the resource owner.
1503 */
1504 if (bo->resource->mem_type == AMDGPU_PL_PREEMPT)
1505 return false;
1506
1507 if (bo->resource->mem_type == TTM_PL_TT &&
1508 amdgpu_bo_encrypted(ttm_to_amdgpu_bo(bo)))
1509 return false;
1510
1511 return ttm_bo_eviction_valuable(bo, place);
1512 }
1513
amdgpu_ttm_vram_mm_access(struct amdgpu_device * adev,loff_t pos,void * buf,size_t size,bool write)1514 static void amdgpu_ttm_vram_mm_access(struct amdgpu_device *adev, loff_t pos,
1515 void *buf, size_t size, bool write)
1516 {
1517 while (size) {
1518 uint64_t aligned_pos = ALIGN_DOWN(pos, 4);
1519 uint64_t bytes = 4 - (pos & 0x3);
1520 uint32_t shift = (pos & 0x3) * 8;
1521 uint32_t mask = 0xffffffff << shift;
1522 uint32_t value = 0;
1523
1524 if (size < bytes) {
1525 mask &= 0xffffffff >> (bytes - size) * 8;
1526 bytes = size;
1527 }
1528
1529 if (mask != 0xffffffff) {
1530 amdgpu_device_mm_access(adev, aligned_pos, &value, 4, false);
1531 if (write) {
1532 value &= ~mask;
1533 value |= (*(uint32_t *)buf << shift) & mask;
1534 amdgpu_device_mm_access(adev, aligned_pos, &value, 4, true);
1535 } else {
1536 value = (value & mask) >> shift;
1537 memcpy(buf, &value, bytes);
1538 }
1539 } else {
1540 amdgpu_device_mm_access(adev, aligned_pos, buf, 4, write);
1541 }
1542
1543 pos += bytes;
1544 buf += bytes;
1545 size -= bytes;
1546 }
1547 }
1548
amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object * bo,unsigned long offset,void * buf,int len,int write)1549 static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo,
1550 unsigned long offset, void *buf,
1551 int len, int write)
1552 {
1553 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1554 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
1555 struct amdgpu_res_cursor src_mm;
1556 struct amdgpu_job *job;
1557 struct dma_fence *fence;
1558 uint64_t src_addr, dst_addr;
1559 unsigned int num_dw;
1560 int r, idx;
1561
1562 if (len != PAGE_SIZE)
1563 return -EINVAL;
1564
1565 if (!adev->mman.sdma_access_ptr)
1566 return -EACCES;
1567
1568 if (!drm_dev_enter(adev_to_drm(adev), &idx))
1569 return -ENODEV;
1570
1571 if (write)
1572 memcpy(adev->mman.sdma_access_ptr, buf, len);
1573
1574 num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
1575 r = amdgpu_job_alloc_with_ib(adev, &adev->mman.default_entity.base,
1576 AMDGPU_FENCE_OWNER_UNDEFINED,
1577 num_dw * 4, AMDGPU_IB_POOL_DELAYED,
1578 &job,
1579 AMDGPU_KERNEL_JOB_ID_TTM_ACCESS_MEMORY_SDMA);
1580 if (r)
1581 goto out;
1582
1583 mutex_lock(&adev->mman.gtt_window_lock);
1584 amdgpu_res_first(abo->tbo.resource, offset, len, &src_mm);
1585 src_addr = amdgpu_ttm_domain_start(adev, bo->resource->mem_type) +
1586 src_mm.start;
1587 dst_addr = amdgpu_bo_gpu_offset(adev->mman.sdma_access_bo);
1588 if (write)
1589 swap(src_addr, dst_addr);
1590
1591 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, dst_addr,
1592 PAGE_SIZE, 0);
1593
1594 fence = amdgpu_ttm_job_submit(adev, job, num_dw);
1595 mutex_unlock(&adev->mman.gtt_window_lock);
1596
1597 if (!dma_fence_wait_timeout(fence, false, adev->sdma_timeout))
1598 r = -ETIMEDOUT;
1599 dma_fence_put(fence);
1600
1601 if (!(r || write))
1602 memcpy(buf, adev->mman.sdma_access_ptr, len);
1603 out:
1604 drm_dev_exit(idx);
1605 return r;
1606 }
1607
1608 /**
1609 * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object.
1610 *
1611 * @bo: The buffer object to read/write
1612 * @offset: Offset into buffer object
1613 * @buf: Secondary buffer to write/read from
1614 * @len: Length in bytes of access
1615 * @write: true if writing
1616 *
1617 * This is used to access VRAM that backs a buffer object via MMIO
1618 * access for debugging purposes.
1619 */
amdgpu_ttm_access_memory(struct ttm_buffer_object * bo,unsigned long offset,void * buf,int len,int write)1620 static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
1621 unsigned long offset, void *buf, int len,
1622 int write)
1623 {
1624 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1625 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
1626 struct amdgpu_res_cursor cursor;
1627 int ret = 0;
1628
1629 if (bo->resource->mem_type != TTM_PL_VRAM)
1630 return -EIO;
1631
1632 if (amdgpu_device_has_timeouts_enabled(adev) &&
1633 !amdgpu_ttm_access_memory_sdma(bo, offset, buf, len, write))
1634 return len;
1635
1636 amdgpu_res_first(bo->resource, offset, len, &cursor);
1637 while (cursor.remaining) {
1638 size_t count, size = cursor.size;
1639 loff_t pos = cursor.start;
1640
1641 count = amdgpu_device_aper_access(adev, pos, buf, size, write);
1642 size -= count;
1643 if (size) {
1644 /* using MM to access rest vram and handle un-aligned address */
1645 pos += count;
1646 buf += count;
1647 amdgpu_ttm_vram_mm_access(adev, pos, buf, size, write);
1648 }
1649
1650 ret += cursor.size;
1651 buf += cursor.size;
1652 amdgpu_res_next(&cursor, cursor.size);
1653 }
1654
1655 return ret;
1656 }
1657
1658 static void
amdgpu_bo_delete_mem_notify(struct ttm_buffer_object * bo)1659 amdgpu_bo_delete_mem_notify(struct ttm_buffer_object *bo)
1660 {
1661 amdgpu_bo_move_notify(bo, false, NULL);
1662 }
1663
1664 static struct ttm_device_funcs amdgpu_bo_driver = {
1665 .ttm_tt_create = &amdgpu_ttm_tt_create,
1666 .ttm_tt_populate = &amdgpu_ttm_tt_populate,
1667 .ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate,
1668 .ttm_tt_destroy = &amdgpu_ttm_backend_destroy,
1669 .eviction_valuable = amdgpu_ttm_bo_eviction_valuable,
1670 .evict_flags = &amdgpu_evict_flags,
1671 .move = &amdgpu_bo_move,
1672 .delete_mem_notify = &amdgpu_bo_delete_mem_notify,
1673 .release_notify = &amdgpu_bo_release_notify,
1674 .io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
1675 .io_mem_pfn = amdgpu_ttm_io_mem_pfn,
1676 .access_memory = &amdgpu_ttm_access_memory,
1677 };
1678
1679 /*
1680 * Firmware Reservation functions
1681 */
1682 /**
1683 * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram
1684 *
1685 * @adev: amdgpu_device pointer
1686 *
1687 * free fw reserved vram if it has been reserved.
1688 */
amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device * adev)1689 static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
1690 {
1691 amdgpu_bo_free_kernel(&adev->mman.fw_vram_usage_reserved_bo,
1692 NULL, &adev->mman.fw_vram_usage_va);
1693 }
1694
1695 /*
1696 * Driver Reservation functions
1697 */
1698 /**
1699 * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
1700 *
1701 * @adev: amdgpu_device pointer
1702 *
1703 * free drv reserved vram if it has been reserved.
1704 */
amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device * adev)1705 static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev)
1706 {
1707 amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,
1708 NULL,
1709 &adev->mman.drv_vram_usage_va);
1710 }
1711
1712 /**
1713 * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
1714 *
1715 * @adev: amdgpu_device pointer
1716 *
1717 * create bo vram reservation from fw.
1718 */
amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device * adev)1719 static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
1720 {
1721 uint64_t vram_size = adev->gmc.visible_vram_size;
1722
1723 adev->mman.fw_vram_usage_va = NULL;
1724 adev->mman.fw_vram_usage_reserved_bo = NULL;
1725
1726 if (adev->mman.fw_vram_usage_size == 0 ||
1727 adev->mman.fw_vram_usage_size > vram_size)
1728 return 0;
1729
1730 return amdgpu_bo_create_kernel_at(adev,
1731 adev->mman.fw_vram_usage_start_offset,
1732 adev->mman.fw_vram_usage_size,
1733 &adev->mman.fw_vram_usage_reserved_bo,
1734 &adev->mman.fw_vram_usage_va);
1735 }
1736
1737 /**
1738 * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
1739 *
1740 * @adev: amdgpu_device pointer
1741 *
1742 * create bo vram reservation from drv.
1743 */
amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device * adev)1744 static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
1745 {
1746 u64 vram_size = adev->gmc.visible_vram_size;
1747
1748 adev->mman.drv_vram_usage_va = NULL;
1749 adev->mman.drv_vram_usage_reserved_bo = NULL;
1750
1751 if (adev->mman.drv_vram_usage_size == 0 ||
1752 adev->mman.drv_vram_usage_size > vram_size)
1753 return 0;
1754
1755 return amdgpu_bo_create_kernel_at(adev,
1756 adev->mman.drv_vram_usage_start_offset,
1757 adev->mman.drv_vram_usage_size,
1758 &adev->mman.drv_vram_usage_reserved_bo,
1759 &adev->mman.drv_vram_usage_va);
1760 }
1761
1762 /*
1763 * Memoy training reservation functions
1764 */
1765
1766 /**
1767 * amdgpu_ttm_training_reserve_vram_fini - free memory training reserved vram
1768 *
1769 * @adev: amdgpu_device pointer
1770 *
1771 * free memory training reserved vram if it has been reserved.
1772 */
amdgpu_ttm_training_reserve_vram_fini(struct amdgpu_device * adev)1773 static int amdgpu_ttm_training_reserve_vram_fini(struct amdgpu_device *adev)
1774 {
1775 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx;
1776
1777 ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT;
1778 amdgpu_bo_free_kernel(&ctx->c2p_bo, NULL, NULL);
1779 ctx->c2p_bo = NULL;
1780
1781 return 0;
1782 }
1783
amdgpu_ttm_training_data_block_init(struct amdgpu_device * adev,uint32_t reserve_size)1784 static void amdgpu_ttm_training_data_block_init(struct amdgpu_device *adev,
1785 uint32_t reserve_size)
1786 {
1787 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx;
1788
1789 memset(ctx, 0, sizeof(*ctx));
1790
1791 ctx->c2p_train_data_offset =
1792 ALIGN((adev->gmc.mc_vram_size - reserve_size - SZ_1M), SZ_1M);
1793 ctx->p2c_train_data_offset =
1794 (adev->gmc.mc_vram_size - GDDR6_MEM_TRAINING_OFFSET);
1795 ctx->train_data_size =
1796 GDDR6_MEM_TRAINING_DATA_SIZE_IN_BYTES;
1797
1798 DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n",
1799 ctx->train_data_size,
1800 ctx->p2c_train_data_offset,
1801 ctx->c2p_train_data_offset);
1802 }
1803
1804 /*
1805 * reserve TMR memory at the top of VRAM which holds
1806 * IP Discovery data and is protected by PSP.
1807 */
amdgpu_ttm_reserve_tmr(struct amdgpu_device * adev)1808 static int amdgpu_ttm_reserve_tmr(struct amdgpu_device *adev)
1809 {
1810 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx;
1811 bool mem_train_support = false;
1812 uint32_t reserve_size = 0;
1813 int ret;
1814
1815 if (adev->bios && !amdgpu_sriov_vf(adev)) {
1816 if (amdgpu_atomfirmware_mem_training_supported(adev))
1817 mem_train_support = true;
1818 else
1819 DRM_DEBUG("memory training does not support!\n");
1820 }
1821
1822 /*
1823 * Query reserved tmr size through atom firmwareinfo for Sienna_Cichlid and onwards for all
1824 * the use cases (IP discovery/G6 memory training/profiling/diagnostic data.etc)
1825 *
1826 * Otherwise, fallback to legacy approach to check and reserve tmr block for ip
1827 * discovery data and G6 memory training data respectively
1828 */
1829 if (adev->bios)
1830 reserve_size =
1831 amdgpu_atomfirmware_get_fw_reserved_fb_size(adev);
1832
1833 if (!adev->bios &&
1834 (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
1835 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) ||
1836 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0)))
1837 reserve_size = max(reserve_size, (uint32_t)280 << 20);
1838 else if (!adev->bios &&
1839 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) {
1840 if (hweight32(adev->aid_mask) == 1)
1841 reserve_size = max(reserve_size, (uint32_t)128 << 20);
1842 else
1843 reserve_size = max(reserve_size, (uint32_t)144 << 20);
1844 } else if (!reserve_size)
1845 reserve_size = DISCOVERY_TMR_OFFSET;
1846
1847 if (mem_train_support) {
1848 /* reserve vram for mem train according to TMR location */
1849 amdgpu_ttm_training_data_block_init(adev, reserve_size);
1850 ret = amdgpu_bo_create_kernel_at(adev,
1851 ctx->c2p_train_data_offset,
1852 ctx->train_data_size,
1853 &ctx->c2p_bo,
1854 NULL);
1855 if (ret) {
1856 dev_err(adev->dev, "alloc c2p_bo failed(%d)!\n", ret);
1857 amdgpu_ttm_training_reserve_vram_fini(adev);
1858 return ret;
1859 }
1860 ctx->init = PSP_MEM_TRAIN_RESERVE_SUCCESS;
1861 }
1862
1863 ret = amdgpu_bo_create_kernel_at(
1864 adev, adev->gmc.real_vram_size - reserve_size, reserve_size,
1865 &adev->mman.fw_reserved_memory, NULL);
1866 if (ret) {
1867 dev_err(adev->dev, "alloc tmr failed(%d)!\n", ret);
1868 amdgpu_bo_free_kernel(&adev->mman.fw_reserved_memory, NULL,
1869 NULL);
1870 return ret;
1871 }
1872
1873 return 0;
1874 }
1875
amdgpu_ttm_pools_init(struct amdgpu_device * adev)1876 static int amdgpu_ttm_pools_init(struct amdgpu_device *adev)
1877 {
1878 int i;
1879
1880 if (!adev->gmc.is_app_apu || !adev->gmc.num_mem_partitions)
1881 return 0;
1882
1883 adev->mman.ttm_pools = kzalloc_objs(*adev->mman.ttm_pools,
1884 adev->gmc.num_mem_partitions);
1885 if (!adev->mman.ttm_pools)
1886 return -ENOMEM;
1887
1888 for (i = 0; i < adev->gmc.num_mem_partitions; i++) {
1889 ttm_pool_init(&adev->mman.ttm_pools[i], adev->dev,
1890 adev->gmc.mem_partitions[i].numa.node,
1891 TTM_ALLOCATION_POOL_BENEFICIAL_ORDER(get_order(SZ_2M)));
1892 }
1893 return 0;
1894 }
1895
amdgpu_ttm_pools_fini(struct amdgpu_device * adev)1896 static void amdgpu_ttm_pools_fini(struct amdgpu_device *adev)
1897 {
1898 int i;
1899
1900 if (!adev->gmc.is_app_apu || !adev->mman.ttm_pools)
1901 return;
1902
1903 for (i = 0; i < adev->gmc.num_mem_partitions; i++)
1904 ttm_pool_fini(&adev->mman.ttm_pools[i]);
1905
1906 kfree(adev->mman.ttm_pools);
1907 adev->mman.ttm_pools = NULL;
1908 }
1909
1910 /**
1911 * amdgpu_ttm_mmio_remap_bo_init - Allocate the singleton MMIO_REMAP BO
1912 * @adev: amdgpu device
1913 *
1914 * Allocates a global BO with backing AMDGPU_PL_MMIO_REMAP when the
1915 * hardware exposes a remap base (adev->rmmio_remap.bus_addr) and the host
1916 * PAGE_SIZE is <= AMDGPU_GPU_PAGE_SIZE (4K). The BO is created as a regular
1917 * GEM object (amdgpu_bo_create).
1918 *
1919 * Return:
1920 * * 0 on success or intentional skip (feature not present/unsupported)
1921 * * negative errno on allocation failure
1922 */
amdgpu_ttm_alloc_mmio_remap_bo(struct amdgpu_device * adev)1923 static int amdgpu_ttm_alloc_mmio_remap_bo(struct amdgpu_device *adev)
1924 {
1925 struct ttm_operation_ctx ctx = { false, false };
1926 struct ttm_placement placement;
1927 struct ttm_buffer_object *tbo;
1928 struct ttm_place placements;
1929 struct amdgpu_bo_param bp;
1930 struct ttm_resource *tmp;
1931 int r;
1932
1933 /* Skip if HW doesn't expose remap, or if PAGE_SIZE > AMDGPU_GPU_PAGE_SIZE (4K). */
1934 if (!adev->rmmio_remap.bus_addr || PAGE_SIZE > AMDGPU_GPU_PAGE_SIZE)
1935 return 0;
1936
1937 /*
1938 * Allocate a BO first and then move it to AMDGPU_PL_MMIO_REMAP.
1939 * The initial TTM resource assigned by amdgpu_bo_create() is
1940 * replaced below with a fixed MMIO_REMAP placement.
1941 */
1942 memset(&bp, 0, sizeof(bp));
1943 bp.type = ttm_bo_type_device;
1944 bp.size = AMDGPU_GPU_PAGE_SIZE;
1945 bp.byte_align = AMDGPU_GPU_PAGE_SIZE;
1946 bp.domain = 0;
1947 bp.flags = 0;
1948 bp.resv = NULL;
1949 bp.bo_ptr_size = sizeof(struct amdgpu_bo);
1950 r = amdgpu_bo_create(adev, &bp, &adev->rmmio_remap.bo);
1951 if (r)
1952 return r;
1953
1954 r = amdgpu_bo_reserve(adev->rmmio_remap.bo, true);
1955 if (r)
1956 goto err_unref;
1957
1958 tbo = &adev->rmmio_remap.bo->tbo;
1959
1960 /*
1961 * MMIO_REMAP is a fixed I/O placement (AMDGPU_PL_MMIO_REMAP).
1962 */
1963 placement.num_placement = 1;
1964 placement.placement = &placements;
1965 placements.fpfn = 0;
1966 placements.lpfn = 0;
1967 placements.mem_type = AMDGPU_PL_MMIO_REMAP;
1968 placements.flags = 0;
1969 /* Force the BO into the fixed MMIO_REMAP placement */
1970 r = ttm_bo_mem_space(tbo, &placement, &tmp, &ctx);
1971 if (unlikely(r))
1972 goto err_unlock;
1973
1974 ttm_resource_free(tbo, &tbo->resource);
1975 ttm_bo_assign_mem(tbo, tmp);
1976 ttm_bo_pin(tbo);
1977
1978 amdgpu_bo_unreserve(adev->rmmio_remap.bo);
1979 return 0;
1980
1981 err_unlock:
1982 amdgpu_bo_unreserve(adev->rmmio_remap.bo);
1983
1984 err_unref:
1985 amdgpu_bo_unref(&adev->rmmio_remap.bo);
1986 adev->rmmio_remap.bo = NULL;
1987 return r;
1988 }
1989
1990 /**
1991 * amdgpu_ttm_free_mmio_remap_bo - Free the singleton MMIO_REMAP BO
1992 * @adev: amdgpu device
1993 *
1994 * Frees the kernel-owned MMIO_REMAP BO if it was allocated by
1995 * amdgpu_ttm_mmio_remap_bo_init().
1996 */
amdgpu_ttm_free_mmio_remap_bo(struct amdgpu_device * adev)1997 static void amdgpu_ttm_free_mmio_remap_bo(struct amdgpu_device *adev)
1998 {
1999 if (!adev->rmmio_remap.bo)
2000 return;
2001
2002 if (!amdgpu_bo_reserve(adev->rmmio_remap.bo, true)) {
2003 ttm_bo_unpin(&adev->rmmio_remap.bo->tbo);
2004 amdgpu_bo_unreserve(adev->rmmio_remap.bo);
2005 }
2006
2007 /*
2008 * At this point we rely on normal DRM teardown ordering:
2009 * no new user ioctls can access the global MMIO_REMAP BO
2010 * once TTM teardown begins.
2011 */
2012 amdgpu_bo_unref(&adev->rmmio_remap.bo);
2013 adev->rmmio_remap.bo = NULL;
2014 }
2015
2016 /*
2017 * amdgpu_ttm_init - Init the memory management (ttm) as well as various
2018 * gtt/vram related fields.
2019 *
2020 * This initializes all of the memory space pools that the TTM layer
2021 * will need such as the GTT space (system memory mapped to the device),
2022 * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which
2023 * can be mapped per VMID.
2024 */
amdgpu_ttm_init(struct amdgpu_device * adev)2025 int amdgpu_ttm_init(struct amdgpu_device *adev)
2026 {
2027 uint64_t gtt_size;
2028 int r;
2029
2030 mutex_init(&adev->mman.gtt_window_lock);
2031
2032 dma_set_max_seg_size(adev->dev, UINT_MAX);
2033 /* No others user of address space so set it to 0 */
2034 r = ttm_device_init(&adev->mman.bdev, &amdgpu_bo_driver, adev->dev,
2035 adev_to_drm(adev)->anon_inode->i_mapping,
2036 adev_to_drm(adev)->vma_offset_manager,
2037 (adev->need_swiotlb ?
2038 TTM_ALLOCATION_POOL_USE_DMA_ALLOC : 0) |
2039 (dma_addressing_limited(adev->dev) ?
2040 TTM_ALLOCATION_POOL_USE_DMA32 : 0) |
2041 TTM_ALLOCATION_POOL_BENEFICIAL_ORDER(get_order(SZ_2M)));
2042 if (r) {
2043 dev_err(adev->dev,
2044 "failed initializing buffer object driver(%d).\n", r);
2045 return r;
2046 }
2047
2048 r = amdgpu_ttm_pools_init(adev);
2049 if (r) {
2050 dev_err(adev->dev, "failed to init ttm pools(%d).\n", r);
2051 return r;
2052 }
2053 adev->mman.initialized = true;
2054
2055 if (!adev->gmc.is_app_apu) {
2056 /* Initialize VRAM pool with all of VRAM divided into pages */
2057 r = amdgpu_vram_mgr_init(adev);
2058 if (r) {
2059 dev_err(adev->dev, "Failed initializing VRAM heap.\n");
2060 return r;
2061 }
2062 }
2063
2064 /* Change the size here instead of the init above so only lpfn is affected */
2065 amdgpu_ttm_set_buffer_funcs_status(adev, false);
2066 #ifdef CONFIG_64BIT
2067 #ifdef CONFIG_X86
2068 if (adev->gmc.xgmi.connected_to_cpu)
2069 adev->mman.aper_base_kaddr = ioremap_cache(adev->gmc.aper_base,
2070 adev->gmc.visible_vram_size);
2071
2072 else if (adev->gmc.is_app_apu)
2073 DRM_DEBUG_DRIVER(
2074 "No need to ioremap when real vram size is 0\n");
2075 else
2076 #endif
2077 adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base,
2078 adev->gmc.visible_vram_size);
2079 #endif
2080
2081 /*
2082 *The reserved vram for firmware must be pinned to the specified
2083 *place on the VRAM, so reserve it early.
2084 */
2085 r = amdgpu_ttm_fw_reserve_vram_init(adev);
2086 if (r)
2087 return r;
2088
2089 /*
2090 * The reserved VRAM for the driver must be pinned to a specific
2091 * location in VRAM, so reserve it early.
2092 */
2093 r = amdgpu_ttm_drv_reserve_vram_init(adev);
2094 if (r)
2095 return r;
2096
2097 /*
2098 * only NAVI10 and later ASICs support IP discovery.
2099 * If IP discovery is enabled, a block of memory should be
2100 * reserved for it.
2101 */
2102 if (adev->discovery.reserve_tmr) {
2103 r = amdgpu_ttm_reserve_tmr(adev);
2104 if (r)
2105 return r;
2106 }
2107
2108 /* allocate memory as required for VGA
2109 * This is used for VGA emulation and pre-OS scanout buffers to
2110 * avoid display artifacts while transitioning between pre-OS
2111 * and driver.
2112 */
2113 if (!adev->gmc.is_app_apu) {
2114 r = amdgpu_bo_create_kernel_at(adev, 0,
2115 adev->mman.stolen_vga_size,
2116 &adev->mman.stolen_vga_memory,
2117 NULL);
2118 if (r)
2119 return r;
2120
2121 r = amdgpu_bo_create_kernel_at(adev, adev->mman.stolen_vga_size,
2122 adev->mman.stolen_extended_size,
2123 &adev->mman.stolen_extended_memory,
2124 NULL);
2125
2126 if (r)
2127 return r;
2128
2129 r = amdgpu_bo_create_kernel_at(adev,
2130 adev->mman.stolen_reserved_offset,
2131 adev->mman.stolen_reserved_size,
2132 &adev->mman.stolen_reserved_memory,
2133 NULL);
2134 if (r)
2135 return r;
2136 } else {
2137 DRM_DEBUG_DRIVER("Skipped stolen memory reservation\n");
2138 }
2139
2140 dev_info(adev->dev, " %uM of VRAM memory ready\n",
2141 (unsigned int)(adev->gmc.real_vram_size / (1024 * 1024)));
2142
2143 /* Compute GTT size, either based on TTM limit
2144 * or whatever the user passed on module init.
2145 */
2146 gtt_size = ttm_tt_pages_limit() << PAGE_SHIFT;
2147 if (amdgpu_gtt_size != -1) {
2148 uint64_t configured_size = (uint64_t)amdgpu_gtt_size << 20;
2149
2150 drm_warn(&adev->ddev,
2151 "Configuring gttsize via module parameter is deprecated, please use ttm.pages_limit\n");
2152 if (gtt_size != configured_size)
2153 drm_warn(&adev->ddev,
2154 "GTT size has been set as %llu but TTM size has been set as %llu, this is unusual\n",
2155 configured_size, gtt_size);
2156
2157 gtt_size = configured_size;
2158 }
2159
2160 /* Initialize GTT memory pool */
2161 r = amdgpu_gtt_mgr_init(adev, gtt_size);
2162 if (r) {
2163 dev_err(adev->dev, "Failed initializing GTT heap.\n");
2164 return r;
2165 }
2166 dev_info(adev->dev, " %uM of GTT memory ready.\n",
2167 (unsigned int)(gtt_size / (1024 * 1024)));
2168
2169 if (adev->flags & AMD_IS_APU) {
2170 if (adev->gmc.real_vram_size < gtt_size)
2171 adev->apu_prefer_gtt = true;
2172 }
2173
2174 /* Initialize doorbell pool on PCI BAR */
2175 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_DOORBELL, adev->doorbell.size / PAGE_SIZE);
2176 if (r) {
2177 dev_err(adev->dev, "Failed initializing doorbell heap.\n");
2178 return r;
2179 }
2180
2181 /* Create a boorbell page for kernel usages */
2182 r = amdgpu_doorbell_create_kernel_doorbells(adev);
2183 if (r) {
2184 dev_err(adev->dev, "Failed to initialize kernel doorbells.\n");
2185 return r;
2186 }
2187
2188 /* Initialize MMIO-remap pool (single page 4K) */
2189 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_MMIO_REMAP, 1);
2190 if (r) {
2191 dev_err(adev->dev, "Failed initializing MMIO-remap heap.\n");
2192 return r;
2193 }
2194
2195 /* Allocate the singleton MMIO_REMAP BO if supported */
2196 r = amdgpu_ttm_alloc_mmio_remap_bo(adev);
2197 if (r)
2198 return r;
2199
2200 /* Initialize preemptible memory pool */
2201 r = amdgpu_preempt_mgr_init(adev);
2202 if (r) {
2203 dev_err(adev->dev, "Failed initializing PREEMPT heap.\n");
2204 return r;
2205 }
2206
2207 /* Initialize various on-chip memory pools */
2208 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GDS, adev->gds.gds_size);
2209 if (r) {
2210 dev_err(adev->dev, "Failed initializing GDS heap.\n");
2211 return r;
2212 }
2213
2214 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GWS, adev->gds.gws_size);
2215 if (r) {
2216 dev_err(adev->dev, "Failed initializing gws heap.\n");
2217 return r;
2218 }
2219
2220 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_OA, adev->gds.oa_size);
2221 if (r) {
2222 dev_err(adev->dev, "Failed initializing oa heap.\n");
2223 return r;
2224 }
2225 if (amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE,
2226 AMDGPU_GEM_DOMAIN_GTT,
2227 &adev->mman.sdma_access_bo, NULL,
2228 &adev->mman.sdma_access_ptr))
2229 drm_warn(adev_to_drm(adev),
2230 "Debug VRAM access will use slowpath MM access\n");
2231
2232 return 0;
2233 }
2234
2235 /*
2236 * amdgpu_ttm_fini - De-initialize the TTM memory pools
2237 */
amdgpu_ttm_fini(struct amdgpu_device * adev)2238 void amdgpu_ttm_fini(struct amdgpu_device *adev)
2239 {
2240 int idx;
2241
2242 if (!adev->mman.initialized)
2243 return;
2244
2245 amdgpu_ttm_pools_fini(adev);
2246
2247 amdgpu_ttm_training_reserve_vram_fini(adev);
2248 /* return the stolen vga memory back to VRAM */
2249 if (!adev->gmc.is_app_apu) {
2250 amdgpu_bo_free_kernel(&adev->mman.stolen_vga_memory, NULL, NULL);
2251 amdgpu_bo_free_kernel(&adev->mman.stolen_extended_memory, NULL, NULL);
2252 /* return the FW reserved memory back to VRAM */
2253 amdgpu_bo_free_kernel(&adev->mman.fw_reserved_memory, NULL,
2254 NULL);
2255 amdgpu_bo_free_kernel(&adev->mman.fw_reserved_memory_extend, NULL,
2256 NULL);
2257 if (adev->mman.stolen_reserved_size)
2258 amdgpu_bo_free_kernel(&adev->mman.stolen_reserved_memory,
2259 NULL, NULL);
2260 }
2261 amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
2262 &adev->mman.sdma_access_ptr);
2263
2264 amdgpu_ttm_free_mmio_remap_bo(adev);
2265 amdgpu_ttm_fw_reserve_vram_fini(adev);
2266 amdgpu_ttm_drv_reserve_vram_fini(adev);
2267
2268 if (drm_dev_enter(adev_to_drm(adev), &idx)) {
2269
2270 if (adev->mman.aper_base_kaddr)
2271 iounmap(adev->mman.aper_base_kaddr);
2272 adev->mman.aper_base_kaddr = NULL;
2273
2274 drm_dev_exit(idx);
2275 }
2276
2277 if (!adev->gmc.is_app_apu)
2278 amdgpu_vram_mgr_fini(adev);
2279 amdgpu_gtt_mgr_fini(adev);
2280 amdgpu_preempt_mgr_fini(adev);
2281 amdgpu_doorbell_fini(adev);
2282
2283 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS);
2284 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GWS);
2285 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_OA);
2286 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_DOORBELL);
2287 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_MMIO_REMAP);
2288 ttm_device_fini(&adev->mman.bdev);
2289 adev->mman.initialized = false;
2290 dev_info(adev->dev, " ttm finalized\n");
2291 }
2292
2293 /**
2294 * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions
2295 *
2296 * @adev: amdgpu_device pointer
2297 * @enable: true when we can use buffer functions.
2298 *
2299 * Enable/disable use of buffer functions during suspend/resume. This should
2300 * only be called at bootup or when userspace isn't running.
2301 */
amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device * adev,bool enable)2302 void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
2303 {
2304 struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
2305 uint64_t size;
2306 int r;
2307
2308 if (!adev->mman.initialized || amdgpu_in_reset(adev) ||
2309 adev->mman.buffer_funcs_enabled == enable || adev->gmc.is_app_apu)
2310 return;
2311
2312 if (enable) {
2313 struct amdgpu_ring *ring;
2314 struct drm_gpu_scheduler *sched;
2315
2316 ring = adev->mman.buffer_funcs_ring;
2317 sched = &ring->sched;
2318 r = drm_sched_entity_init(&adev->mman.default_entity.base,
2319 DRM_SCHED_PRIORITY_KERNEL, &sched,
2320 1, NULL);
2321 if (r) {
2322 dev_err(adev->dev,
2323 "Failed setting up TTM BO move entity (%d)\n",
2324 r);
2325 return;
2326 }
2327
2328 r = drm_sched_entity_init(&adev->mman.clear_entity.base,
2329 DRM_SCHED_PRIORITY_NORMAL, &sched,
2330 1, NULL);
2331 if (r) {
2332 dev_err(adev->dev,
2333 "Failed setting up TTM BO clear entity (%d)\n",
2334 r);
2335 goto error_free_entity;
2336 }
2337
2338 r = drm_sched_entity_init(&adev->mman.move_entity.base,
2339 DRM_SCHED_PRIORITY_NORMAL, &sched,
2340 1, NULL);
2341 if (r) {
2342 dev_err(adev->dev,
2343 "Failed setting up TTM BO move entity (%d)\n",
2344 r);
2345 drm_sched_entity_destroy(&adev->mman.clear_entity.base);
2346 goto error_free_entity;
2347 }
2348 } else {
2349 drm_sched_entity_destroy(&adev->mman.default_entity.base);
2350 drm_sched_entity_destroy(&adev->mman.clear_entity.base);
2351 drm_sched_entity_destroy(&adev->mman.move_entity.base);
2352 /* Drop all the old fences since re-creating the scheduler entities
2353 * will allocate new contexts.
2354 */
2355 ttm_resource_manager_cleanup(man);
2356 }
2357
2358 /* this just adjusts TTM size idea, which sets lpfn to the correct value */
2359 if (enable)
2360 size = adev->gmc.real_vram_size;
2361 else
2362 size = adev->gmc.visible_vram_size;
2363 man->size = size;
2364 adev->mman.buffer_funcs_enabled = enable;
2365
2366 return;
2367
2368 error_free_entity:
2369 drm_sched_entity_destroy(&adev->mman.default_entity.base);
2370 }
2371
amdgpu_ttm_prepare_job(struct amdgpu_device * adev,struct amdgpu_ttm_buffer_entity * entity,unsigned int num_dw,struct dma_resv * resv,bool vm_needs_flush,struct amdgpu_job ** job,u64 k_job_id)2372 static int amdgpu_ttm_prepare_job(struct amdgpu_device *adev,
2373 struct amdgpu_ttm_buffer_entity *entity,
2374 unsigned int num_dw,
2375 struct dma_resv *resv,
2376 bool vm_needs_flush,
2377 struct amdgpu_job **job,
2378 u64 k_job_id)
2379 {
2380 enum amdgpu_ib_pool_type pool = AMDGPU_IB_POOL_DELAYED;
2381 int r;
2382 r = amdgpu_job_alloc_with_ib(adev, &entity->base,
2383 AMDGPU_FENCE_OWNER_UNDEFINED,
2384 num_dw * 4, pool, job, k_job_id);
2385 if (r)
2386 return r;
2387
2388 if (vm_needs_flush) {
2389 (*job)->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo ?
2390 adev->gmc.pdb0_bo :
2391 adev->gart.bo);
2392 (*job)->vm_needs_flush = true;
2393 }
2394 if (!resv)
2395 return 0;
2396
2397 return drm_sched_job_add_resv_dependencies(&(*job)->base, resv,
2398 DMA_RESV_USAGE_BOOKKEEP);
2399 }
2400
amdgpu_copy_buffer(struct amdgpu_device * adev,struct amdgpu_ttm_buffer_entity * entity,uint64_t src_offset,uint64_t dst_offset,uint32_t byte_count,struct dma_resv * resv,struct dma_fence ** fence,bool vm_needs_flush,uint32_t copy_flags)2401 int amdgpu_copy_buffer(struct amdgpu_device *adev,
2402 struct amdgpu_ttm_buffer_entity *entity,
2403 uint64_t src_offset,
2404 uint64_t dst_offset, uint32_t byte_count,
2405 struct dma_resv *resv,
2406 struct dma_fence **fence,
2407 bool vm_needs_flush, uint32_t copy_flags)
2408 {
2409 unsigned int num_loops, num_dw;
2410 struct amdgpu_ring *ring;
2411 struct amdgpu_job *job;
2412 uint32_t max_bytes;
2413 unsigned int i;
2414 int r;
2415
2416 ring = adev->mman.buffer_funcs_ring;
2417
2418 if (!ring->sched.ready) {
2419 dev_err(adev->dev,
2420 "Trying to move memory with ring turned off.\n");
2421 return -EINVAL;
2422 }
2423
2424 max_bytes = adev->mman.buffer_funcs->copy_max_bytes;
2425 num_loops = DIV_ROUND_UP(byte_count, max_bytes);
2426 num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->copy_num_dw, 8);
2427 r = amdgpu_ttm_prepare_job(adev, entity, num_dw,
2428 resv, vm_needs_flush, &job,
2429 AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER);
2430 if (r)
2431 goto error_free;
2432
2433 for (i = 0; i < num_loops; i++) {
2434 uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
2435
2436 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset,
2437 dst_offset, cur_size_in_bytes, copy_flags);
2438 src_offset += cur_size_in_bytes;
2439 dst_offset += cur_size_in_bytes;
2440 byte_count -= cur_size_in_bytes;
2441 }
2442
2443 *fence = amdgpu_ttm_job_submit(adev, job, num_dw);
2444
2445 return 0;
2446
2447 error_free:
2448 amdgpu_job_free(job);
2449 dev_err(adev->dev, "Error scheduling IBs (%d)\n", r);
2450 return r;
2451 }
2452
amdgpu_ttm_fill_mem(struct amdgpu_device * adev,struct amdgpu_ttm_buffer_entity * entity,uint32_t src_data,uint64_t dst_addr,uint32_t byte_count,struct dma_resv * resv,struct dma_fence ** fence,bool vm_needs_flush,u64 k_job_id)2453 static int amdgpu_ttm_fill_mem(struct amdgpu_device *adev,
2454 struct amdgpu_ttm_buffer_entity *entity,
2455 uint32_t src_data,
2456 uint64_t dst_addr, uint32_t byte_count,
2457 struct dma_resv *resv,
2458 struct dma_fence **fence,
2459 bool vm_needs_flush,
2460 u64 k_job_id)
2461 {
2462 unsigned int num_loops, num_dw;
2463 struct amdgpu_job *job;
2464 uint32_t max_bytes;
2465 unsigned int i;
2466 int r;
2467
2468 max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
2469 num_loops = DIV_ROUND_UP_ULL(byte_count, max_bytes);
2470 num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->fill_num_dw, 8);
2471 r = amdgpu_ttm_prepare_job(adev, entity, num_dw, resv,
2472 vm_needs_flush, &job, k_job_id);
2473 if (r)
2474 return r;
2475
2476 for (i = 0; i < num_loops; i++) {
2477 uint32_t cur_size = min(byte_count, max_bytes);
2478
2479 amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data, dst_addr,
2480 cur_size);
2481
2482 dst_addr += cur_size;
2483 byte_count -= cur_size;
2484 }
2485
2486 *fence = amdgpu_ttm_job_submit(adev, job, num_dw);
2487 return 0;
2488 }
2489
2490 /**
2491 * amdgpu_ttm_clear_buffer - clear memory buffers
2492 * @bo: amdgpu buffer object
2493 * @resv: reservation object
2494 * @fence: dma_fence associated with the operation
2495 *
2496 * Clear the memory buffer resource.
2497 *
2498 * Returns:
2499 * 0 for success or a negative error code on failure.
2500 */
amdgpu_ttm_clear_buffer(struct amdgpu_bo * bo,struct dma_resv * resv,struct dma_fence ** fence)2501 int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo,
2502 struct dma_resv *resv,
2503 struct dma_fence **fence)
2504 {
2505 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
2506 struct amdgpu_res_cursor cursor;
2507 u64 addr;
2508 int r = 0;
2509
2510 if (!adev->mman.buffer_funcs_enabled)
2511 return -EINVAL;
2512
2513 if (!fence)
2514 return -EINVAL;
2515
2516 *fence = dma_fence_get_stub();
2517
2518 amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &cursor);
2519
2520 mutex_lock(&adev->mman.gtt_window_lock);
2521 while (cursor.remaining) {
2522 struct dma_fence *next = NULL;
2523 u64 size;
2524
2525 if (amdgpu_res_cleared(&cursor)) {
2526 amdgpu_res_next(&cursor, cursor.size);
2527 continue;
2528 }
2529
2530 /* Never clear more than 256MiB at once to avoid timeouts */
2531 size = min(cursor.size, 256ULL << 20);
2532
2533 r = amdgpu_ttm_map_buffer(&adev->mman.clear_entity,
2534 &bo->tbo, bo->tbo.resource, &cursor,
2535 1, false, &size, &addr);
2536 if (r)
2537 goto err;
2538
2539 r = amdgpu_ttm_fill_mem(adev, &adev->mman.clear_entity, 0, addr, size, resv,
2540 &next, true,
2541 AMDGPU_KERNEL_JOB_ID_TTM_CLEAR_BUFFER);
2542 if (r)
2543 goto err;
2544
2545 dma_fence_put(*fence);
2546 *fence = next;
2547
2548 amdgpu_res_next(&cursor, size);
2549 }
2550 err:
2551 mutex_unlock(&adev->mman.gtt_window_lock);
2552
2553 return r;
2554 }
2555
amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity * entity,struct amdgpu_bo * bo,uint32_t src_data,struct dma_resv * resv,struct dma_fence ** f,u64 k_job_id)2556 int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity,
2557 struct amdgpu_bo *bo,
2558 uint32_t src_data,
2559 struct dma_resv *resv,
2560 struct dma_fence **f,
2561 u64 k_job_id)
2562 {
2563 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
2564 struct dma_fence *fence = NULL;
2565 struct amdgpu_res_cursor dst;
2566 int r;
2567
2568 if (!adev->mman.buffer_funcs_enabled) {
2569 dev_err(adev->dev,
2570 "Trying to clear memory with ring turned off.\n");
2571 return -EINVAL;
2572 }
2573
2574 amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &dst);
2575
2576 mutex_lock(&adev->mman.gtt_window_lock);
2577 while (dst.remaining) {
2578 struct dma_fence *next;
2579 uint64_t cur_size, to;
2580
2581 /* Never fill more than 256MiB at once to avoid timeouts */
2582 cur_size = min(dst.size, 256ULL << 20);
2583
2584 r = amdgpu_ttm_map_buffer(entity, &bo->tbo, bo->tbo.resource, &dst,
2585 1, false, &cur_size, &to);
2586 if (r)
2587 goto error;
2588
2589 r = amdgpu_ttm_fill_mem(adev, entity,
2590 src_data, to, cur_size, resv,
2591 &next, true, k_job_id);
2592 if (r)
2593 goto error;
2594
2595 dma_fence_put(fence);
2596 fence = next;
2597
2598 amdgpu_res_next(&dst, cur_size);
2599 }
2600 error:
2601 mutex_unlock(&adev->mman.gtt_window_lock);
2602 if (f)
2603 *f = dma_fence_get(fence);
2604 dma_fence_put(fence);
2605 return r;
2606 }
2607
2608 /**
2609 * amdgpu_ttm_evict_resources - evict memory buffers
2610 * @adev: amdgpu device object
2611 * @mem_type: evicted BO's memory type
2612 *
2613 * Evicts all @mem_type buffers on the lru list of the memory type.
2614 *
2615 * Returns:
2616 * 0 for success or a negative error code on failure.
2617 */
amdgpu_ttm_evict_resources(struct amdgpu_device * adev,int mem_type)2618 int amdgpu_ttm_evict_resources(struct amdgpu_device *adev, int mem_type)
2619 {
2620 struct ttm_resource_manager *man;
2621
2622 switch (mem_type) {
2623 case TTM_PL_VRAM:
2624 case TTM_PL_TT:
2625 case AMDGPU_PL_GWS:
2626 case AMDGPU_PL_GDS:
2627 case AMDGPU_PL_OA:
2628 man = ttm_manager_type(&adev->mman.bdev, mem_type);
2629 break;
2630 default:
2631 dev_err(adev->dev, "Trying to evict invalid memory type\n");
2632 return -EINVAL;
2633 }
2634
2635 return ttm_resource_manager_evict_all(&adev->mman.bdev, man);
2636 }
2637
2638 #if defined(CONFIG_DEBUG_FS)
2639
amdgpu_ttm_page_pool_show(struct seq_file * m,void * unused)2640 static int amdgpu_ttm_page_pool_show(struct seq_file *m, void *unused)
2641 {
2642 struct amdgpu_device *adev = m->private;
2643
2644 return ttm_pool_debugfs(&adev->mman.bdev.pool, m);
2645 }
2646
2647 DEFINE_SHOW_ATTRIBUTE(amdgpu_ttm_page_pool);
2648
2649 /*
2650 * amdgpu_ttm_vram_read - Linear read access to VRAM
2651 *
2652 * Accesses VRAM via MMIO for debugging purposes.
2653 */
amdgpu_ttm_vram_read(struct file * f,char __user * buf,size_t size,loff_t * pos)2654 static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
2655 size_t size, loff_t *pos)
2656 {
2657 struct amdgpu_device *adev = file_inode(f)->i_private;
2658 ssize_t result = 0;
2659
2660 if (size & 0x3 || *pos & 0x3)
2661 return -EINVAL;
2662
2663 if (*pos >= adev->gmc.mc_vram_size)
2664 return -ENXIO;
2665
2666 size = min(size, (size_t)(adev->gmc.mc_vram_size - *pos));
2667 while (size) {
2668 size_t bytes = min(size, AMDGPU_TTM_VRAM_MAX_DW_READ * 4);
2669 uint32_t value[AMDGPU_TTM_VRAM_MAX_DW_READ];
2670
2671 amdgpu_device_vram_access(adev, *pos, value, bytes, false);
2672 if (copy_to_user(buf, value, bytes))
2673 return -EFAULT;
2674
2675 result += bytes;
2676 buf += bytes;
2677 *pos += bytes;
2678 size -= bytes;
2679 }
2680
2681 return result;
2682 }
2683
2684 /*
2685 * amdgpu_ttm_vram_write - Linear write access to VRAM
2686 *
2687 * Accesses VRAM via MMIO for debugging purposes.
2688 */
amdgpu_ttm_vram_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)2689 static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf,
2690 size_t size, loff_t *pos)
2691 {
2692 struct amdgpu_device *adev = file_inode(f)->i_private;
2693 ssize_t result = 0;
2694 int r;
2695
2696 if (size & 0x3 || *pos & 0x3)
2697 return -EINVAL;
2698
2699 if (*pos >= adev->gmc.mc_vram_size)
2700 return -ENXIO;
2701
2702 while (size) {
2703 uint32_t value;
2704
2705 if (*pos >= adev->gmc.mc_vram_size)
2706 return result;
2707
2708 r = get_user(value, (uint32_t *)buf);
2709 if (r)
2710 return r;
2711
2712 amdgpu_device_mm_access(adev, *pos, &value, 4, true);
2713
2714 result += 4;
2715 buf += 4;
2716 *pos += 4;
2717 size -= 4;
2718 }
2719
2720 return result;
2721 }
2722
2723 static const struct file_operations amdgpu_ttm_vram_fops = {
2724 .owner = THIS_MODULE,
2725 .read = amdgpu_ttm_vram_read,
2726 .write = amdgpu_ttm_vram_write,
2727 .llseek = default_llseek,
2728 };
2729
2730 /*
2731 * amdgpu_iomem_read - Virtual read access to GPU mapped memory
2732 *
2733 * This function is used to read memory that has been mapped to the
2734 * GPU and the known addresses are not physical addresses but instead
2735 * bus addresses (e.g., what you'd put in an IB or ring buffer).
2736 */
amdgpu_iomem_read(struct file * f,char __user * buf,size_t size,loff_t * pos)2737 static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf,
2738 size_t size, loff_t *pos)
2739 {
2740 struct amdgpu_device *adev = file_inode(f)->i_private;
2741 struct iommu_domain *dom;
2742 ssize_t result = 0;
2743 int r;
2744
2745 /* retrieve the IOMMU domain if any for this device */
2746 dom = iommu_get_domain_for_dev(adev->dev);
2747
2748 while (size) {
2749 phys_addr_t addr = *pos & PAGE_MASK;
2750 loff_t off = *pos & ~PAGE_MASK;
2751 size_t bytes = PAGE_SIZE - off;
2752 unsigned long pfn;
2753 struct page *p;
2754 void *ptr;
2755
2756 bytes = min(bytes, size);
2757
2758 /* Translate the bus address to a physical address. If
2759 * the domain is NULL it means there is no IOMMU active
2760 * and the address translation is the identity
2761 */
2762 addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
2763
2764 pfn = addr >> PAGE_SHIFT;
2765 if (!pfn_valid(pfn))
2766 return -EPERM;
2767
2768 p = pfn_to_page(pfn);
2769 if (p->mapping != adev->mman.bdev.dev_mapping)
2770 return -EPERM;
2771
2772 ptr = kmap_local_page(p);
2773 r = copy_to_user(buf, ptr + off, bytes);
2774 kunmap_local(ptr);
2775 if (r)
2776 return -EFAULT;
2777
2778 size -= bytes;
2779 *pos += bytes;
2780 result += bytes;
2781 }
2782
2783 return result;
2784 }
2785
2786 /*
2787 * amdgpu_iomem_write - Virtual write access to GPU mapped memory
2788 *
2789 * This function is used to write memory that has been mapped to the
2790 * GPU and the known addresses are not physical addresses but instead
2791 * bus addresses (e.g., what you'd put in an IB or ring buffer).
2792 */
amdgpu_iomem_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)2793 static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf,
2794 size_t size, loff_t *pos)
2795 {
2796 struct amdgpu_device *adev = file_inode(f)->i_private;
2797 struct iommu_domain *dom;
2798 ssize_t result = 0;
2799 int r;
2800
2801 dom = iommu_get_domain_for_dev(adev->dev);
2802
2803 while (size) {
2804 phys_addr_t addr = *pos & PAGE_MASK;
2805 loff_t off = *pos & ~PAGE_MASK;
2806 size_t bytes = PAGE_SIZE - off;
2807 unsigned long pfn;
2808 struct page *p;
2809 void *ptr;
2810
2811 bytes = min(bytes, size);
2812
2813 addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
2814
2815 pfn = addr >> PAGE_SHIFT;
2816 if (!pfn_valid(pfn))
2817 return -EPERM;
2818
2819 p = pfn_to_page(pfn);
2820 if (p->mapping != adev->mman.bdev.dev_mapping)
2821 return -EPERM;
2822
2823 ptr = kmap_local_page(p);
2824 r = copy_from_user(ptr + off, buf, bytes);
2825 kunmap_local(ptr);
2826 if (r)
2827 return -EFAULT;
2828
2829 size -= bytes;
2830 *pos += bytes;
2831 result += bytes;
2832 }
2833
2834 return result;
2835 }
2836
2837 static const struct file_operations amdgpu_ttm_iomem_fops = {
2838 .owner = THIS_MODULE,
2839 .read = amdgpu_iomem_read,
2840 .write = amdgpu_iomem_write,
2841 .llseek = default_llseek
2842 };
2843
2844 #endif
2845
amdgpu_ttm_debugfs_init(struct amdgpu_device * adev)2846 void amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
2847 {
2848 #if defined(CONFIG_DEBUG_FS)
2849 struct drm_minor *minor = adev_to_drm(adev)->primary;
2850 struct dentry *root = minor->debugfs_root;
2851
2852 debugfs_create_file_size("amdgpu_vram", 0444, root, adev,
2853 &amdgpu_ttm_vram_fops, adev->gmc.mc_vram_size);
2854 debugfs_create_file("amdgpu_iomem", 0444, root, adev,
2855 &amdgpu_ttm_iomem_fops);
2856 debugfs_create_file("ttm_page_pool", 0444, root, adev,
2857 &amdgpu_ttm_page_pool_fops);
2858 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,
2859 TTM_PL_VRAM),
2860 root, "amdgpu_vram_mm");
2861 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,
2862 TTM_PL_TT),
2863 root, "amdgpu_gtt_mm");
2864 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,
2865 AMDGPU_PL_GDS),
2866 root, "amdgpu_gds_mm");
2867 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,
2868 AMDGPU_PL_GWS),
2869 root, "amdgpu_gws_mm");
2870 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,
2871 AMDGPU_PL_OA),
2872 root, "amdgpu_oa_mm");
2873
2874 #endif
2875 }
2876