Lines Matching +full:gfx +full:- +full:mem
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
33 #include <linux/dma-mapping.h>
41 #include <linux/dma-buf.h>
78 return ttm_range_man_init(&adev->mman.bdev, type, in amdgpu_ttm_init_on_chip()
83 * amdgpu_evict_flags - Compute placement flags
93 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_evict_flags()
103 if (bo->type == ttm_bo_type_sg) { in amdgpu_evict_flags()
104 placement->num_placement = 0; in amdgpu_evict_flags()
110 placement->placement = &placements; in amdgpu_evict_flags()
111 placement->num_placement = 1; in amdgpu_evict_flags()
116 if (abo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) { in amdgpu_evict_flags()
117 placement->num_placement = 0; in amdgpu_evict_flags()
121 switch (bo->resource->mem_type) { in amdgpu_evict_flags()
127 placement->num_placement = 0; in amdgpu_evict_flags()
131 if (!adev->mman.buffer_funcs_enabled) { in amdgpu_evict_flags()
135 } else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && in amdgpu_evict_flags()
136 !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) && in amdgpu_evict_flags()
137 amdgpu_res_cpu_visible(adev, bo->resource)) { in amdgpu_evict_flags()
147 abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; in amdgpu_evict_flags()
148 abo->placements[0].lpfn = 0; in amdgpu_evict_flags()
149 abo->placements[0].flags |= TTM_PL_FLAG_DESIRED; in amdgpu_evict_flags()
162 *placement = abo->placement; in amdgpu_evict_flags()
166 * amdgpu_ttm_map_buffer - Map memory into the GART windows
168 * @mem: memory object to map
180 struct ttm_resource *mem, in amdgpu_ttm_map_buffer() argument
185 struct amdgpu_device *adev = ring->adev; in amdgpu_ttm_map_buffer()
194 BUG_ON(adev->mman.buffer_funcs->copy_max_bytes < in amdgpu_ttm_map_buffer()
197 if (WARN_ON(mem->mem_type == AMDGPU_PL_PREEMPT)) in amdgpu_ttm_map_buffer()
198 return -EINVAL; in amdgpu_ttm_map_buffer()
201 if (!tmz && mem->start != AMDGPU_BO_INVALID_OFFSET) { in amdgpu_ttm_map_buffer()
202 *addr = amdgpu_ttm_domain_start(adev, mem->mem_type) + in amdgpu_ttm_map_buffer()
203 mm_cur->start; in amdgpu_ttm_map_buffer()
212 offset = mm_cur->start & ~PAGE_MASK; in amdgpu_ttm_map_buffer()
217 *size = min(*size, (uint64_t)num_pages * PAGE_SIZE - offset); in amdgpu_ttm_map_buffer()
219 *addr = adev->gmc.gart_start; in amdgpu_ttm_map_buffer()
224 num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); in amdgpu_ttm_map_buffer()
227 r = amdgpu_job_alloc_with_ib(adev, &adev->mman.high_pr, in amdgpu_ttm_map_buffer()
236 src_addr += job->ibs[0].gpu_addr; in amdgpu_ttm_map_buffer()
238 dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo); in amdgpu_ttm_map_buffer()
240 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, in amdgpu_ttm_map_buffer()
243 amdgpu_ring_pad_ib(ring, &job->ibs[0]); in amdgpu_ttm_map_buffer()
244 WARN_ON(job->ibs[0].length_dw > num_dw); in amdgpu_ttm_map_buffer()
246 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, mem); in amdgpu_ttm_map_buffer()
250 cpu_addr = &job->ibs[0].ptr[num_dw]; in amdgpu_ttm_map_buffer()
252 if (mem->mem_type == TTM_PL_TT) { in amdgpu_ttm_map_buffer()
255 dma_addr = &bo->ttm->dma_address[mm_cur->start >> PAGE_SHIFT]; in amdgpu_ttm_map_buffer()
260 dma_address = mm_cur->start; in amdgpu_ttm_map_buffer()
261 dma_address += adev->vm_manager.vram_base_offset; in amdgpu_ttm_map_buffer()
275 * amdgpu_ttm_copy_mem_to_mem - Helper function for copy
284 * The function copies @size bytes from {src->mem + src->offset} to
285 * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a
296 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; in amdgpu_ttm_copy_mem_to_mem()
303 if (!adev->mman.buffer_funcs_enabled) { in amdgpu_ttm_copy_mem_to_mem()
304 dev_err(adev->dev, in amdgpu_ttm_copy_mem_to_mem()
306 return -EINVAL; in amdgpu_ttm_copy_mem_to_mem()
309 amdgpu_res_first(src->mem, src->offset, size, &src_mm); in amdgpu_ttm_copy_mem_to_mem()
310 amdgpu_res_first(dst->mem, dst->offset, size, &dst_mm); in amdgpu_ttm_copy_mem_to_mem()
312 mutex_lock(&adev->mman.gtt_window_lock); in amdgpu_ttm_copy_mem_to_mem()
322 r = amdgpu_ttm_map_buffer(src->bo, src->mem, &src_mm, in amdgpu_ttm_copy_mem_to_mem()
327 r = amdgpu_ttm_map_buffer(dst->bo, dst->mem, &dst_mm, in amdgpu_ttm_copy_mem_to_mem()
332 abo_src = ttm_to_amdgpu_bo(src->bo); in amdgpu_ttm_copy_mem_to_mem()
333 abo_dst = ttm_to_amdgpu_bo(dst->bo); in amdgpu_ttm_copy_mem_to_mem()
336 if ((abo_src->flags & AMDGPU_GEM_CREATE_GFX12_DCC) && in amdgpu_ttm_copy_mem_to_mem()
337 (abo_src->tbo.resource->mem_type == TTM_PL_VRAM)) in amdgpu_ttm_copy_mem_to_mem()
339 if ((abo_dst->flags & AMDGPU_GEM_CREATE_GFX12_DCC) && in amdgpu_ttm_copy_mem_to_mem()
340 (dst->mem->mem_type == TTM_PL_VRAM)) { in amdgpu_ttm_copy_mem_to_mem()
367 mutex_unlock(&adev->mman.gtt_window_lock); in amdgpu_ttm_copy_mem_to_mem()
375 * amdgpu_move_blit - Copy an entire buffer to another buffer
385 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_move_blit()
393 src.mem = old_mem; in amdgpu_move_blit()
394 dst.mem = new_mem; in amdgpu_move_blit()
399 new_mem->size, in amdgpu_move_blit()
401 bo->base.resv, &fence); in amdgpu_move_blit()
406 if (old_mem->mem_type == TTM_PL_VRAM && in amdgpu_move_blit()
407 (abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) { in amdgpu_move_blit()
415 amdgpu_vram_mgr_set_cleared(bo->resource); in amdgpu_move_blit()
422 if (bo->type == ttm_bo_type_kernel) in amdgpu_move_blit()
437 * amdgpu_res_cpu_visible - Check that resource can be accessed by CPU
451 if (res->mem_type == TTM_PL_SYSTEM || res->mem_type == TTM_PL_TT || in amdgpu_res_cpu_visible()
452 res->mem_type == AMDGPU_PL_PREEMPT || res->mem_type == AMDGPU_PL_DOORBELL || in amdgpu_res_cpu_visible()
453 res->mem_type == AMDGPU_PL_MMIO_REMAP) in amdgpu_res_cpu_visible()
456 if (res->mem_type != TTM_PL_VRAM) in amdgpu_res_cpu_visible()
459 amdgpu_res_first(res, 0, res->size, &cursor); in amdgpu_res_cpu_visible()
461 if ((cursor.start + cursor.size) > adev->gmc.visible_vram_size) in amdgpu_res_cpu_visible()
470 * amdgpu_res_copyable - Check that memory can be accessed by ttm_bo_move_memcpy
475 struct ttm_resource *mem) in amdgpu_res_copyable() argument
477 if (!amdgpu_res_cpu_visible(adev, mem)) in amdgpu_res_copyable()
481 if (mem->mem_type == TTM_PL_VRAM && in amdgpu_res_copyable()
482 !(mem->placement & TTM_PL_FLAG_CONTIGUOUS)) in amdgpu_res_copyable()
489 * amdgpu_bo_move - Move a buffer object to a new memory location
500 struct ttm_resource *old_mem = bo->resource; in amdgpu_bo_move()
503 if (new_mem->mem_type == TTM_PL_TT || in amdgpu_bo_move()
504 new_mem->mem_type == AMDGPU_PL_PREEMPT) { in amdgpu_bo_move()
505 r = amdgpu_ttm_backend_bind(bo->bdev, bo->ttm, new_mem); in amdgpu_bo_move()
511 adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_bo_move()
513 if (!old_mem || (old_mem->mem_type == TTM_PL_SYSTEM && in amdgpu_bo_move()
514 bo->ttm == NULL)) { in amdgpu_bo_move()
519 if (old_mem->mem_type == TTM_PL_SYSTEM && in amdgpu_bo_move()
520 (new_mem->mem_type == TTM_PL_TT || in amdgpu_bo_move()
521 new_mem->mem_type == AMDGPU_PL_PREEMPT)) { in amdgpu_bo_move()
526 if ((old_mem->mem_type == TTM_PL_TT || in amdgpu_bo_move()
527 old_mem->mem_type == AMDGPU_PL_PREEMPT) && in amdgpu_bo_move()
528 new_mem->mem_type == TTM_PL_SYSTEM) { in amdgpu_bo_move()
533 amdgpu_ttm_backend_unbind(bo->bdev, bo->ttm); in amdgpu_bo_move()
535 ttm_resource_free(bo, &bo->resource); in amdgpu_bo_move()
540 if (old_mem->mem_type == AMDGPU_PL_GDS || in amdgpu_bo_move()
541 old_mem->mem_type == AMDGPU_PL_GWS || in amdgpu_bo_move()
542 old_mem->mem_type == AMDGPU_PL_OA || in amdgpu_bo_move()
543 old_mem->mem_type == AMDGPU_PL_DOORBELL || in amdgpu_bo_move()
544 old_mem->mem_type == AMDGPU_PL_MMIO_REMAP || in amdgpu_bo_move()
545 new_mem->mem_type == AMDGPU_PL_GDS || in amdgpu_bo_move()
546 new_mem->mem_type == AMDGPU_PL_GWS || in amdgpu_bo_move()
547 new_mem->mem_type == AMDGPU_PL_OA || in amdgpu_bo_move()
548 new_mem->mem_type == AMDGPU_PL_DOORBELL || in amdgpu_bo_move()
549 new_mem->mem_type == AMDGPU_PL_MMIO_REMAP) { in amdgpu_bo_move()
556 if (bo->type == ttm_bo_type_device && in amdgpu_bo_move()
557 new_mem->mem_type == TTM_PL_VRAM && in amdgpu_bo_move()
558 old_mem->mem_type != TTM_PL_VRAM) { in amdgpu_bo_move()
559 /* amdgpu_bo_fault_reserve_notify will re-set this if the CPU in amdgpu_bo_move()
562 abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; in amdgpu_bo_move()
565 if (adev->mman.buffer_funcs_enabled && in amdgpu_bo_move()
566 ((old_mem->mem_type == TTM_PL_SYSTEM && in amdgpu_bo_move()
567 new_mem->mem_type == TTM_PL_VRAM) || in amdgpu_bo_move()
568 (old_mem->mem_type == TTM_PL_VRAM && in amdgpu_bo_move()
569 new_mem->mem_type == TTM_PL_SYSTEM))) { in amdgpu_bo_move()
570 hop->fpfn = 0; in amdgpu_bo_move()
571 hop->lpfn = 0; in amdgpu_bo_move()
572 hop->mem_type = TTM_PL_TT; in amdgpu_bo_move()
573 hop->flags = TTM_PL_FLAG_TEMPORARY; in amdgpu_bo_move()
574 return -EMULTIHOP; in amdgpu_bo_move()
578 if (adev->mman.buffer_funcs_enabled) in amdgpu_bo_move()
581 r = -ENODEV; in amdgpu_bo_move()
598 atomic64_inc(&adev->num_evictions); in amdgpu_bo_move()
599 atomic64_add(bo->base.size, &adev->num_bytes_moved); in amdgpu_bo_move()
604 * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault
609 struct ttm_resource *mem) in amdgpu_ttm_io_mem_reserve() argument
613 switch (mem->mem_type) { in amdgpu_ttm_io_mem_reserve()
621 mem->bus.offset = mem->start << PAGE_SHIFT; in amdgpu_ttm_io_mem_reserve()
623 if (adev->mman.aper_base_kaddr && in amdgpu_ttm_io_mem_reserve()
624 mem->placement & TTM_PL_FLAG_CONTIGUOUS) in amdgpu_ttm_io_mem_reserve()
625 mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr + in amdgpu_ttm_io_mem_reserve()
626 mem->bus.offset; in amdgpu_ttm_io_mem_reserve()
628 mem->bus.offset += adev->gmc.aper_base; in amdgpu_ttm_io_mem_reserve()
629 mem->bus.is_iomem = true; in amdgpu_ttm_io_mem_reserve()
632 mem->bus.offset = mem->start << PAGE_SHIFT; in amdgpu_ttm_io_mem_reserve()
633 mem->bus.offset += adev->doorbell.base; in amdgpu_ttm_io_mem_reserve()
634 mem->bus.is_iomem = true; in amdgpu_ttm_io_mem_reserve()
635 mem->bus.caching = ttm_uncached; in amdgpu_ttm_io_mem_reserve()
638 mem->bus.offset = mem->start << PAGE_SHIFT; in amdgpu_ttm_io_mem_reserve()
639 mem->bus.offset += adev->rmmio_remap.bus_addr; in amdgpu_ttm_io_mem_reserve()
640 mem->bus.is_iomem = true; in amdgpu_ttm_io_mem_reserve()
641 mem->bus.caching = ttm_uncached; in amdgpu_ttm_io_mem_reserve()
644 return -EINVAL; in amdgpu_ttm_io_mem_reserve()
652 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_ttm_io_mem_pfn()
655 amdgpu_res_first(bo->resource, (u64)page_offset << PAGE_SHIFT, 0, in amdgpu_ttm_io_mem_pfn()
658 if (bo->resource->mem_type == AMDGPU_PL_DOORBELL) in amdgpu_ttm_io_mem_pfn()
659 return ((uint64_t)(adev->doorbell.base + cursor.start)) >> PAGE_SHIFT; in amdgpu_ttm_io_mem_pfn()
660 else if (bo->resource->mem_type == AMDGPU_PL_MMIO_REMAP) in amdgpu_ttm_io_mem_pfn()
661 return ((uint64_t)(adev->rmmio_remap.bus_addr + cursor.start)) >> PAGE_SHIFT; in amdgpu_ttm_io_mem_pfn()
663 return (adev->gmc.aper_base + cursor.start) >> PAGE_SHIFT; in amdgpu_ttm_io_mem_pfn()
667 * amdgpu_ttm_domain_start - Returns GPU start address
679 return adev->gmc.gart_start; in amdgpu_ttm_domain_start()
681 return adev->gmc.vram_start; in amdgpu_ttm_domain_start()
705 * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user
714 struct ttm_tt *ttm = bo->tbo.ttm; in amdgpu_ttm_tt_get_user_pages()
716 unsigned long start = gtt->userptr; in amdgpu_ttm_tt_get_user_pages()
725 mm = bo->notifier.mm; in amdgpu_ttm_tt_get_user_pages()
728 return -EFAULT; in amdgpu_ttm_tt_get_user_pages()
732 return -ESRCH; in amdgpu_ttm_tt_get_user_pages()
737 r = -EFAULT; in amdgpu_ttm_tt_get_user_pages()
740 if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) && in amdgpu_ttm_tt_get_user_pages()
741 vma->vm_file)) { in amdgpu_ttm_tt_get_user_pages()
742 r = -EPERM; in amdgpu_ttm_tt_get_user_pages()
747 r = amdgpu_hmm_range_get_pages(&bo->notifier, start, ttm->num_pages, in amdgpu_ttm_tt_get_user_pages()
759 /* amdgpu_ttm_tt_discard_user_pages - Discard range and pfn array allocations
766 if (gtt && gtt->userptr && range) in amdgpu_ttm_tt_discard_user_pages()
771 * amdgpu_ttm_tt_get_user_pages_done - stop HMM track the CPU page table change
781 if (!gtt || !gtt->userptr || !range) in amdgpu_ttm_tt_get_user_pages_done()
785 gtt->userptr, ttm->num_pages); in amdgpu_ttm_tt_get_user_pages_done()
787 WARN_ONCE(!range->hmm_pfns, "No user pages to check\n"); in amdgpu_ttm_tt_get_user_pages_done()
794 * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
804 for (i = 0; i < ttm->num_pages; ++i) in amdgpu_ttm_tt_set_user_pages()
805 ttm->pages[i] = range ? hmm_pfn_to_page(range->hmm_pfns[i]) : NULL; in amdgpu_ttm_tt_set_user_pages()
809 * amdgpu_ttm_tt_pin_userptr - prepare the sg table with the user pages
818 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); in amdgpu_ttm_tt_pin_userptr()
824 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, in amdgpu_ttm_tt_pin_userptr()
825 (u64)ttm->num_pages << PAGE_SHIFT, in amdgpu_ttm_tt_pin_userptr()
831 r = dma_map_sgtable(adev->dev, ttm->sg, direction, 0); in amdgpu_ttm_tt_pin_userptr()
836 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, in amdgpu_ttm_tt_pin_userptr()
837 ttm->num_pages); in amdgpu_ttm_tt_pin_userptr()
842 sg_free_table(ttm->sg); in amdgpu_ttm_tt_pin_userptr()
844 kfree(ttm->sg); in amdgpu_ttm_tt_pin_userptr()
845 ttm->sg = NULL; in amdgpu_ttm_tt_pin_userptr()
850 * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages
857 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); in amdgpu_ttm_tt_unpin_userptr()
862 if (!ttm->sg || !ttm->sg->sgl) in amdgpu_ttm_tt_unpin_userptr()
866 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); in amdgpu_ttm_tt_unpin_userptr()
867 sg_free_table(ttm->sg); in amdgpu_ttm_tt_unpin_userptr()
881 uint64_t total_pages = ttm->num_pages; in amdgpu_ttm_gart_bind_gfx9_mqd()
882 int num_xcc = max(1U, adev->gfx.num_xcc_per_xcp); in amdgpu_ttm_gart_bind_gfx9_mqd()
893 gtt->offset + (page_idx << PAGE_SHIFT), in amdgpu_ttm_gart_bind_gfx9_mqd()
894 1, >t->ttm.dma_address[page_idx], flags); in amdgpu_ttm_gart_bind_gfx9_mqd()
896 * Ctrl pages - modify the memory type to NC (ctrl_flags) from in amdgpu_ttm_gart_bind_gfx9_mqd()
900 gtt->offset + ((page_idx + 1) << PAGE_SHIFT), in amdgpu_ttm_gart_bind_gfx9_mqd()
901 pages_per_xcc - 1, in amdgpu_ttm_gart_bind_gfx9_mqd()
902 >t->ttm.dma_address[page_idx + 1], in amdgpu_ttm_gart_bind_gfx9_mqd()
912 struct ttm_tt *ttm = tbo->ttm; in amdgpu_ttm_gart_bind()
918 if (abo->flags & AMDGPU_GEM_CREATE_CP_MQD_GFX9) { in amdgpu_ttm_gart_bind()
921 amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, in amdgpu_ttm_gart_bind()
922 gtt->ttm.dma_address, flags); in amdgpu_ttm_gart_bind()
924 gtt->bound = true; in amdgpu_ttm_gart_bind()
928 * amdgpu_ttm_backend_bind - Bind GTT memory
943 return -EINVAL; in amdgpu_ttm_backend_bind()
945 if (gtt->bound) in amdgpu_ttm_backend_bind()
948 if (gtt->userptr) { in amdgpu_ttm_backend_bind()
951 dev_err(adev->dev, "failed to pin userptr\n"); in amdgpu_ttm_backend_bind()
954 } else if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) { in amdgpu_ttm_backend_bind()
955 if (!ttm->sg) { in amdgpu_ttm_backend_bind()
959 attach = gtt->gobj->import_attach; in amdgpu_ttm_backend_bind()
964 ttm->sg = sgt; in amdgpu_ttm_backend_bind()
967 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, in amdgpu_ttm_backend_bind()
968 ttm->num_pages); in amdgpu_ttm_backend_bind()
971 if (!ttm->num_pages) { in amdgpu_ttm_backend_bind()
973 ttm->num_pages, bo_mem, ttm); in amdgpu_ttm_backend_bind()
976 if (bo_mem->mem_type != TTM_PL_TT || in amdgpu_ttm_backend_bind()
978 gtt->offset = AMDGPU_BO_INVALID_OFFSET; in amdgpu_ttm_backend_bind()
986 gtt->offset = (u64)bo_mem->start << PAGE_SHIFT; in amdgpu_ttm_backend_bind()
987 amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, in amdgpu_ttm_backend_bind()
988 gtt->ttm.dma_address, flags); in amdgpu_ttm_backend_bind()
989 gtt->bound = true; in amdgpu_ttm_backend_bind()
994 * amdgpu_ttm_alloc_gart - Make sure buffer object is accessible either
1003 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_ttm_alloc_gart()
1005 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(bo->ttm); in amdgpu_ttm_alloc_gart()
1012 if (bo->resource->start != AMDGPU_BO_INVALID_OFFSET) in amdgpu_ttm_alloc_gart()
1023 placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT; in amdgpu_ttm_alloc_gart()
1025 placements.flags = bo->resource->placement; in amdgpu_ttm_alloc_gart()
1032 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, tmp); in amdgpu_ttm_alloc_gart()
1035 gtt->offset = (u64)tmp->start << PAGE_SHIFT; in amdgpu_ttm_alloc_gart()
1038 ttm_resource_free(bo, &bo->resource); in amdgpu_ttm_alloc_gart()
1045 * amdgpu_ttm_recover_gart - Rebind GTT pages
1052 struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev); in amdgpu_ttm_recover_gart()
1055 if (!tbo->ttm) in amdgpu_ttm_recover_gart()
1058 flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, tbo->resource); in amdgpu_ttm_recover_gart()
1063 * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages
1075 if (gtt->userptr) { in amdgpu_ttm_backend_unbind()
1077 } else if (ttm->sg && drm_gem_is_imported(gtt->gobj)) { in amdgpu_ttm_backend_unbind()
1080 attach = gtt->gobj->import_attach; in amdgpu_ttm_backend_unbind()
1081 dma_buf_unmap_attachment(attach, ttm->sg, DMA_BIDIRECTIONAL); in amdgpu_ttm_backend_unbind()
1082 ttm->sg = NULL; in amdgpu_ttm_backend_unbind()
1085 if (!gtt->bound) in amdgpu_ttm_backend_unbind()
1088 if (gtt->offset == AMDGPU_BO_INVALID_OFFSET) in amdgpu_ttm_backend_unbind()
1092 amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages); in amdgpu_ttm_backend_unbind()
1093 gtt->bound = false; in amdgpu_ttm_backend_unbind()
1101 if (gtt->usertask) in amdgpu_ttm_backend_destroy()
1102 put_task_struct(gtt->usertask); in amdgpu_ttm_backend_destroy()
1104 ttm_tt_fini(>t->ttm); in amdgpu_ttm_backend_destroy()
1109 * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
1119 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_ttm_tt_create()
1128 gtt->gobj = &bo->base; in amdgpu_ttm_tt_create()
1129 if (adev->gmc.mem_partitions && abo->xcp_id >= 0) in amdgpu_ttm_tt_create()
1130 gtt->pool_id = KFD_XCP_MEM_ID(adev, abo->xcp_id); in amdgpu_ttm_tt_create()
1132 gtt->pool_id = abo->xcp_id; in amdgpu_ttm_tt_create()
1134 if (abo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) in amdgpu_ttm_tt_create()
1140 if (ttm_sg_tt_init(>t->ttm, bo, page_flags, caching)) { in amdgpu_ttm_tt_create()
1144 return >t->ttm; in amdgpu_ttm_tt_create()
1148 * amdgpu_ttm_tt_populate - Map GTT pages visible to the device
1164 if (gtt->userptr) { in amdgpu_ttm_tt_populate()
1165 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL); in amdgpu_ttm_tt_populate()
1166 if (!ttm->sg) in amdgpu_ttm_tt_populate()
1167 return -ENOMEM; in amdgpu_ttm_tt_populate()
1171 if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) in amdgpu_ttm_tt_populate()
1174 if (adev->mman.ttm_pools && gtt->pool_id >= 0) in amdgpu_ttm_tt_populate()
1175 pool = &adev->mman.ttm_pools[gtt->pool_id]; in amdgpu_ttm_tt_populate()
1177 pool = &adev->mman.bdev.pool; in amdgpu_ttm_tt_populate()
1182 for (i = 0; i < ttm->num_pages; ++i) in amdgpu_ttm_tt_populate()
1183 ttm->pages[i]->mapping = bdev->dev_mapping; in amdgpu_ttm_tt_populate()
1189 * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays
1204 if (gtt->userptr) { in amdgpu_ttm_tt_unpopulate()
1206 kfree(ttm->sg); in amdgpu_ttm_tt_unpopulate()
1207 ttm->sg = NULL; in amdgpu_ttm_tt_unpopulate()
1211 if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) in amdgpu_ttm_tt_unpopulate()
1214 for (i = 0; i < ttm->num_pages; ++i) in amdgpu_ttm_tt_unpopulate()
1215 ttm->pages[i]->mapping = NULL; in amdgpu_ttm_tt_unpopulate()
1219 if (adev->mman.ttm_pools && gtt->pool_id >= 0) in amdgpu_ttm_tt_unpopulate()
1220 pool = &adev->mman.ttm_pools[gtt->pool_id]; in amdgpu_ttm_tt_unpopulate()
1222 pool = &adev->mman.bdev.pool; in amdgpu_ttm_tt_unpopulate()
1228 * amdgpu_ttm_tt_get_userptr - Return the userptr GTT ttm_tt for the current
1239 if (!tbo->ttm) in amdgpu_ttm_tt_get_userptr()
1240 return -EINVAL; in amdgpu_ttm_tt_get_userptr()
1242 gtt = (void *)tbo->ttm; in amdgpu_ttm_tt_get_userptr()
1243 *user_addr = gtt->userptr; in amdgpu_ttm_tt_get_userptr()
1248 * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current
1264 if (!bo->ttm) { in amdgpu_ttm_tt_set_userptr()
1266 bo->ttm = amdgpu_ttm_tt_create(bo, 0); in amdgpu_ttm_tt_set_userptr()
1267 if (bo->ttm == NULL) in amdgpu_ttm_tt_set_userptr()
1268 return -ENOMEM; in amdgpu_ttm_tt_set_userptr()
1272 bo->ttm->page_flags |= TTM_TT_FLAG_EXTERNAL; in amdgpu_ttm_tt_set_userptr()
1274 gtt = ttm_to_amdgpu_ttm_tt(bo->ttm); in amdgpu_ttm_tt_set_userptr()
1275 gtt->userptr = addr; in amdgpu_ttm_tt_set_userptr()
1276 gtt->userflags = flags; in amdgpu_ttm_tt_set_userptr()
1278 if (gtt->usertask) in amdgpu_ttm_tt_set_userptr()
1279 put_task_struct(gtt->usertask); in amdgpu_ttm_tt_set_userptr()
1280 gtt->usertask = current->group_leader; in amdgpu_ttm_tt_set_userptr()
1281 get_task_struct(gtt->usertask); in amdgpu_ttm_tt_set_userptr()
1287 * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object
1296 if (gtt->usertask == NULL) in amdgpu_ttm_tt_get_usermm()
1299 return gtt->usertask->mm; in amdgpu_ttm_tt_get_usermm()
1303 * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an
1313 if (gtt == NULL || !gtt->userptr) in amdgpu_ttm_tt_affect_userptr()
1319 size = (unsigned long)gtt->ttm.num_pages * PAGE_SIZE; in amdgpu_ttm_tt_affect_userptr()
1320 if (gtt->userptr > end || gtt->userptr + size <= start) in amdgpu_ttm_tt_affect_userptr()
1324 *userptr = gtt->userptr; in amdgpu_ttm_tt_affect_userptr()
1329 * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr?
1335 if (gtt == NULL || !gtt->userptr) in amdgpu_ttm_tt_is_userptr()
1342 * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only?
1351 return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); in amdgpu_ttm_tt_is_readonly()
1355 * amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object
1358 * @mem: The memory registry backing this ttm_tt object
1362 uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_resource *mem) in amdgpu_ttm_tt_pde_flags() argument
1366 if (mem && mem->mem_type != TTM_PL_SYSTEM) in amdgpu_ttm_tt_pde_flags()
1369 if (mem && (mem->mem_type == TTM_PL_TT || in amdgpu_ttm_tt_pde_flags()
1370 mem->mem_type == AMDGPU_PL_DOORBELL || in amdgpu_ttm_tt_pde_flags()
1371 mem->mem_type == AMDGPU_PL_PREEMPT || in amdgpu_ttm_tt_pde_flags()
1372 mem->mem_type == AMDGPU_PL_MMIO_REMAP)) { in amdgpu_ttm_tt_pde_flags()
1375 if (ttm && ttm->caching == ttm_cached) in amdgpu_ttm_tt_pde_flags()
1379 if (mem && mem->mem_type == TTM_PL_VRAM && in amdgpu_ttm_tt_pde_flags()
1380 mem->bus.caching == ttm_cached) in amdgpu_ttm_tt_pde_flags()
1387 * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object
1391 * @mem: The memory registry backing this ttm_tt object
1396 struct ttm_resource *mem) in amdgpu_ttm_tt_pte_flags() argument
1398 uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem); in amdgpu_ttm_tt_pte_flags()
1400 flags |= adev->gart.gart_pte_flags; in amdgpu_ttm_tt_pte_flags()
1410 * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer
1428 if (bo->resource->mem_type == TTM_PL_SYSTEM) in amdgpu_ttm_bo_eviction_valuable()
1431 if (bo->type == ttm_bo_type_kernel && in amdgpu_ttm_bo_eviction_valuable()
1439 dma_resv_for_each_fence(&resv_cursor, bo->base.resv, in amdgpu_ttm_bo_eviction_valuable()
1441 if (amdkfd_fence_check_mm(f, current->mm) && in amdgpu_ttm_bo_eviction_valuable()
1442 !(place->flags & TTM_PL_FLAG_CONTIGUOUS)) in amdgpu_ttm_bo_eviction_valuable()
1453 if (bo->resource->mem_type == AMDGPU_PL_PREEMPT) in amdgpu_ttm_bo_eviction_valuable()
1456 if (bo->resource->mem_type == TTM_PL_TT && in amdgpu_ttm_bo_eviction_valuable()
1468 uint64_t bytes = 4 - (pos & 0x3); in amdgpu_ttm_vram_mm_access()
1474 mask &= 0xffffffff >> (bytes - size) * 8; in amdgpu_ttm_vram_mm_access()
1494 size -= bytes; in amdgpu_ttm_vram_mm_access()
1503 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); in amdgpu_ttm_access_memory_sdma()
1512 return -EINVAL; in amdgpu_ttm_access_memory_sdma()
1514 if (!adev->mman.sdma_access_ptr) in amdgpu_ttm_access_memory_sdma()
1515 return -EACCES; in amdgpu_ttm_access_memory_sdma()
1518 return -ENODEV; in amdgpu_ttm_access_memory_sdma()
1521 memcpy(adev->mman.sdma_access_ptr, buf, len); in amdgpu_ttm_access_memory_sdma()
1523 num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); in amdgpu_ttm_access_memory_sdma()
1524 r = amdgpu_job_alloc_with_ib(adev, &adev->mman.high_pr, in amdgpu_ttm_access_memory_sdma()
1532 amdgpu_res_first(abo->tbo.resource, offset, len, &src_mm); in amdgpu_ttm_access_memory_sdma()
1533 src_addr = amdgpu_ttm_domain_start(adev, bo->resource->mem_type) + in amdgpu_ttm_access_memory_sdma()
1535 dst_addr = amdgpu_bo_gpu_offset(adev->mman.sdma_access_bo); in amdgpu_ttm_access_memory_sdma()
1539 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, dst_addr, in amdgpu_ttm_access_memory_sdma()
1542 amdgpu_ring_pad_ib(adev->mman.buffer_funcs_ring, &job->ibs[0]); in amdgpu_ttm_access_memory_sdma()
1543 WARN_ON(job->ibs[0].length_dw > num_dw); in amdgpu_ttm_access_memory_sdma()
1547 if (!dma_fence_wait_timeout(fence, false, adev->sdma_timeout)) in amdgpu_ttm_access_memory_sdma()
1548 r = -ETIMEDOUT; in amdgpu_ttm_access_memory_sdma()
1552 memcpy(buf, adev->mman.sdma_access_ptr, len); in amdgpu_ttm_access_memory_sdma()
1559 * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object.
1575 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); in amdgpu_ttm_access_memory()
1579 if (bo->resource->mem_type != TTM_PL_VRAM) in amdgpu_ttm_access_memory()
1580 return -EIO; in amdgpu_ttm_access_memory()
1586 amdgpu_res_first(bo->resource, offset, len, &cursor); in amdgpu_ttm_access_memory()
1592 size -= count; in amdgpu_ttm_access_memory()
1594 /* using MM to access rest vram and handle un-aligned address */ in amdgpu_ttm_access_memory()
1633 * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram
1641 amdgpu_bo_free_kernel(&adev->mman.fw_vram_usage_reserved_bo, in amdgpu_ttm_fw_reserve_vram_fini()
1642 NULL, &adev->mman.fw_vram_usage_va); in amdgpu_ttm_fw_reserve_vram_fini()
1649 * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
1657 amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo, in amdgpu_ttm_drv_reserve_vram_fini()
1659 &adev->mman.drv_vram_usage_va); in amdgpu_ttm_drv_reserve_vram_fini()
1663 * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
1671 uint64_t vram_size = adev->gmc.visible_vram_size; in amdgpu_ttm_fw_reserve_vram_init()
1673 adev->mman.fw_vram_usage_va = NULL; in amdgpu_ttm_fw_reserve_vram_init()
1674 adev->mman.fw_vram_usage_reserved_bo = NULL; in amdgpu_ttm_fw_reserve_vram_init()
1676 if (adev->mman.fw_vram_usage_size == 0 || in amdgpu_ttm_fw_reserve_vram_init()
1677 adev->mman.fw_vram_usage_size > vram_size) in amdgpu_ttm_fw_reserve_vram_init()
1681 adev->mman.fw_vram_usage_start_offset, in amdgpu_ttm_fw_reserve_vram_init()
1682 adev->mman.fw_vram_usage_size, in amdgpu_ttm_fw_reserve_vram_init()
1683 &adev->mman.fw_vram_usage_reserved_bo, in amdgpu_ttm_fw_reserve_vram_init()
1684 &adev->mman.fw_vram_usage_va); in amdgpu_ttm_fw_reserve_vram_init()
1688 * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
1696 u64 vram_size = adev->gmc.visible_vram_size; in amdgpu_ttm_drv_reserve_vram_init()
1698 adev->mman.drv_vram_usage_va = NULL; in amdgpu_ttm_drv_reserve_vram_init()
1699 adev->mman.drv_vram_usage_reserved_bo = NULL; in amdgpu_ttm_drv_reserve_vram_init()
1701 if (adev->mman.drv_vram_usage_size == 0 || in amdgpu_ttm_drv_reserve_vram_init()
1702 adev->mman.drv_vram_usage_size > vram_size) in amdgpu_ttm_drv_reserve_vram_init()
1706 adev->mman.drv_vram_usage_start_offset, in amdgpu_ttm_drv_reserve_vram_init()
1707 adev->mman.drv_vram_usage_size, in amdgpu_ttm_drv_reserve_vram_init()
1708 &adev->mman.drv_vram_usage_reserved_bo, in amdgpu_ttm_drv_reserve_vram_init()
1709 &adev->mman.drv_vram_usage_va); in amdgpu_ttm_drv_reserve_vram_init()
1717 * amdgpu_ttm_training_reserve_vram_fini - free memory training reserved vram
1725 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; in amdgpu_ttm_training_reserve_vram_fini()
1727 ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT; in amdgpu_ttm_training_reserve_vram_fini()
1728 amdgpu_bo_free_kernel(&ctx->c2p_bo, NULL, NULL); in amdgpu_ttm_training_reserve_vram_fini()
1729 ctx->c2p_bo = NULL; in amdgpu_ttm_training_reserve_vram_fini()
1737 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; in amdgpu_ttm_training_data_block_init()
1741 ctx->c2p_train_data_offset = in amdgpu_ttm_training_data_block_init()
1742 ALIGN((adev->gmc.mc_vram_size - reserve_size - SZ_1M), SZ_1M); in amdgpu_ttm_training_data_block_init()
1743 ctx->p2c_train_data_offset = in amdgpu_ttm_training_data_block_init()
1744 (adev->gmc.mc_vram_size - GDDR6_MEM_TRAINING_OFFSET); in amdgpu_ttm_training_data_block_init()
1745 ctx->train_data_size = in amdgpu_ttm_training_data_block_init()
1749 ctx->train_data_size, in amdgpu_ttm_training_data_block_init()
1750 ctx->p2c_train_data_offset, in amdgpu_ttm_training_data_block_init()
1751 ctx->c2p_train_data_offset); in amdgpu_ttm_training_data_block_init()
1760 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; in amdgpu_ttm_reserve_tmr()
1765 if (adev->bios && !amdgpu_sriov_vf(adev)) { in amdgpu_ttm_reserve_tmr()
1779 if (adev->bios) in amdgpu_ttm_reserve_tmr()
1783 if (!adev->bios && in amdgpu_ttm_reserve_tmr()
1792 /* reserve vram for mem train according to TMR location */ in amdgpu_ttm_reserve_tmr()
1795 ctx->c2p_train_data_offset, in amdgpu_ttm_reserve_tmr()
1796 ctx->train_data_size, in amdgpu_ttm_reserve_tmr()
1797 &ctx->c2p_bo, in amdgpu_ttm_reserve_tmr()
1800 dev_err(adev->dev, "alloc c2p_bo failed(%d)!\n", ret); in amdgpu_ttm_reserve_tmr()
1804 ctx->init = PSP_MEM_TRAIN_RESERVE_SUCCESS; in amdgpu_ttm_reserve_tmr()
1807 if (!adev->gmc.is_app_apu) { in amdgpu_ttm_reserve_tmr()
1809 adev, adev->gmc.real_vram_size - reserve_size, in amdgpu_ttm_reserve_tmr()
1810 reserve_size, &adev->mman.fw_reserved_memory, NULL); in amdgpu_ttm_reserve_tmr()
1812 dev_err(adev->dev, "alloc tmr failed(%d)!\n", ret); in amdgpu_ttm_reserve_tmr()
1813 amdgpu_bo_free_kernel(&adev->mman.fw_reserved_memory, in amdgpu_ttm_reserve_tmr()
1828 if (!adev->gmc.is_app_apu || !adev->gmc.num_mem_partitions) in amdgpu_ttm_pools_init()
1831 adev->mman.ttm_pools = kcalloc(adev->gmc.num_mem_partitions, in amdgpu_ttm_pools_init()
1832 sizeof(*adev->mman.ttm_pools), in amdgpu_ttm_pools_init()
1834 if (!adev->mman.ttm_pools) in amdgpu_ttm_pools_init()
1835 return -ENOMEM; in amdgpu_ttm_pools_init()
1837 for (i = 0; i < adev->gmc.num_mem_partitions; i++) { in amdgpu_ttm_pools_init()
1838 ttm_pool_init(&adev->mman.ttm_pools[i], adev->dev, in amdgpu_ttm_pools_init()
1839 adev->gmc.mem_partitions[i].numa.node, in amdgpu_ttm_pools_init()
1849 if (!adev->gmc.is_app_apu || !adev->mman.ttm_pools) in amdgpu_ttm_pools_fini()
1852 for (i = 0; i < adev->gmc.num_mem_partitions; i++) in amdgpu_ttm_pools_fini()
1853 ttm_pool_fini(&adev->mman.ttm_pools[i]); in amdgpu_ttm_pools_fini()
1855 kfree(adev->mman.ttm_pools); in amdgpu_ttm_pools_fini()
1856 adev->mman.ttm_pools = NULL; in amdgpu_ttm_pools_fini()
1860 * amdgpu_ttm_mmio_remap_bo_init - Allocate the singleton 4K MMIO_REMAP BO
1863 * Allocates a one-page (4K) GEM BO in AMDGPU_GEM_DOMAIN_MMIO_REMAP when the
1864 * hardware exposes a remap base (adev->rmmio_remap.bus_addr) and the host
1878 if (!adev->rmmio_remap.bus_addr || PAGE_SIZE > AMDGPU_GPU_PAGE_SIZE) in amdgpu_ttm_mmio_remap_bo_init()
1884 bp.type = ttm_bo_type_device; /* userspace-mappable GEM */ in amdgpu_ttm_mmio_remap_bo_init()
1892 r = amdgpu_bo_create(adev, &bp, &adev->rmmio_remap.bo); in amdgpu_ttm_mmio_remap_bo_init()
1900 * amdgpu_ttm_mmio_remap_bo_fini - Free the singleton MMIO_REMAP BO
1903 * Frees the kernel-owned MMIO_REMAP BO if it was allocated by
1908 amdgpu_bo_unref(&adev->rmmio_remap.bo); in amdgpu_ttm_mmio_remap_bo_fini()
1909 adev->rmmio_remap.bo = NULL; in amdgpu_ttm_mmio_remap_bo_fini()
1913 * amdgpu_ttm_init - Init the memory management (ttm) as well as various
1918 * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which
1926 mutex_init(&adev->mman.gtt_window_lock); in amdgpu_ttm_init()
1928 dma_set_max_seg_size(adev->dev, UINT_MAX); in amdgpu_ttm_init()
1930 r = ttm_device_init(&adev->mman.bdev, &amdgpu_bo_driver, adev->dev, in amdgpu_ttm_init()
1931 adev_to_drm(adev)->anon_inode->i_mapping, in amdgpu_ttm_init()
1932 adev_to_drm(adev)->vma_offset_manager, in amdgpu_ttm_init()
1933 adev->need_swiotlb, in amdgpu_ttm_init()
1934 dma_addressing_limited(adev->dev)); in amdgpu_ttm_init()
1936 dev_err(adev->dev, in amdgpu_ttm_init()
1943 dev_err(adev->dev, "failed to init ttm pools(%d).\n", r); in amdgpu_ttm_init()
1946 adev->mman.initialized = true; in amdgpu_ttm_init()
1948 if (!adev->gmc.is_app_apu) { in amdgpu_ttm_init()
1952 dev_err(adev->dev, "Failed initializing VRAM heap.\n"); in amdgpu_ttm_init()
1961 if (adev->gmc.xgmi.connected_to_cpu) in amdgpu_ttm_init()
1962 adev->mman.aper_base_kaddr = ioremap_cache(adev->gmc.aper_base, in amdgpu_ttm_init()
1963 adev->gmc.visible_vram_size); in amdgpu_ttm_init()
1965 else if (adev->gmc.is_app_apu) in amdgpu_ttm_init()
1970 adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base, in amdgpu_ttm_init()
1971 adev->gmc.visible_vram_size); in amdgpu_ttm_init()
1995 if (adev->mman.discovery_bin) { in amdgpu_ttm_init()
2002 * This is used for VGA emulation and pre-OS scanout buffers to in amdgpu_ttm_init()
2003 * avoid display artifacts while transitioning between pre-OS in amdgpu_ttm_init()
2006 if (!adev->gmc.is_app_apu) { in amdgpu_ttm_init()
2008 adev->mman.stolen_vga_size, in amdgpu_ttm_init()
2009 &adev->mman.stolen_vga_memory, in amdgpu_ttm_init()
2014 r = amdgpu_bo_create_kernel_at(adev, adev->mman.stolen_vga_size, in amdgpu_ttm_init()
2015 adev->mman.stolen_extended_size, in amdgpu_ttm_init()
2016 &adev->mman.stolen_extended_memory, in amdgpu_ttm_init()
2023 adev->mman.stolen_reserved_offset, in amdgpu_ttm_init()
2024 adev->mman.stolen_reserved_size, in amdgpu_ttm_init()
2025 &adev->mman.stolen_reserved_memory, in amdgpu_ttm_init()
2033 dev_info(adev->dev, "amdgpu: %uM of VRAM memory ready\n", in amdgpu_ttm_init()
2034 (unsigned int)(adev->gmc.real_vram_size / (1024 * 1024))); in amdgpu_ttm_init()
2040 if (amdgpu_gtt_size != -1) { in amdgpu_ttm_init()
2043 drm_warn(&adev->ddev, in amdgpu_ttm_init()
2046 drm_warn(&adev->ddev, in amdgpu_ttm_init()
2056 dev_err(adev->dev, "Failed initializing GTT heap.\n"); in amdgpu_ttm_init()
2059 dev_info(adev->dev, "amdgpu: %uM of GTT memory ready.\n", in amdgpu_ttm_init()
2062 if (adev->flags & AMD_IS_APU) { in amdgpu_ttm_init()
2063 if (adev->gmc.real_vram_size < gtt_size) in amdgpu_ttm_init()
2064 adev->apu_prefer_gtt = true; in amdgpu_ttm_init()
2068 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_DOORBELL, adev->doorbell.size / PAGE_SIZE); in amdgpu_ttm_init()
2070 dev_err(adev->dev, "Failed initializing doorbell heap.\n"); in amdgpu_ttm_init()
2077 dev_err(adev->dev, "Failed to initialize kernel doorbells.\n"); in amdgpu_ttm_init()
2081 /* Initialize MMIO-remap pool (single page 4K) */ in amdgpu_ttm_init()
2084 dev_err(adev->dev, "Failed initializing MMIO-remap heap.\n"); in amdgpu_ttm_init()
2096 dev_err(adev->dev, "Failed initializing PREEMPT heap.\n"); in amdgpu_ttm_init()
2100 /* Initialize various on-chip memory pools */ in amdgpu_ttm_init()
2101 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GDS, adev->gds.gds_size); in amdgpu_ttm_init()
2103 dev_err(adev->dev, "Failed initializing GDS heap.\n"); in amdgpu_ttm_init()
2107 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GWS, adev->gds.gws_size); in amdgpu_ttm_init()
2109 dev_err(adev->dev, "Failed initializing gws heap.\n"); in amdgpu_ttm_init()
2113 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_OA, adev->gds.oa_size); in amdgpu_ttm_init()
2115 dev_err(adev->dev, "Failed initializing oa heap.\n"); in amdgpu_ttm_init()
2120 &adev->mman.sdma_access_bo, NULL, in amdgpu_ttm_init()
2121 &adev->mman.sdma_access_ptr)) in amdgpu_ttm_init()
2128 * amdgpu_ttm_fini - De-initialize the TTM memory pools
2134 if (!adev->mman.initialized) in amdgpu_ttm_fini()
2141 if (!adev->gmc.is_app_apu) { in amdgpu_ttm_fini()
2142 amdgpu_bo_free_kernel(&adev->mman.stolen_vga_memory, NULL, NULL); in amdgpu_ttm_fini()
2143 amdgpu_bo_free_kernel(&adev->mman.stolen_extended_memory, NULL, NULL); in amdgpu_ttm_fini()
2145 amdgpu_bo_free_kernel(&adev->mman.fw_reserved_memory, NULL, in amdgpu_ttm_fini()
2147 amdgpu_bo_free_kernel(&adev->mman.fw_reserved_memory_extend, NULL, in amdgpu_ttm_fini()
2149 if (adev->mman.stolen_reserved_size) in amdgpu_ttm_fini()
2150 amdgpu_bo_free_kernel(&adev->mman.stolen_reserved_memory, in amdgpu_ttm_fini()
2153 amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL, in amdgpu_ttm_fini()
2154 &adev->mman.sdma_access_ptr); in amdgpu_ttm_fini()
2162 if (adev->mman.aper_base_kaddr) in amdgpu_ttm_fini()
2163 iounmap(adev->mman.aper_base_kaddr); in amdgpu_ttm_fini()
2164 adev->mman.aper_base_kaddr = NULL; in amdgpu_ttm_fini()
2169 if (!adev->gmc.is_app_apu) in amdgpu_ttm_fini()
2175 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS); in amdgpu_ttm_fini()
2176 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GWS); in amdgpu_ttm_fini()
2177 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_OA); in amdgpu_ttm_fini()
2178 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_DOORBELL); in amdgpu_ttm_fini()
2179 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_MMIO_REMAP); in amdgpu_ttm_fini()
2180 ttm_device_fini(&adev->mman.bdev); in amdgpu_ttm_fini()
2181 adev->mman.initialized = false; in amdgpu_ttm_fini()
2182 dev_info(adev->dev, "amdgpu: ttm finalized\n"); in amdgpu_ttm_fini()
2186 * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions
2196 struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); in amdgpu_ttm_set_buffer_funcs_status()
2200 if (!adev->mman.initialized || amdgpu_in_reset(adev) || in amdgpu_ttm_set_buffer_funcs_status()
2201 adev->mman.buffer_funcs_enabled == enable || adev->gmc.is_app_apu) in amdgpu_ttm_set_buffer_funcs_status()
2208 ring = adev->mman.buffer_funcs_ring; in amdgpu_ttm_set_buffer_funcs_status()
2209 sched = &ring->sched; in amdgpu_ttm_set_buffer_funcs_status()
2210 r = drm_sched_entity_init(&adev->mman.high_pr, in amdgpu_ttm_set_buffer_funcs_status()
2214 dev_err(adev->dev, in amdgpu_ttm_set_buffer_funcs_status()
2220 r = drm_sched_entity_init(&adev->mman.low_pr, in amdgpu_ttm_set_buffer_funcs_status()
2224 dev_err(adev->dev, in amdgpu_ttm_set_buffer_funcs_status()
2230 drm_sched_entity_destroy(&adev->mman.high_pr); in amdgpu_ttm_set_buffer_funcs_status()
2231 drm_sched_entity_destroy(&adev->mman.low_pr); in amdgpu_ttm_set_buffer_funcs_status()
2232 dma_fence_put(man->move); in amdgpu_ttm_set_buffer_funcs_status()
2233 man->move = NULL; in amdgpu_ttm_set_buffer_funcs_status()
2238 size = adev->gmc.real_vram_size; in amdgpu_ttm_set_buffer_funcs_status()
2240 size = adev->gmc.visible_vram_size; in amdgpu_ttm_set_buffer_funcs_status()
2241 man->size = size; in amdgpu_ttm_set_buffer_funcs_status()
2242 adev->mman.buffer_funcs_enabled = enable; in amdgpu_ttm_set_buffer_funcs_status()
2247 drm_sched_entity_destroy(&adev->mman.high_pr); in amdgpu_ttm_set_buffer_funcs_status()
2262 struct drm_sched_entity *entity = delayed ? &adev->mman.low_pr : in amdgpu_ttm_prepare_job()
2263 &adev->mman.high_pr; in amdgpu_ttm_prepare_job()
2271 (*job)->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo ? in amdgpu_ttm_prepare_job()
2272 adev->gmc.pdb0_bo : in amdgpu_ttm_prepare_job()
2273 adev->gart.bo); in amdgpu_ttm_prepare_job()
2274 (*job)->vm_needs_flush = true; in amdgpu_ttm_prepare_job()
2279 return drm_sched_job_add_resv_dependencies(&(*job)->base, resv, in amdgpu_ttm_prepare_job()
2289 struct amdgpu_device *adev = ring->adev; in amdgpu_copy_buffer()
2296 if (!direct_submit && !ring->sched.ready) { in amdgpu_copy_buffer()
2297 dev_err(adev->dev, in amdgpu_copy_buffer()
2299 return -EINVAL; in amdgpu_copy_buffer()
2302 max_bytes = adev->mman.buffer_funcs->copy_max_bytes; in amdgpu_copy_buffer()
2304 num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->copy_num_dw, 8); in amdgpu_copy_buffer()
2314 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset, in amdgpu_copy_buffer()
2318 byte_count -= cur_size_in_bytes; in amdgpu_copy_buffer()
2321 amdgpu_ring_pad_ib(ring, &job->ibs[0]); in amdgpu_copy_buffer()
2322 WARN_ON(job->ibs[0].length_dw > num_dw); in amdgpu_copy_buffer()
2334 dev_err(adev->dev, "Error scheduling IBs (%d)\n", r); in amdgpu_copy_buffer()
2345 struct amdgpu_device *adev = ring->adev; in amdgpu_ttm_fill_mem()
2352 max_bytes = adev->mman.buffer_funcs->fill_max_bytes; in amdgpu_ttm_fill_mem()
2354 num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->fill_num_dw, 8); in amdgpu_ttm_fill_mem()
2363 amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data, dst_addr, in amdgpu_ttm_fill_mem()
2367 byte_count -= cur_size; in amdgpu_ttm_fill_mem()
2370 amdgpu_ring_pad_ib(ring, &job->ibs[0]); in amdgpu_ttm_fill_mem()
2371 WARN_ON(job->ibs[0].length_dw > num_dw); in amdgpu_ttm_fill_mem()
2377 * amdgpu_ttm_clear_buffer - clear memory buffers
2391 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); in amdgpu_ttm_clear_buffer()
2392 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; in amdgpu_ttm_clear_buffer()
2397 if (!adev->mman.buffer_funcs_enabled) in amdgpu_ttm_clear_buffer()
2398 return -EINVAL; in amdgpu_ttm_clear_buffer()
2401 return -EINVAL; in amdgpu_ttm_clear_buffer()
2405 amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &cursor); in amdgpu_ttm_clear_buffer()
2407 mutex_lock(&adev->mman.gtt_window_lock); in amdgpu_ttm_clear_buffer()
2420 r = amdgpu_ttm_map_buffer(&bo->tbo, bo->tbo.resource, &cursor, in amdgpu_ttm_clear_buffer()
2437 mutex_unlock(&adev->mman.gtt_window_lock); in amdgpu_ttm_clear_buffer()
2449 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); in amdgpu_fill_buffer()
2450 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; in amdgpu_fill_buffer()
2455 if (!adev->mman.buffer_funcs_enabled) { in amdgpu_fill_buffer()
2456 dev_err(adev->dev, in amdgpu_fill_buffer()
2458 return -EINVAL; in amdgpu_fill_buffer()
2461 amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &dst); in amdgpu_fill_buffer()
2463 mutex_lock(&adev->mman.gtt_window_lock); in amdgpu_fill_buffer()
2471 r = amdgpu_ttm_map_buffer(&bo->tbo, bo->tbo.resource, &dst, in amdgpu_fill_buffer()
2487 mutex_unlock(&adev->mman.gtt_window_lock); in amdgpu_fill_buffer()
2495 * amdgpu_ttm_evict_resources - evict memory buffers
2514 man = ttm_manager_type(&adev->mman.bdev, mem_type); in amdgpu_ttm_evict_resources()
2517 dev_err(adev->dev, "Trying to evict invalid memory type\n"); in amdgpu_ttm_evict_resources()
2518 return -EINVAL; in amdgpu_ttm_evict_resources()
2521 return ttm_resource_manager_evict_all(&adev->mman.bdev, man); in amdgpu_ttm_evict_resources()
2528 struct amdgpu_device *adev = m->private; in amdgpu_ttm_page_pool_show()
2530 return ttm_pool_debugfs(&adev->mman.bdev.pool, m); in amdgpu_ttm_page_pool_show()
2536 * amdgpu_ttm_vram_read - Linear read access to VRAM
2543 struct amdgpu_device *adev = file_inode(f)->i_private; in amdgpu_ttm_vram_read()
2547 return -EINVAL; in amdgpu_ttm_vram_read()
2549 if (*pos >= adev->gmc.mc_vram_size) in amdgpu_ttm_vram_read()
2550 return -ENXIO; in amdgpu_ttm_vram_read()
2552 size = min(size, (size_t)(adev->gmc.mc_vram_size - *pos)); in amdgpu_ttm_vram_read()
2559 return -EFAULT; in amdgpu_ttm_vram_read()
2564 size -= bytes; in amdgpu_ttm_vram_read()
2571 * amdgpu_ttm_vram_write - Linear write access to VRAM
2578 struct amdgpu_device *adev = file_inode(f)->i_private; in amdgpu_ttm_vram_write()
2583 return -EINVAL; in amdgpu_ttm_vram_write()
2585 if (*pos >= adev->gmc.mc_vram_size) in amdgpu_ttm_vram_write()
2586 return -ENXIO; in amdgpu_ttm_vram_write()
2591 if (*pos >= adev->gmc.mc_vram_size) in amdgpu_ttm_vram_write()
2603 size -= 4; in amdgpu_ttm_vram_write()
2617 * amdgpu_iomem_read - Virtual read access to GPU mapped memory
2626 struct amdgpu_device *adev = file_inode(f)->i_private; in amdgpu_iomem_read()
2632 dom = iommu_get_domain_for_dev(adev->dev); in amdgpu_iomem_read()
2637 size_t bytes = PAGE_SIZE - off; in amdgpu_iomem_read()
2652 return -EPERM; in amdgpu_iomem_read()
2655 if (p->mapping != adev->mman.bdev.dev_mapping) in amdgpu_iomem_read()
2656 return -EPERM; in amdgpu_iomem_read()
2662 return -EFAULT; in amdgpu_iomem_read()
2664 size -= bytes; in amdgpu_iomem_read()
2673 * amdgpu_iomem_write - Virtual write access to GPU mapped memory
2682 struct amdgpu_device *adev = file_inode(f)->i_private; in amdgpu_iomem_write()
2687 dom = iommu_get_domain_for_dev(adev->dev); in amdgpu_iomem_write()
2692 size_t bytes = PAGE_SIZE - off; in amdgpu_iomem_write()
2703 return -EPERM; in amdgpu_iomem_write()
2706 if (p->mapping != adev->mman.bdev.dev_mapping) in amdgpu_iomem_write()
2707 return -EPERM; in amdgpu_iomem_write()
2713 return -EFAULT; in amdgpu_iomem_write()
2715 size -= bytes; in amdgpu_iomem_write()
2735 struct drm_minor *minor = adev_to_drm(adev)->primary; in amdgpu_ttm_debugfs_init()
2736 struct dentry *root = minor->debugfs_root; in amdgpu_ttm_debugfs_init()
2739 &amdgpu_ttm_vram_fops, adev->gmc.mc_vram_size); in amdgpu_ttm_debugfs_init()
2744 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()
2747 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()
2750 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()
2753 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()
2756 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()