Lines Matching +full:tmr +full:- +full:manager
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
33 #include <linux/dma-mapping.h>
41 #include <linux/dma-buf.h>
78 return ttm_range_man_init(&adev->mman.bdev, type, in amdgpu_ttm_init_on_chip()
83 * amdgpu_evict_flags - Compute placement flags
93 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_evict_flags()
103 if (bo->type == ttm_bo_type_sg) { in amdgpu_evict_flags()
104 placement->num_placement = 0; in amdgpu_evict_flags()
110 placement->placement = &placements; in amdgpu_evict_flags()
111 placement->num_placement = 1; in amdgpu_evict_flags()
116 if (abo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) { in amdgpu_evict_flags()
117 placement->num_placement = 0; in amdgpu_evict_flags()
121 switch (bo->resource->mem_type) { in amdgpu_evict_flags()
126 placement->num_placement = 0; in amdgpu_evict_flags()
130 if (!adev->mman.buffer_funcs_enabled) { in amdgpu_evict_flags()
134 } else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && in amdgpu_evict_flags()
135 !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) && in amdgpu_evict_flags()
136 amdgpu_res_cpu_visible(adev, bo->resource)) { in amdgpu_evict_flags()
146 abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; in amdgpu_evict_flags()
147 abo->placements[0].lpfn = 0; in amdgpu_evict_flags()
148 abo->placements[0].flags |= TTM_PL_FLAG_DESIRED; in amdgpu_evict_flags()
161 *placement = abo->placement; in amdgpu_evict_flags()
165 * amdgpu_ttm_map_buffer - Map memory into the GART windows
184 struct amdgpu_device *adev = ring->adev; in amdgpu_ttm_map_buffer()
193 BUG_ON(adev->mman.buffer_funcs->copy_max_bytes < in amdgpu_ttm_map_buffer()
196 if (WARN_ON(mem->mem_type == AMDGPU_PL_PREEMPT)) in amdgpu_ttm_map_buffer()
197 return -EINVAL; in amdgpu_ttm_map_buffer()
200 if (!tmz && mem->start != AMDGPU_BO_INVALID_OFFSET) { in amdgpu_ttm_map_buffer()
201 *addr = amdgpu_ttm_domain_start(adev, mem->mem_type) + in amdgpu_ttm_map_buffer()
202 mm_cur->start; in amdgpu_ttm_map_buffer()
211 offset = mm_cur->start & ~PAGE_MASK; in amdgpu_ttm_map_buffer()
216 *size = min(*size, (uint64_t)num_pages * PAGE_SIZE - offset); in amdgpu_ttm_map_buffer()
218 *addr = adev->gmc.gart_start; in amdgpu_ttm_map_buffer()
223 num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); in amdgpu_ttm_map_buffer()
226 r = amdgpu_job_alloc_with_ib(adev, &adev->mman.high_pr, in amdgpu_ttm_map_buffer()
234 src_addr += job->ibs[0].gpu_addr; in amdgpu_ttm_map_buffer()
236 dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo); in amdgpu_ttm_map_buffer()
238 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, in amdgpu_ttm_map_buffer()
241 amdgpu_ring_pad_ib(ring, &job->ibs[0]); in amdgpu_ttm_map_buffer()
242 WARN_ON(job->ibs[0].length_dw > num_dw); in amdgpu_ttm_map_buffer()
244 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, mem); in amdgpu_ttm_map_buffer()
248 cpu_addr = &job->ibs[0].ptr[num_dw]; in amdgpu_ttm_map_buffer()
250 if (mem->mem_type == TTM_PL_TT) { in amdgpu_ttm_map_buffer()
253 dma_addr = &bo->ttm->dma_address[mm_cur->start >> PAGE_SHIFT]; in amdgpu_ttm_map_buffer()
258 dma_address = mm_cur->start; in amdgpu_ttm_map_buffer()
259 dma_address += adev->vm_manager.vram_base_offset; in amdgpu_ttm_map_buffer()
273 * amdgpu_ttm_copy_mem_to_mem - Helper function for copy
282 * The function copies @size bytes from {src->mem + src->offset} to
283 * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a
294 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; in amdgpu_ttm_copy_mem_to_mem()
301 if (!adev->mman.buffer_funcs_enabled) { in amdgpu_ttm_copy_mem_to_mem()
303 return -EINVAL; in amdgpu_ttm_copy_mem_to_mem()
306 amdgpu_res_first(src->mem, src->offset, size, &src_mm); in amdgpu_ttm_copy_mem_to_mem()
307 amdgpu_res_first(dst->mem, dst->offset, size, &dst_mm); in amdgpu_ttm_copy_mem_to_mem()
309 mutex_lock(&adev->mman.gtt_window_lock); in amdgpu_ttm_copy_mem_to_mem()
319 r = amdgpu_ttm_map_buffer(src->bo, src->mem, &src_mm, in amdgpu_ttm_copy_mem_to_mem()
324 r = amdgpu_ttm_map_buffer(dst->bo, dst->mem, &dst_mm, in amdgpu_ttm_copy_mem_to_mem()
329 abo_src = ttm_to_amdgpu_bo(src->bo); in amdgpu_ttm_copy_mem_to_mem()
330 abo_dst = ttm_to_amdgpu_bo(dst->bo); in amdgpu_ttm_copy_mem_to_mem()
333 if ((abo_src->flags & AMDGPU_GEM_CREATE_GFX12_DCC) && in amdgpu_ttm_copy_mem_to_mem()
334 (abo_src->tbo.resource->mem_type == TTM_PL_VRAM)) in amdgpu_ttm_copy_mem_to_mem()
336 if ((abo_dst->flags & AMDGPU_GEM_CREATE_GFX12_DCC) && in amdgpu_ttm_copy_mem_to_mem()
337 (dst->mem->mem_type == TTM_PL_VRAM)) { in amdgpu_ttm_copy_mem_to_mem()
360 mutex_unlock(&adev->mman.gtt_window_lock); in amdgpu_ttm_copy_mem_to_mem()
368 * amdgpu_move_blit - Copy an entire buffer to another buffer
378 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_move_blit()
392 new_mem->size, in amdgpu_move_blit()
394 bo->base.resv, &fence); in amdgpu_move_blit()
399 if (old_mem->mem_type == TTM_PL_VRAM && in amdgpu_move_blit()
400 (abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) { in amdgpu_move_blit()
408 amdgpu_vram_mgr_set_cleared(bo->resource); in amdgpu_move_blit()
415 if (bo->type == ttm_bo_type_kernel) in amdgpu_move_blit()
430 * amdgpu_res_cpu_visible - Check that resource can be accessed by CPU
444 if (res->mem_type == TTM_PL_SYSTEM || res->mem_type == TTM_PL_TT || in amdgpu_res_cpu_visible()
445 res->mem_type == AMDGPU_PL_PREEMPT || res->mem_type == AMDGPU_PL_DOORBELL) in amdgpu_res_cpu_visible()
448 if (res->mem_type != TTM_PL_VRAM) in amdgpu_res_cpu_visible()
451 amdgpu_res_first(res, 0, res->size, &cursor); in amdgpu_res_cpu_visible()
453 if ((cursor.start + cursor.size) > adev->gmc.visible_vram_size) in amdgpu_res_cpu_visible()
462 * amdgpu_res_copyable - Check that memory can be accessed by ttm_bo_move_memcpy
473 if (mem->mem_type == TTM_PL_VRAM && in amdgpu_res_copyable()
474 !(mem->placement & TTM_PL_FLAG_CONTIGUOUS)) in amdgpu_res_copyable()
481 * amdgpu_bo_move - Move a buffer object to a new memory location
492 struct ttm_resource *old_mem = bo->resource; in amdgpu_bo_move()
495 if (new_mem->mem_type == TTM_PL_TT || in amdgpu_bo_move()
496 new_mem->mem_type == AMDGPU_PL_PREEMPT) { in amdgpu_bo_move()
497 r = amdgpu_ttm_backend_bind(bo->bdev, bo->ttm, new_mem); in amdgpu_bo_move()
503 adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_bo_move()
505 if (!old_mem || (old_mem->mem_type == TTM_PL_SYSTEM && in amdgpu_bo_move()
506 bo->ttm == NULL)) { in amdgpu_bo_move()
511 if (old_mem->mem_type == TTM_PL_SYSTEM && in amdgpu_bo_move()
512 (new_mem->mem_type == TTM_PL_TT || in amdgpu_bo_move()
513 new_mem->mem_type == AMDGPU_PL_PREEMPT)) { in amdgpu_bo_move()
518 if ((old_mem->mem_type == TTM_PL_TT || in amdgpu_bo_move()
519 old_mem->mem_type == AMDGPU_PL_PREEMPT) && in amdgpu_bo_move()
520 new_mem->mem_type == TTM_PL_SYSTEM) { in amdgpu_bo_move()
525 amdgpu_ttm_backend_unbind(bo->bdev, bo->ttm); in amdgpu_bo_move()
527 ttm_resource_free(bo, &bo->resource); in amdgpu_bo_move()
532 if (old_mem->mem_type == AMDGPU_PL_GDS || in amdgpu_bo_move()
533 old_mem->mem_type == AMDGPU_PL_GWS || in amdgpu_bo_move()
534 old_mem->mem_type == AMDGPU_PL_OA || in amdgpu_bo_move()
535 old_mem->mem_type == AMDGPU_PL_DOORBELL || in amdgpu_bo_move()
536 new_mem->mem_type == AMDGPU_PL_GDS || in amdgpu_bo_move()
537 new_mem->mem_type == AMDGPU_PL_GWS || in amdgpu_bo_move()
538 new_mem->mem_type == AMDGPU_PL_OA || in amdgpu_bo_move()
539 new_mem->mem_type == AMDGPU_PL_DOORBELL) { in amdgpu_bo_move()
546 if (bo->type == ttm_bo_type_device && in amdgpu_bo_move()
547 new_mem->mem_type == TTM_PL_VRAM && in amdgpu_bo_move()
548 old_mem->mem_type != TTM_PL_VRAM) { in amdgpu_bo_move()
549 /* amdgpu_bo_fault_reserve_notify will re-set this if the CPU in amdgpu_bo_move()
552 abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; in amdgpu_bo_move()
555 if (adev->mman.buffer_funcs_enabled && in amdgpu_bo_move()
556 ((old_mem->mem_type == TTM_PL_SYSTEM && in amdgpu_bo_move()
557 new_mem->mem_type == TTM_PL_VRAM) || in amdgpu_bo_move()
558 (old_mem->mem_type == TTM_PL_VRAM && in amdgpu_bo_move()
559 new_mem->mem_type == TTM_PL_SYSTEM))) { in amdgpu_bo_move()
560 hop->fpfn = 0; in amdgpu_bo_move()
561 hop->lpfn = 0; in amdgpu_bo_move()
562 hop->mem_type = TTM_PL_TT; in amdgpu_bo_move()
563 hop->flags = TTM_PL_FLAG_TEMPORARY; in amdgpu_bo_move()
564 return -EMULTIHOP; in amdgpu_bo_move()
568 if (adev->mman.buffer_funcs_enabled) in amdgpu_bo_move()
571 r = -ENODEV; in amdgpu_bo_move()
588 atomic64_inc(&adev->num_evictions); in amdgpu_bo_move()
589 atomic64_add(bo->base.size, &adev->num_bytes_moved); in amdgpu_bo_move()
594 * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault
603 switch (mem->mem_type) { in amdgpu_ttm_io_mem_reserve()
611 mem->bus.offset = mem->start << PAGE_SHIFT; in amdgpu_ttm_io_mem_reserve()
613 if (adev->mman.aper_base_kaddr && in amdgpu_ttm_io_mem_reserve()
614 mem->placement & TTM_PL_FLAG_CONTIGUOUS) in amdgpu_ttm_io_mem_reserve()
615 mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr + in amdgpu_ttm_io_mem_reserve()
616 mem->bus.offset; in amdgpu_ttm_io_mem_reserve()
618 mem->bus.offset += adev->gmc.aper_base; in amdgpu_ttm_io_mem_reserve()
619 mem->bus.is_iomem = true; in amdgpu_ttm_io_mem_reserve()
622 mem->bus.offset = mem->start << PAGE_SHIFT; in amdgpu_ttm_io_mem_reserve()
623 mem->bus.offset += adev->doorbell.base; in amdgpu_ttm_io_mem_reserve()
624 mem->bus.is_iomem = true; in amdgpu_ttm_io_mem_reserve()
625 mem->bus.caching = ttm_uncached; in amdgpu_ttm_io_mem_reserve()
628 return -EINVAL; in amdgpu_ttm_io_mem_reserve()
636 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_ttm_io_mem_pfn()
639 amdgpu_res_first(bo->resource, (u64)page_offset << PAGE_SHIFT, 0, in amdgpu_ttm_io_mem_pfn()
642 if (bo->resource->mem_type == AMDGPU_PL_DOORBELL) in amdgpu_ttm_io_mem_pfn()
643 return ((uint64_t)(adev->doorbell.base + cursor.start)) >> PAGE_SHIFT; in amdgpu_ttm_io_mem_pfn()
645 return (adev->gmc.aper_base + cursor.start) >> PAGE_SHIFT; in amdgpu_ttm_io_mem_pfn()
649 * amdgpu_ttm_domain_start - Returns GPU start address
661 return adev->gmc.gart_start; in amdgpu_ttm_domain_start()
663 return adev->gmc.vram_start; in amdgpu_ttm_domain_start()
687 * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user
696 struct ttm_tt *ttm = bo->tbo.ttm; in amdgpu_ttm_tt_get_user_pages()
698 unsigned long start = gtt->userptr; in amdgpu_ttm_tt_get_user_pages()
707 mm = bo->notifier.mm; in amdgpu_ttm_tt_get_user_pages()
710 return -EFAULT; in amdgpu_ttm_tt_get_user_pages()
714 return -ESRCH; in amdgpu_ttm_tt_get_user_pages()
719 r = -EFAULT; in amdgpu_ttm_tt_get_user_pages()
722 if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) && in amdgpu_ttm_tt_get_user_pages()
723 vma->vm_file)) { in amdgpu_ttm_tt_get_user_pages()
724 r = -EPERM; in amdgpu_ttm_tt_get_user_pages()
729 r = amdgpu_hmm_range_get_pages(&bo->notifier, start, ttm->num_pages, in amdgpu_ttm_tt_get_user_pages()
741 /* amdgpu_ttm_tt_discard_user_pages - Discard range and pfn array allocations
748 if (gtt && gtt->userptr && range) in amdgpu_ttm_tt_discard_user_pages()
753 * amdgpu_ttm_tt_get_user_pages_done - stop HMM track the CPU page table change
763 if (!gtt || !gtt->userptr || !range) in amdgpu_ttm_tt_get_user_pages_done()
767 gtt->userptr, ttm->num_pages); in amdgpu_ttm_tt_get_user_pages_done()
769 WARN_ONCE(!range->hmm_pfns, "No user pages to check\n"); in amdgpu_ttm_tt_get_user_pages_done()
776 * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
786 for (i = 0; i < ttm->num_pages; ++i) in amdgpu_ttm_tt_set_user_pages()
787 ttm->pages[i] = pages ? pages[i] : NULL; in amdgpu_ttm_tt_set_user_pages()
791 * amdgpu_ttm_tt_pin_userptr - prepare the sg table with the user pages
800 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); in amdgpu_ttm_tt_pin_userptr()
806 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, in amdgpu_ttm_tt_pin_userptr()
807 (u64)ttm->num_pages << PAGE_SHIFT, in amdgpu_ttm_tt_pin_userptr()
813 r = dma_map_sgtable(adev->dev, ttm->sg, direction, 0); in amdgpu_ttm_tt_pin_userptr()
818 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, in amdgpu_ttm_tt_pin_userptr()
819 ttm->num_pages); in amdgpu_ttm_tt_pin_userptr()
824 sg_free_table(ttm->sg); in amdgpu_ttm_tt_pin_userptr()
826 kfree(ttm->sg); in amdgpu_ttm_tt_pin_userptr()
827 ttm->sg = NULL; in amdgpu_ttm_tt_pin_userptr()
832 * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages
839 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); in amdgpu_ttm_tt_unpin_userptr()
844 if (!ttm->sg || !ttm->sg->sgl) in amdgpu_ttm_tt_unpin_userptr()
848 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); in amdgpu_ttm_tt_unpin_userptr()
849 sg_free_table(ttm->sg); in amdgpu_ttm_tt_unpin_userptr()
863 uint64_t total_pages = ttm->num_pages; in amdgpu_ttm_gart_bind_gfx9_mqd()
864 int num_xcc = max(1U, adev->gfx.num_xcc_per_xcp); in amdgpu_ttm_gart_bind_gfx9_mqd()
875 gtt->offset + (page_idx << PAGE_SHIFT), in amdgpu_ttm_gart_bind_gfx9_mqd()
876 1, >t->ttm.dma_address[page_idx], flags); in amdgpu_ttm_gart_bind_gfx9_mqd()
878 * Ctrl pages - modify the memory type to NC (ctrl_flags) from in amdgpu_ttm_gart_bind_gfx9_mqd()
882 gtt->offset + ((page_idx + 1) << PAGE_SHIFT), in amdgpu_ttm_gart_bind_gfx9_mqd()
883 pages_per_xcc - 1, in amdgpu_ttm_gart_bind_gfx9_mqd()
884 >t->ttm.dma_address[page_idx + 1], in amdgpu_ttm_gart_bind_gfx9_mqd()
894 struct ttm_tt *ttm = tbo->ttm; in amdgpu_ttm_gart_bind()
900 if (abo->flags & AMDGPU_GEM_CREATE_CP_MQD_GFX9) { in amdgpu_ttm_gart_bind()
903 amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, in amdgpu_ttm_gart_bind()
904 gtt->ttm.dma_address, flags); in amdgpu_ttm_gart_bind()
906 gtt->bound = true; in amdgpu_ttm_gart_bind()
910 * amdgpu_ttm_backend_bind - Bind GTT memory
925 return -EINVAL; in amdgpu_ttm_backend_bind()
927 if (gtt->bound) in amdgpu_ttm_backend_bind()
930 if (gtt->userptr) { in amdgpu_ttm_backend_bind()
936 } else if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) { in amdgpu_ttm_backend_bind()
937 if (!ttm->sg) { in amdgpu_ttm_backend_bind()
941 attach = gtt->gobj->import_attach; in amdgpu_ttm_backend_bind()
946 ttm->sg = sgt; in amdgpu_ttm_backend_bind()
949 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, in amdgpu_ttm_backend_bind()
950 ttm->num_pages); in amdgpu_ttm_backend_bind()
953 if (!ttm->num_pages) { in amdgpu_ttm_backend_bind()
955 ttm->num_pages, bo_mem, ttm); in amdgpu_ttm_backend_bind()
958 if (bo_mem->mem_type != TTM_PL_TT || in amdgpu_ttm_backend_bind()
960 gtt->offset = AMDGPU_BO_INVALID_OFFSET; in amdgpu_ttm_backend_bind()
968 gtt->offset = (u64)bo_mem->start << PAGE_SHIFT; in amdgpu_ttm_backend_bind()
969 amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, in amdgpu_ttm_backend_bind()
970 gtt->ttm.dma_address, flags); in amdgpu_ttm_backend_bind()
971 gtt->bound = true; in amdgpu_ttm_backend_bind()
976 * amdgpu_ttm_alloc_gart - Make sure buffer object is accessible either
985 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_ttm_alloc_gart()
987 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(bo->ttm); in amdgpu_ttm_alloc_gart()
994 if (bo->resource->start != AMDGPU_BO_INVALID_OFFSET) in amdgpu_ttm_alloc_gart()
1005 placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT; in amdgpu_ttm_alloc_gart()
1007 placements.flags = bo->resource->placement; in amdgpu_ttm_alloc_gart()
1014 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, tmp); in amdgpu_ttm_alloc_gart()
1017 gtt->offset = (u64)tmp->start << PAGE_SHIFT; in amdgpu_ttm_alloc_gart()
1020 ttm_resource_free(bo, &bo->resource); in amdgpu_ttm_alloc_gart()
1027 * amdgpu_ttm_recover_gart - Rebind GTT pages
1034 struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev); in amdgpu_ttm_recover_gart()
1037 if (!tbo->ttm) in amdgpu_ttm_recover_gart()
1040 flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, tbo->resource); in amdgpu_ttm_recover_gart()
1045 * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages
1057 if (gtt->userptr) { in amdgpu_ttm_backend_unbind()
1059 } else if (ttm->sg && gtt->gobj->import_attach) { in amdgpu_ttm_backend_unbind()
1062 attach = gtt->gobj->import_attach; in amdgpu_ttm_backend_unbind()
1063 dma_buf_unmap_attachment(attach, ttm->sg, DMA_BIDIRECTIONAL); in amdgpu_ttm_backend_unbind()
1064 ttm->sg = NULL; in amdgpu_ttm_backend_unbind()
1067 if (!gtt->bound) in amdgpu_ttm_backend_unbind()
1070 if (gtt->offset == AMDGPU_BO_INVALID_OFFSET) in amdgpu_ttm_backend_unbind()
1074 amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages); in amdgpu_ttm_backend_unbind()
1075 gtt->bound = false; in amdgpu_ttm_backend_unbind()
1083 if (gtt->usertask) in amdgpu_ttm_backend_destroy()
1084 put_task_struct(gtt->usertask); in amdgpu_ttm_backend_destroy()
1086 ttm_tt_fini(>t->ttm); in amdgpu_ttm_backend_destroy()
1091 * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
1101 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_ttm_tt_create()
1110 gtt->gobj = &bo->base; in amdgpu_ttm_tt_create()
1111 if (adev->gmc.mem_partitions && abo->xcp_id >= 0) in amdgpu_ttm_tt_create()
1112 gtt->pool_id = KFD_XCP_MEM_ID(adev, abo->xcp_id); in amdgpu_ttm_tt_create()
1114 gtt->pool_id = abo->xcp_id; in amdgpu_ttm_tt_create()
1116 if (abo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) in amdgpu_ttm_tt_create()
1122 if (ttm_sg_tt_init(>t->ttm, bo, page_flags, caching)) { in amdgpu_ttm_tt_create()
1126 return >t->ttm; in amdgpu_ttm_tt_create()
1130 * amdgpu_ttm_tt_populate - Map GTT pages visible to the device
1146 if (gtt->userptr) { in amdgpu_ttm_tt_populate()
1147 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL); in amdgpu_ttm_tt_populate()
1148 if (!ttm->sg) in amdgpu_ttm_tt_populate()
1149 return -ENOMEM; in amdgpu_ttm_tt_populate()
1153 if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) in amdgpu_ttm_tt_populate()
1156 if (adev->mman.ttm_pools && gtt->pool_id >= 0) in amdgpu_ttm_tt_populate()
1157 pool = &adev->mman.ttm_pools[gtt->pool_id]; in amdgpu_ttm_tt_populate()
1159 pool = &adev->mman.bdev.pool; in amdgpu_ttm_tt_populate()
1164 for (i = 0; i < ttm->num_pages; ++i) in amdgpu_ttm_tt_populate()
1165 ttm->pages[i]->mapping = bdev->dev_mapping; in amdgpu_ttm_tt_populate()
1171 * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays
1186 if (gtt->userptr) { in amdgpu_ttm_tt_unpopulate()
1188 kfree(ttm->sg); in amdgpu_ttm_tt_unpopulate()
1189 ttm->sg = NULL; in amdgpu_ttm_tt_unpopulate()
1193 if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) in amdgpu_ttm_tt_unpopulate()
1196 for (i = 0; i < ttm->num_pages; ++i) in amdgpu_ttm_tt_unpopulate()
1197 ttm->pages[i]->mapping = NULL; in amdgpu_ttm_tt_unpopulate()
1201 if (adev->mman.ttm_pools && gtt->pool_id >= 0) in amdgpu_ttm_tt_unpopulate()
1202 pool = &adev->mman.ttm_pools[gtt->pool_id]; in amdgpu_ttm_tt_unpopulate()
1204 pool = &adev->mman.bdev.pool; in amdgpu_ttm_tt_unpopulate()
1210 * amdgpu_ttm_tt_get_userptr - Return the userptr GTT ttm_tt for the current
1221 if (!tbo->ttm) in amdgpu_ttm_tt_get_userptr()
1222 return -EINVAL; in amdgpu_ttm_tt_get_userptr()
1224 gtt = (void *)tbo->ttm; in amdgpu_ttm_tt_get_userptr()
1225 *user_addr = gtt->userptr; in amdgpu_ttm_tt_get_userptr()
1230 * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current
1246 if (!bo->ttm) { in amdgpu_ttm_tt_set_userptr()
1248 bo->ttm = amdgpu_ttm_tt_create(bo, 0); in amdgpu_ttm_tt_set_userptr()
1249 if (bo->ttm == NULL) in amdgpu_ttm_tt_set_userptr()
1250 return -ENOMEM; in amdgpu_ttm_tt_set_userptr()
1254 bo->ttm->page_flags |= TTM_TT_FLAG_EXTERNAL; in amdgpu_ttm_tt_set_userptr()
1256 gtt = ttm_to_amdgpu_ttm_tt(bo->ttm); in amdgpu_ttm_tt_set_userptr()
1257 gtt->userptr = addr; in amdgpu_ttm_tt_set_userptr()
1258 gtt->userflags = flags; in amdgpu_ttm_tt_set_userptr()
1260 if (gtt->usertask) in amdgpu_ttm_tt_set_userptr()
1261 put_task_struct(gtt->usertask); in amdgpu_ttm_tt_set_userptr()
1262 gtt->usertask = current->group_leader; in amdgpu_ttm_tt_set_userptr()
1263 get_task_struct(gtt->usertask); in amdgpu_ttm_tt_set_userptr()
1269 * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object
1278 if (gtt->usertask == NULL) in amdgpu_ttm_tt_get_usermm()
1281 return gtt->usertask->mm; in amdgpu_ttm_tt_get_usermm()
1285 * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an
1295 if (gtt == NULL || !gtt->userptr) in amdgpu_ttm_tt_affect_userptr()
1301 size = (unsigned long)gtt->ttm.num_pages * PAGE_SIZE; in amdgpu_ttm_tt_affect_userptr()
1302 if (gtt->userptr > end || gtt->userptr + size <= start) in amdgpu_ttm_tt_affect_userptr()
1306 *userptr = gtt->userptr; in amdgpu_ttm_tt_affect_userptr()
1311 * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr?
1317 if (gtt == NULL || !gtt->userptr) in amdgpu_ttm_tt_is_userptr()
1324 * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only?
1333 return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); in amdgpu_ttm_tt_is_readonly()
1337 * amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object
1348 if (mem && mem->mem_type != TTM_PL_SYSTEM) in amdgpu_ttm_tt_pde_flags()
1351 if (mem && (mem->mem_type == TTM_PL_TT || in amdgpu_ttm_tt_pde_flags()
1352 mem->mem_type == AMDGPU_PL_DOORBELL || in amdgpu_ttm_tt_pde_flags()
1353 mem->mem_type == AMDGPU_PL_PREEMPT)) { in amdgpu_ttm_tt_pde_flags()
1356 if (ttm->caching == ttm_cached) in amdgpu_ttm_tt_pde_flags()
1360 if (mem && mem->mem_type == TTM_PL_VRAM && in amdgpu_ttm_tt_pde_flags()
1361 mem->bus.caching == ttm_cached) in amdgpu_ttm_tt_pde_flags()
1368 * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object
1381 flags |= adev->gart.gart_pte_flags; in amdgpu_ttm_tt_pte_flags()
1391 * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer
1409 if (bo->resource->mem_type == TTM_PL_SYSTEM) in amdgpu_ttm_bo_eviction_valuable()
1412 if (bo->type == ttm_bo_type_kernel && in amdgpu_ttm_bo_eviction_valuable()
1420 dma_resv_for_each_fence(&resv_cursor, bo->base.resv, in amdgpu_ttm_bo_eviction_valuable()
1422 if (amdkfd_fence_check_mm(f, current->mm) && in amdgpu_ttm_bo_eviction_valuable()
1423 !(place->flags & TTM_PL_FLAG_CONTIGUOUS)) in amdgpu_ttm_bo_eviction_valuable()
1434 if (bo->resource->mem_type == AMDGPU_PL_PREEMPT) in amdgpu_ttm_bo_eviction_valuable()
1437 if (bo->resource->mem_type == TTM_PL_TT && in amdgpu_ttm_bo_eviction_valuable()
1449 uint64_t bytes = 4 - (pos & 0x3); in amdgpu_ttm_vram_mm_access()
1455 mask &= 0xffffffff >> (bytes - size) * 8; in amdgpu_ttm_vram_mm_access()
1475 size -= bytes; in amdgpu_ttm_vram_mm_access()
1484 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); in amdgpu_ttm_access_memory_sdma()
1493 return -EINVAL; in amdgpu_ttm_access_memory_sdma()
1495 if (!adev->mman.sdma_access_ptr) in amdgpu_ttm_access_memory_sdma()
1496 return -EACCES; in amdgpu_ttm_access_memory_sdma()
1499 return -ENODEV; in amdgpu_ttm_access_memory_sdma()
1502 memcpy(adev->mman.sdma_access_ptr, buf, len); in amdgpu_ttm_access_memory_sdma()
1504 num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); in amdgpu_ttm_access_memory_sdma()
1505 r = amdgpu_job_alloc_with_ib(adev, &adev->mman.high_pr, in amdgpu_ttm_access_memory_sdma()
1512 amdgpu_res_first(abo->tbo.resource, offset, len, &src_mm); in amdgpu_ttm_access_memory_sdma()
1513 src_addr = amdgpu_ttm_domain_start(adev, bo->resource->mem_type) + in amdgpu_ttm_access_memory_sdma()
1515 dst_addr = amdgpu_bo_gpu_offset(adev->mman.sdma_access_bo); in amdgpu_ttm_access_memory_sdma()
1519 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, dst_addr, in amdgpu_ttm_access_memory_sdma()
1522 amdgpu_ring_pad_ib(adev->mman.buffer_funcs_ring, &job->ibs[0]); in amdgpu_ttm_access_memory_sdma()
1523 WARN_ON(job->ibs[0].length_dw > num_dw); in amdgpu_ttm_access_memory_sdma()
1527 if (!dma_fence_wait_timeout(fence, false, adev->sdma_timeout)) in amdgpu_ttm_access_memory_sdma()
1528 r = -ETIMEDOUT; in amdgpu_ttm_access_memory_sdma()
1532 memcpy(buf, adev->mman.sdma_access_ptr, len); in amdgpu_ttm_access_memory_sdma()
1539 * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object.
1555 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); in amdgpu_ttm_access_memory()
1559 if (bo->resource->mem_type != TTM_PL_VRAM) in amdgpu_ttm_access_memory()
1560 return -EIO; in amdgpu_ttm_access_memory()
1566 amdgpu_res_first(bo->resource, offset, len, &cursor); in amdgpu_ttm_access_memory()
1572 size -= count; in amdgpu_ttm_access_memory()
1574 /* using MM to access rest vram and handle un-aligned address */ in amdgpu_ttm_access_memory()
1613 * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram
1621 amdgpu_bo_free_kernel(&adev->mman.fw_vram_usage_reserved_bo, in amdgpu_ttm_fw_reserve_vram_fini()
1622 NULL, &adev->mman.fw_vram_usage_va); in amdgpu_ttm_fw_reserve_vram_fini()
1629 * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
1637 amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo, in amdgpu_ttm_drv_reserve_vram_fini()
1639 &adev->mman.drv_vram_usage_va); in amdgpu_ttm_drv_reserve_vram_fini()
1643 * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
1651 uint64_t vram_size = adev->gmc.visible_vram_size; in amdgpu_ttm_fw_reserve_vram_init()
1653 adev->mman.fw_vram_usage_va = NULL; in amdgpu_ttm_fw_reserve_vram_init()
1654 adev->mman.fw_vram_usage_reserved_bo = NULL; in amdgpu_ttm_fw_reserve_vram_init()
1656 if (adev->mman.fw_vram_usage_size == 0 || in amdgpu_ttm_fw_reserve_vram_init()
1657 adev->mman.fw_vram_usage_size > vram_size) in amdgpu_ttm_fw_reserve_vram_init()
1661 adev->mman.fw_vram_usage_start_offset, in amdgpu_ttm_fw_reserve_vram_init()
1662 adev->mman.fw_vram_usage_size, in amdgpu_ttm_fw_reserve_vram_init()
1663 &adev->mman.fw_vram_usage_reserved_bo, in amdgpu_ttm_fw_reserve_vram_init()
1664 &adev->mman.fw_vram_usage_va); in amdgpu_ttm_fw_reserve_vram_init()
1668 * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
1676 u64 vram_size = adev->gmc.visible_vram_size; in amdgpu_ttm_drv_reserve_vram_init()
1678 adev->mman.drv_vram_usage_va = NULL; in amdgpu_ttm_drv_reserve_vram_init()
1679 adev->mman.drv_vram_usage_reserved_bo = NULL; in amdgpu_ttm_drv_reserve_vram_init()
1681 if (adev->mman.drv_vram_usage_size == 0 || in amdgpu_ttm_drv_reserve_vram_init()
1682 adev->mman.drv_vram_usage_size > vram_size) in amdgpu_ttm_drv_reserve_vram_init()
1686 adev->mman.drv_vram_usage_start_offset, in amdgpu_ttm_drv_reserve_vram_init()
1687 adev->mman.drv_vram_usage_size, in amdgpu_ttm_drv_reserve_vram_init()
1688 &adev->mman.drv_vram_usage_reserved_bo, in amdgpu_ttm_drv_reserve_vram_init()
1689 &adev->mman.drv_vram_usage_va); in amdgpu_ttm_drv_reserve_vram_init()
1697 * amdgpu_ttm_training_reserve_vram_fini - free memory training reserved vram
1705 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; in amdgpu_ttm_training_reserve_vram_fini()
1707 ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT; in amdgpu_ttm_training_reserve_vram_fini()
1708 amdgpu_bo_free_kernel(&ctx->c2p_bo, NULL, NULL); in amdgpu_ttm_training_reserve_vram_fini()
1709 ctx->c2p_bo = NULL; in amdgpu_ttm_training_reserve_vram_fini()
1717 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; in amdgpu_ttm_training_data_block_init()
1721 ctx->c2p_train_data_offset = in amdgpu_ttm_training_data_block_init()
1722 ALIGN((adev->gmc.mc_vram_size - reserve_size - SZ_1M), SZ_1M); in amdgpu_ttm_training_data_block_init()
1723 ctx->p2c_train_data_offset = in amdgpu_ttm_training_data_block_init()
1724 (adev->gmc.mc_vram_size - GDDR6_MEM_TRAINING_OFFSET); in amdgpu_ttm_training_data_block_init()
1725 ctx->train_data_size = in amdgpu_ttm_training_data_block_init()
1729 ctx->train_data_size, in amdgpu_ttm_training_data_block_init()
1730 ctx->p2c_train_data_offset, in amdgpu_ttm_training_data_block_init()
1731 ctx->c2p_train_data_offset); in amdgpu_ttm_training_data_block_init()
1735 * reserve TMR memory at the top of VRAM which holds
1740 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; in amdgpu_ttm_reserve_tmr()
1745 if (adev->bios && !amdgpu_sriov_vf(adev)) { in amdgpu_ttm_reserve_tmr()
1753 * Query reserved tmr size through atom firmwareinfo for Sienna_Cichlid and onwards for all in amdgpu_ttm_reserve_tmr()
1756 * Otherwise, fallback to legacy approach to check and reserve tmr block for ip in amdgpu_ttm_reserve_tmr()
1759 if (adev->bios) in amdgpu_ttm_reserve_tmr()
1763 if (!adev->bios && in amdgpu_ttm_reserve_tmr()
1771 /* reserve vram for mem train according to TMR location */ in amdgpu_ttm_reserve_tmr()
1774 ctx->c2p_train_data_offset, in amdgpu_ttm_reserve_tmr()
1775 ctx->train_data_size, in amdgpu_ttm_reserve_tmr()
1776 &ctx->c2p_bo, in amdgpu_ttm_reserve_tmr()
1783 ctx->init = PSP_MEM_TRAIN_RESERVE_SUCCESS; in amdgpu_ttm_reserve_tmr()
1786 if (!adev->gmc.is_app_apu) { in amdgpu_ttm_reserve_tmr()
1788 adev, adev->gmc.real_vram_size - reserve_size, in amdgpu_ttm_reserve_tmr()
1789 reserve_size, &adev->mman.fw_reserved_memory, NULL); in amdgpu_ttm_reserve_tmr()
1791 DRM_ERROR("alloc tmr failed(%d)!\n", ret); in amdgpu_ttm_reserve_tmr()
1792 amdgpu_bo_free_kernel(&adev->mman.fw_reserved_memory, in amdgpu_ttm_reserve_tmr()
1797 DRM_DEBUG_DRIVER("backdoor fw loading path for PSP TMR, no reservation needed\n"); in amdgpu_ttm_reserve_tmr()
1807 if (!adev->gmc.is_app_apu || !adev->gmc.num_mem_partitions) in amdgpu_ttm_pools_init()
1810 adev->mman.ttm_pools = kcalloc(adev->gmc.num_mem_partitions, in amdgpu_ttm_pools_init()
1811 sizeof(*adev->mman.ttm_pools), in amdgpu_ttm_pools_init()
1813 if (!adev->mman.ttm_pools) in amdgpu_ttm_pools_init()
1814 return -ENOMEM; in amdgpu_ttm_pools_init()
1816 for (i = 0; i < adev->gmc.num_mem_partitions; i++) { in amdgpu_ttm_pools_init()
1817 ttm_pool_init(&adev->mman.ttm_pools[i], adev->dev, in amdgpu_ttm_pools_init()
1818 adev->gmc.mem_partitions[i].numa.node, in amdgpu_ttm_pools_init()
1828 if (!adev->gmc.is_app_apu || !adev->mman.ttm_pools) in amdgpu_ttm_pools_fini()
1831 for (i = 0; i < adev->gmc.num_mem_partitions; i++) in amdgpu_ttm_pools_fini()
1832 ttm_pool_fini(&adev->mman.ttm_pools[i]); in amdgpu_ttm_pools_fini()
1834 kfree(adev->mman.ttm_pools); in amdgpu_ttm_pools_fini()
1835 adev->mman.ttm_pools = NULL; in amdgpu_ttm_pools_fini()
1839 * amdgpu_ttm_init - Init the memory management (ttm) as well as various
1844 * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which
1852 mutex_init(&adev->mman.gtt_window_lock); in amdgpu_ttm_init()
1854 dma_set_max_seg_size(adev->dev, UINT_MAX); in amdgpu_ttm_init()
1856 r = ttm_device_init(&adev->mman.bdev, &amdgpu_bo_driver, adev->dev, in amdgpu_ttm_init()
1857 adev_to_drm(adev)->anon_inode->i_mapping, in amdgpu_ttm_init()
1858 adev_to_drm(adev)->vma_offset_manager, in amdgpu_ttm_init()
1859 adev->need_swiotlb, in amdgpu_ttm_init()
1860 dma_addressing_limited(adev->dev)); in amdgpu_ttm_init()
1871 adev->mman.initialized = true; in amdgpu_ttm_init()
1884 if (adev->gmc.xgmi.connected_to_cpu) in amdgpu_ttm_init()
1885 adev->mman.aper_base_kaddr = ioremap_cache(adev->gmc.aper_base, in amdgpu_ttm_init()
1886 adev->gmc.visible_vram_size); in amdgpu_ttm_init()
1888 else if (adev->gmc.is_app_apu) in amdgpu_ttm_init()
1893 adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base, in amdgpu_ttm_init()
1894 adev->gmc.visible_vram_size); in amdgpu_ttm_init()
1918 if (adev->mman.discovery_bin) { in amdgpu_ttm_init()
1925 * This is used for VGA emulation and pre-OS scanout buffers to in amdgpu_ttm_init()
1926 * avoid display artifacts while transitioning between pre-OS in amdgpu_ttm_init()
1929 if (!adev->gmc.is_app_apu) { in amdgpu_ttm_init()
1931 adev->mman.stolen_vga_size, in amdgpu_ttm_init()
1932 &adev->mman.stolen_vga_memory, in amdgpu_ttm_init()
1937 r = amdgpu_bo_create_kernel_at(adev, adev->mman.stolen_vga_size, in amdgpu_ttm_init()
1938 adev->mman.stolen_extended_size, in amdgpu_ttm_init()
1939 &adev->mman.stolen_extended_memory, in amdgpu_ttm_init()
1946 adev->mman.stolen_reserved_offset, in amdgpu_ttm_init()
1947 adev->mman.stolen_reserved_size, in amdgpu_ttm_init()
1948 &adev->mman.stolen_reserved_memory, in amdgpu_ttm_init()
1957 (unsigned int)(adev->gmc.real_vram_size / (1024 * 1024))); in amdgpu_ttm_init()
1962 if (amdgpu_gtt_size == -1) in amdgpu_ttm_init()
1977 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_DOORBELL, adev->doorbell.size / PAGE_SIZE); in amdgpu_ttm_init()
1997 /* Initialize various on-chip memory pools */ in amdgpu_ttm_init()
1998 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GDS, adev->gds.gds_size); in amdgpu_ttm_init()
2004 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GWS, adev->gds.gws_size); in amdgpu_ttm_init()
2010 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_OA, adev->gds.oa_size); in amdgpu_ttm_init()
2017 &adev->mman.sdma_access_bo, NULL, in amdgpu_ttm_init()
2018 &adev->mman.sdma_access_ptr)) in amdgpu_ttm_init()
2025 * amdgpu_ttm_fini - De-initialize the TTM memory pools
2031 if (!adev->mman.initialized) in amdgpu_ttm_fini()
2038 if (!adev->gmc.is_app_apu) { in amdgpu_ttm_fini()
2039 amdgpu_bo_free_kernel(&adev->mman.stolen_vga_memory, NULL, NULL); in amdgpu_ttm_fini()
2040 amdgpu_bo_free_kernel(&adev->mman.stolen_extended_memory, NULL, NULL); in amdgpu_ttm_fini()
2042 amdgpu_bo_free_kernel(&adev->mman.fw_reserved_memory, NULL, in amdgpu_ttm_fini()
2044 if (adev->mman.stolen_reserved_size) in amdgpu_ttm_fini()
2045 amdgpu_bo_free_kernel(&adev->mman.stolen_reserved_memory, in amdgpu_ttm_fini()
2048 amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL, in amdgpu_ttm_fini()
2049 &adev->mman.sdma_access_ptr); in amdgpu_ttm_fini()
2055 if (adev->mman.aper_base_kaddr) in amdgpu_ttm_fini()
2056 iounmap(adev->mman.aper_base_kaddr); in amdgpu_ttm_fini()
2057 adev->mman.aper_base_kaddr = NULL; in amdgpu_ttm_fini()
2065 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS); in amdgpu_ttm_fini()
2066 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GWS); in amdgpu_ttm_fini()
2067 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_OA); in amdgpu_ttm_fini()
2068 ttm_device_fini(&adev->mman.bdev); in amdgpu_ttm_fini()
2069 adev->mman.initialized = false; in amdgpu_ttm_fini()
2074 * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions
2084 struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); in amdgpu_ttm_set_buffer_funcs_status()
2088 if (!adev->mman.initialized || amdgpu_in_reset(adev) || in amdgpu_ttm_set_buffer_funcs_status()
2089 adev->mman.buffer_funcs_enabled == enable || adev->gmc.is_app_apu) in amdgpu_ttm_set_buffer_funcs_status()
2096 ring = adev->mman.buffer_funcs_ring; in amdgpu_ttm_set_buffer_funcs_status()
2097 sched = &ring->sched; in amdgpu_ttm_set_buffer_funcs_status()
2098 r = drm_sched_entity_init(&adev->mman.high_pr, in amdgpu_ttm_set_buffer_funcs_status()
2107 r = drm_sched_entity_init(&adev->mman.low_pr, in amdgpu_ttm_set_buffer_funcs_status()
2116 drm_sched_entity_destroy(&adev->mman.high_pr); in amdgpu_ttm_set_buffer_funcs_status()
2117 drm_sched_entity_destroy(&adev->mman.low_pr); in amdgpu_ttm_set_buffer_funcs_status()
2118 dma_fence_put(man->move); in amdgpu_ttm_set_buffer_funcs_status()
2119 man->move = NULL; in amdgpu_ttm_set_buffer_funcs_status()
2124 size = adev->gmc.real_vram_size; in amdgpu_ttm_set_buffer_funcs_status()
2126 size = adev->gmc.visible_vram_size; in amdgpu_ttm_set_buffer_funcs_status()
2127 man->size = size; in amdgpu_ttm_set_buffer_funcs_status()
2128 adev->mman.buffer_funcs_enabled = enable; in amdgpu_ttm_set_buffer_funcs_status()
2133 drm_sched_entity_destroy(&adev->mman.high_pr); in amdgpu_ttm_set_buffer_funcs_status()
2148 struct drm_sched_entity *entity = delayed ? &adev->mman.low_pr : in amdgpu_ttm_prepare_job()
2149 &adev->mman.high_pr; in amdgpu_ttm_prepare_job()
2157 (*job)->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo ? in amdgpu_ttm_prepare_job()
2158 adev->gmc.pdb0_bo : in amdgpu_ttm_prepare_job()
2159 adev->gart.bo); in amdgpu_ttm_prepare_job()
2160 (*job)->vm_needs_flush = true; in amdgpu_ttm_prepare_job()
2165 return drm_sched_job_add_resv_dependencies(&(*job)->base, resv, in amdgpu_ttm_prepare_job()
2175 struct amdgpu_device *adev = ring->adev; in amdgpu_copy_buffer()
2182 if (!direct_submit && !ring->sched.ready) { in amdgpu_copy_buffer()
2184 return -EINVAL; in amdgpu_copy_buffer()
2187 max_bytes = adev->mman.buffer_funcs->copy_max_bytes; in amdgpu_copy_buffer()
2189 num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->copy_num_dw, 8); in amdgpu_copy_buffer()
2198 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset, in amdgpu_copy_buffer()
2202 byte_count -= cur_size_in_bytes; in amdgpu_copy_buffer()
2205 amdgpu_ring_pad_ib(ring, &job->ibs[0]); in amdgpu_copy_buffer()
2206 WARN_ON(job->ibs[0].length_dw > num_dw); in amdgpu_copy_buffer()
2228 struct amdgpu_device *adev = ring->adev; in amdgpu_ttm_fill_mem()
2235 max_bytes = adev->mman.buffer_funcs->fill_max_bytes; in amdgpu_ttm_fill_mem()
2237 num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->fill_num_dw, 8); in amdgpu_ttm_fill_mem()
2246 amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data, dst_addr, in amdgpu_ttm_fill_mem()
2250 byte_count -= cur_size; in amdgpu_ttm_fill_mem()
2253 amdgpu_ring_pad_ib(ring, &job->ibs[0]); in amdgpu_ttm_fill_mem()
2254 WARN_ON(job->ibs[0].length_dw > num_dw); in amdgpu_ttm_fill_mem()
2260 * amdgpu_ttm_clear_buffer - clear memory buffers
2274 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); in amdgpu_ttm_clear_buffer()
2275 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; in amdgpu_ttm_clear_buffer()
2280 if (!adev->mman.buffer_funcs_enabled) in amdgpu_ttm_clear_buffer()
2281 return -EINVAL; in amdgpu_ttm_clear_buffer()
2284 return -EINVAL; in amdgpu_ttm_clear_buffer()
2288 amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &cursor); in amdgpu_ttm_clear_buffer()
2290 mutex_lock(&adev->mman.gtt_window_lock); in amdgpu_ttm_clear_buffer()
2303 r = amdgpu_ttm_map_buffer(&bo->tbo, bo->tbo.resource, &cursor, in amdgpu_ttm_clear_buffer()
2319 mutex_unlock(&adev->mman.gtt_window_lock); in amdgpu_ttm_clear_buffer()
2330 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); in amdgpu_fill_buffer()
2331 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; in amdgpu_fill_buffer()
2336 if (!adev->mman.buffer_funcs_enabled) { in amdgpu_fill_buffer()
2338 return -EINVAL; in amdgpu_fill_buffer()
2341 amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &dst); in amdgpu_fill_buffer()
2343 mutex_lock(&adev->mman.gtt_window_lock); in amdgpu_fill_buffer()
2351 r = amdgpu_ttm_map_buffer(&bo->tbo, bo->tbo.resource, &dst, in amdgpu_fill_buffer()
2367 mutex_unlock(&adev->mman.gtt_window_lock); in amdgpu_fill_buffer()
2375 * amdgpu_ttm_evict_resources - evict memory buffers
2394 man = ttm_manager_type(&adev->mman.bdev, mem_type); in amdgpu_ttm_evict_resources()
2398 return -EINVAL; in amdgpu_ttm_evict_resources()
2401 return ttm_resource_manager_evict_all(&adev->mman.bdev, man); in amdgpu_ttm_evict_resources()
2408 struct amdgpu_device *adev = m->private; in amdgpu_ttm_page_pool_show()
2410 return ttm_pool_debugfs(&adev->mman.bdev.pool, m); in amdgpu_ttm_page_pool_show()
2416 * amdgpu_ttm_vram_read - Linear read access to VRAM
2423 struct amdgpu_device *adev = file_inode(f)->i_private; in amdgpu_ttm_vram_read()
2427 return -EINVAL; in amdgpu_ttm_vram_read()
2429 if (*pos >= adev->gmc.mc_vram_size) in amdgpu_ttm_vram_read()
2430 return -ENXIO; in amdgpu_ttm_vram_read()
2432 size = min(size, (size_t)(adev->gmc.mc_vram_size - *pos)); in amdgpu_ttm_vram_read()
2439 return -EFAULT; in amdgpu_ttm_vram_read()
2444 size -= bytes; in amdgpu_ttm_vram_read()
2451 * amdgpu_ttm_vram_write - Linear write access to VRAM
2458 struct amdgpu_device *adev = file_inode(f)->i_private; in amdgpu_ttm_vram_write()
2463 return -EINVAL; in amdgpu_ttm_vram_write()
2465 if (*pos >= adev->gmc.mc_vram_size) in amdgpu_ttm_vram_write()
2466 return -ENXIO; in amdgpu_ttm_vram_write()
2471 if (*pos >= adev->gmc.mc_vram_size) in amdgpu_ttm_vram_write()
2483 size -= 4; in amdgpu_ttm_vram_write()
2497 * amdgpu_iomem_read - Virtual read access to GPU mapped memory
2506 struct amdgpu_device *adev = file_inode(f)->i_private; in amdgpu_iomem_read()
2512 dom = iommu_get_domain_for_dev(adev->dev); in amdgpu_iomem_read()
2517 size_t bytes = PAGE_SIZE - off; in amdgpu_iomem_read()
2532 return -EPERM; in amdgpu_iomem_read()
2535 if (p->mapping != adev->mman.bdev.dev_mapping) in amdgpu_iomem_read()
2536 return -EPERM; in amdgpu_iomem_read()
2542 return -EFAULT; in amdgpu_iomem_read()
2544 size -= bytes; in amdgpu_iomem_read()
2553 * amdgpu_iomem_write - Virtual write access to GPU mapped memory
2562 struct amdgpu_device *adev = file_inode(f)->i_private; in amdgpu_iomem_write()
2567 dom = iommu_get_domain_for_dev(adev->dev); in amdgpu_iomem_write()
2572 size_t bytes = PAGE_SIZE - off; in amdgpu_iomem_write()
2583 return -EPERM; in amdgpu_iomem_write()
2586 if (p->mapping != adev->mman.bdev.dev_mapping) in amdgpu_iomem_write()
2587 return -EPERM; in amdgpu_iomem_write()
2593 return -EFAULT; in amdgpu_iomem_write()
2595 size -= bytes; in amdgpu_iomem_write()
2615 struct drm_minor *minor = adev_to_drm(adev)->primary; in amdgpu_ttm_debugfs_init()
2616 struct dentry *root = minor->debugfs_root; in amdgpu_ttm_debugfs_init()
2619 &amdgpu_ttm_vram_fops, adev->gmc.mc_vram_size); in amdgpu_ttm_debugfs_init()
2624 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()
2627 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()
2630 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()
2633 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()
2636 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, in amdgpu_ttm_debugfs_init()