Lines Matching refs:ttm
69 struct ttm_tt *ttm,
72 struct ttm_tt *ttm);
255 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, mem); in amdgpu_ttm_map_buffer()
264 dma_addr = &bo->ttm->dma_address[mm_cur->start >> PAGE_SHIFT]; in amdgpu_ttm_map_buffer()
510 r = amdgpu_ttm_backend_bind(bo->bdev, bo->ttm, new_mem); in amdgpu_bo_move()
519 bo->ttm == NULL)) { in amdgpu_bo_move()
538 amdgpu_ttm_backend_unbind(bo->bdev, bo->ttm); in amdgpu_bo_move()
696 struct ttm_tt ttm; member
706 #define ttm_to_amdgpu_ttm_tt(ptr) container_of(ptr, struct amdgpu_ttm_tt, ttm)
720 struct ttm_tt *ttm = bo->tbo.ttm; in amdgpu_ttm_tt_get_user_pages() local
721 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); in amdgpu_ttm_tt_get_user_pages()
749 readonly = amdgpu_ttm_tt_is_readonly(ttm); in amdgpu_ttm_tt_get_user_pages()
750 r = amdgpu_hmm_range_get_pages(&bo->notifier, start, ttm->num_pages, in amdgpu_ttm_tt_get_user_pages()
771 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct amdgpu_hmm_range *range) in amdgpu_ttm_tt_set_user_pages() argument
775 for (i = 0; i < ttm->num_pages; ++i) in amdgpu_ttm_tt_set_user_pages()
776 ttm->pages[i] = range ? hmm_pfn_to_page(range->hmm_range.hmm_pfns[i]) : NULL; in amdgpu_ttm_tt_set_user_pages()
785 struct ttm_tt *ttm) in amdgpu_ttm_tt_pin_userptr() argument
788 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); in amdgpu_ttm_tt_pin_userptr()
795 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, in amdgpu_ttm_tt_pin_userptr()
796 (u64)ttm->num_pages << PAGE_SHIFT, in amdgpu_ttm_tt_pin_userptr()
802 r = dma_map_sgtable(adev->dev, ttm->sg, direction, 0); in amdgpu_ttm_tt_pin_userptr()
807 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, in amdgpu_ttm_tt_pin_userptr()
808 ttm->num_pages); in amdgpu_ttm_tt_pin_userptr()
813 sg_free_table(ttm->sg); in amdgpu_ttm_tt_pin_userptr()
815 kfree(ttm->sg); in amdgpu_ttm_tt_pin_userptr()
816 ttm->sg = NULL; in amdgpu_ttm_tt_pin_userptr()
824 struct ttm_tt *ttm) in amdgpu_ttm_tt_unpin_userptr() argument
827 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); in amdgpu_ttm_tt_unpin_userptr()
833 if (!ttm->sg || !ttm->sg->sgl) in amdgpu_ttm_tt_unpin_userptr()
837 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); in amdgpu_ttm_tt_unpin_userptr()
838 sg_free_table(ttm->sg); in amdgpu_ttm_tt_unpin_userptr()
849 struct ttm_tt *ttm, uint64_t flags) in amdgpu_ttm_gart_bind_gfx9_mqd() argument
851 struct amdgpu_ttm_tt *gtt = (void *)ttm; in amdgpu_ttm_gart_bind_gfx9_mqd()
852 uint64_t total_pages = ttm->num_pages; in amdgpu_ttm_gart_bind_gfx9_mqd()
865 1, >t->ttm.dma_address[page_idx], flags); in amdgpu_ttm_gart_bind_gfx9_mqd()
873 >t->ttm.dma_address[page_idx + 1], in amdgpu_ttm_gart_bind_gfx9_mqd()
883 struct ttm_tt *ttm = tbo->ttm; in amdgpu_ttm_gart_bind() local
884 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); in amdgpu_ttm_gart_bind()
890 amdgpu_ttm_gart_bind_gfx9_mqd(adev, ttm, flags); in amdgpu_ttm_gart_bind()
892 amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, in amdgpu_ttm_gart_bind()
893 gtt->ttm.dma_address, flags); in amdgpu_ttm_gart_bind()
905 struct ttm_tt *ttm, in amdgpu_ttm_backend_bind() argument
909 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); in amdgpu_ttm_backend_bind()
920 r = amdgpu_ttm_tt_pin_userptr(bdev, ttm); in amdgpu_ttm_backend_bind()
925 } else if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) { in amdgpu_ttm_backend_bind()
926 if (!ttm->sg) { in amdgpu_ttm_backend_bind()
935 ttm->sg = sgt; in amdgpu_ttm_backend_bind()
938 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, in amdgpu_ttm_backend_bind()
939 ttm->num_pages); in amdgpu_ttm_backend_bind()
942 if (!ttm->num_pages) { in amdgpu_ttm_backend_bind()
944 ttm->num_pages, bo_mem, ttm); in amdgpu_ttm_backend_bind()
954 flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem); in amdgpu_ttm_backend_bind()
958 amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, in amdgpu_ttm_backend_bind()
959 gtt->ttm.dma_address, flags); in amdgpu_ttm_backend_bind()
976 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(bo->ttm); in amdgpu_ttm_alloc_gart()
1003 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, tmp); in amdgpu_ttm_alloc_gart()
1026 if (!tbo->ttm) in amdgpu_ttm_recover_gart()
1029 flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, tbo->resource); in amdgpu_ttm_recover_gart()
1040 struct ttm_tt *ttm) in amdgpu_ttm_backend_unbind() argument
1043 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); in amdgpu_ttm_backend_unbind()
1047 amdgpu_ttm_tt_unpin_userptr(bdev, ttm); in amdgpu_ttm_backend_unbind()
1048 } else if (ttm->sg && drm_gem_is_imported(gtt->gobj)) { in amdgpu_ttm_backend_unbind()
1052 dma_buf_unmap_attachment(attach, ttm->sg, DMA_BIDIRECTIONAL); in amdgpu_ttm_backend_unbind()
1053 ttm->sg = NULL; in amdgpu_ttm_backend_unbind()
1063 amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages); in amdgpu_ttm_backend_unbind()
1068 struct ttm_tt *ttm) in amdgpu_ttm_backend_destroy() argument
1070 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); in amdgpu_ttm_backend_destroy()
1075 ttm_tt_fini(>t->ttm); in amdgpu_ttm_backend_destroy()
1191 if (ttm_sg_tt_init(>t->ttm, bo, page_flags, caching)) { in amdgpu_ttm_tt_create()
1195 return >t->ttm; in amdgpu_ttm_tt_create()
1205 struct ttm_tt *ttm, in amdgpu_ttm_tt_populate() argument
1209 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); in amdgpu_ttm_tt_populate()
1216 ttm->sg = kzalloc_obj(struct sg_table); in amdgpu_ttm_tt_populate()
1217 if (!ttm->sg) in amdgpu_ttm_tt_populate()
1222 if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) in amdgpu_ttm_tt_populate()
1229 ret = ttm_pool_alloc(pool, ttm, ctx); in amdgpu_ttm_tt_populate()
1233 for (i = 0; i < ttm->num_pages; ++i) in amdgpu_ttm_tt_populate()
1234 ttm->pages[i]->mapping = bdev->dev_mapping; in amdgpu_ttm_tt_populate()
1246 struct ttm_tt *ttm) in amdgpu_ttm_tt_unpopulate() argument
1248 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); in amdgpu_ttm_tt_unpopulate()
1253 amdgpu_ttm_backend_unbind(bdev, ttm); in amdgpu_ttm_tt_unpopulate()
1256 amdgpu_ttm_tt_set_user_pages(ttm, NULL); in amdgpu_ttm_tt_unpopulate()
1257 kfree(ttm->sg); in amdgpu_ttm_tt_unpopulate()
1258 ttm->sg = NULL; in amdgpu_ttm_tt_unpopulate()
1262 if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) in amdgpu_ttm_tt_unpopulate()
1265 for (i = 0; i < ttm->num_pages; ++i) in amdgpu_ttm_tt_unpopulate()
1266 ttm->pages[i]->mapping = NULL; in amdgpu_ttm_tt_unpopulate()
1275 return ttm_pool_free(pool, ttm); in amdgpu_ttm_tt_unpopulate()
1290 if (!tbo->ttm) in amdgpu_ttm_tt_get_userptr()
1293 gtt = (void *)tbo->ttm; in amdgpu_ttm_tt_get_userptr()
1315 if (!bo->ttm) { in amdgpu_ttm_tt_set_userptr()
1317 bo->ttm = amdgpu_ttm_tt_create(bo, 0); in amdgpu_ttm_tt_set_userptr()
1318 if (bo->ttm == NULL) in amdgpu_ttm_tt_set_userptr()
1323 bo->ttm->page_flags |= TTM_TT_FLAG_EXTERNAL; in amdgpu_ttm_tt_set_userptr()
1325 gtt = ttm_to_amdgpu_ttm_tt(bo->ttm); in amdgpu_ttm_tt_set_userptr()
1340 struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm) in amdgpu_ttm_tt_get_usermm() argument
1342 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); in amdgpu_ttm_tt_get_usermm()
1358 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, in amdgpu_ttm_tt_affect_userptr() argument
1361 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); in amdgpu_ttm_tt_affect_userptr()
1370 size = (unsigned long)gtt->ttm.num_pages * PAGE_SIZE; in amdgpu_ttm_tt_affect_userptr()
1382 bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm) in amdgpu_ttm_tt_is_userptr() argument
1384 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); in amdgpu_ttm_tt_is_userptr()
1395 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm) in amdgpu_ttm_tt_is_readonly() argument
1397 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); in amdgpu_ttm_tt_is_readonly()
1413 uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_resource *mem) in amdgpu_ttm_tt_pde_flags() argument
1426 if (ttm && ttm->caching == ttm_cached) in amdgpu_ttm_tt_pde_flags()
1446 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm, in amdgpu_ttm_tt_pte_flags() argument
1449 uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem); in amdgpu_ttm_tt_pte_flags()
1454 if (!amdgpu_ttm_tt_is_readonly(ttm)) in amdgpu_ttm_tt_pte_flags()