78 	return ttm_range_man_init(&adev->mman.bdev, type,  in amdgpu_ttm_init_on_chip()
130 		if (!adev->mman.buffer_funcs_enabled) {  in amdgpu_evict_flags()
193 	BUG_ON(adev->mman.buffer_funcs->copy_max_bytes <  in amdgpu_ttm_map_buffer()
223 	num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);  in amdgpu_ttm_map_buffer()
226 	r = amdgpu_job_alloc_with_ib(adev, &adev->mman.high_pr,  in amdgpu_ttm_map_buffer()
294 	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;  in amdgpu_ttm_copy_mem_to_mem()
301 	if (!adev->mman.buffer_funcs_enabled) {  in amdgpu_ttm_copy_mem_to_mem()
309 	mutex_lock(&adev->mman.gtt_window_lock);  in amdgpu_ttm_copy_mem_to_mem()
364 	mutex_unlock(&adev->mman.gtt_window_lock);  in amdgpu_ttm_copy_mem_to_mem()
559 	if (adev->mman.buffer_funcs_enabled &&  in amdgpu_bo_move()
572 	if (adev->mman.buffer_funcs_enabled)  in amdgpu_bo_move()
617 		if (adev->mman.aper_base_kaddr &&  in amdgpu_ttm_io_mem_reserve()
619 			mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr +  in amdgpu_ttm_io_mem_reserve()
1160 	if (adev->mman.ttm_pools && gtt->pool_id >= 0)  in amdgpu_ttm_tt_populate()
1161 		pool = &adev->mman.ttm_pools[gtt->pool_id];  in amdgpu_ttm_tt_populate()
1163 		pool = &adev->mman.bdev.pool;  in amdgpu_ttm_tt_populate()
1205 	if (adev->mman.ttm_pools && gtt->pool_id >= 0)  in amdgpu_ttm_tt_unpopulate()
1206 		pool = &adev->mman.ttm_pools[gtt->pool_id];  in amdgpu_ttm_tt_unpopulate()
1208 		pool = &adev->mman.bdev.pool;  in amdgpu_ttm_tt_unpopulate()
1499 	if (!adev->mman.sdma_access_ptr)  in amdgpu_ttm_access_memory_sdma()
1506 		memcpy(adev->mman.sdma_access_ptr, buf, len);  in amdgpu_ttm_access_memory_sdma()
1508 	num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);  in amdgpu_ttm_access_memory_sdma()
1509 	r = amdgpu_job_alloc_with_ib(adev, &adev->mman.high_pr,  in amdgpu_ttm_access_memory_sdma()
1519 	dst_addr = amdgpu_bo_gpu_offset(adev->mman.sdma_access_bo);  in amdgpu_ttm_access_memory_sdma()
1526 	amdgpu_ring_pad_ib(adev->mman.buffer_funcs_ring, &job->ibs[0]);  in amdgpu_ttm_access_memory_sdma()
1536 		memcpy(buf, adev->mman.sdma_access_ptr, len);  in amdgpu_ttm_access_memory_sdma()
1625 	amdgpu_bo_free_kernel(&adev->mman.fw_vram_usage_reserved_bo,  in amdgpu_ttm_fw_reserve_vram_fini()
1626 		NULL, &adev->mman.fw_vram_usage_va);  in amdgpu_ttm_fw_reserve_vram_fini()
1641 	amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo,  in amdgpu_ttm_drv_reserve_vram_fini()
1643 						  &adev->mman.drv_vram_usage_va);  in amdgpu_ttm_drv_reserve_vram_fini()
1657 	adev->mman.fw_vram_usage_va = NULL;  in amdgpu_ttm_fw_reserve_vram_init()
1658 	adev->mman.fw_vram_usage_reserved_bo = NULL;  in amdgpu_ttm_fw_reserve_vram_init()
1660 	if (adev->mman.fw_vram_usage_size == 0 ||  in amdgpu_ttm_fw_reserve_vram_init()
1661 	    adev->mman.fw_vram_usage_size > vram_size)  in amdgpu_ttm_fw_reserve_vram_init()
1665 					  adev->mman.fw_vram_usage_start_offset,  in amdgpu_ttm_fw_reserve_vram_init()
1666 					  adev->mman.fw_vram_usage_size,  in amdgpu_ttm_fw_reserve_vram_init()
1667 					  &adev->mman.fw_vram_usage_reserved_bo,  in amdgpu_ttm_fw_reserve_vram_init()
1668 					  &adev->mman.fw_vram_usage_va);  in amdgpu_ttm_fw_reserve_vram_init()
1682 	adev->mman.drv_vram_usage_va = NULL;  in amdgpu_ttm_drv_reserve_vram_init()
1683 	adev->mman.drv_vram_usage_reserved_bo = NULL;  in amdgpu_ttm_drv_reserve_vram_init()
1685 	if (adev->mman.drv_vram_usage_size == 0 ||  in amdgpu_ttm_drv_reserve_vram_init()
1686 	    adev->mman.drv_vram_usage_size > vram_size)  in amdgpu_ttm_drv_reserve_vram_init()
1690 					  adev->mman.drv_vram_usage_start_offset,  in amdgpu_ttm_drv_reserve_vram_init()
1691 					  adev->mman.drv_vram_usage_size,  in amdgpu_ttm_drv_reserve_vram_init()
1692 					  &adev->mman.drv_vram_usage_reserved_bo,  in amdgpu_ttm_drv_reserve_vram_init()
1693 					  &adev->mman.drv_vram_usage_va);  in amdgpu_ttm_drv_reserve_vram_init()
1794 			reserve_size, &adev->mman.fw_reserved_memory, NULL);  in amdgpu_ttm_reserve_tmr()
1797 			amdgpu_bo_free_kernel(&adev->mman.fw_reserved_memory,  in amdgpu_ttm_reserve_tmr()
1815 	adev->mman.ttm_pools = kcalloc(adev->gmc.num_mem_partitions,  in amdgpu_ttm_pools_init()
1816 				       sizeof(*adev->mman.ttm_pools),  in amdgpu_ttm_pools_init()
1818 	if (!adev->mman.ttm_pools)  in amdgpu_ttm_pools_init()
1822 		ttm_pool_init(&adev->mman.ttm_pools[i], adev->dev,  in amdgpu_ttm_pools_init()
1833 	if (!adev->gmc.is_app_apu || !adev->mman.ttm_pools)  in amdgpu_ttm_pools_fini()
1837 		ttm_pool_fini(&adev->mman.ttm_pools[i]);  in amdgpu_ttm_pools_fini()
1839 	kfree(adev->mman.ttm_pools);  in amdgpu_ttm_pools_fini()
1840 	adev->mman.ttm_pools = NULL;  in amdgpu_ttm_pools_fini()
1857 	mutex_init(&adev->mman.gtt_window_lock);  in amdgpu_ttm_init()
1861 	r = ttm_device_init(&adev->mman.bdev, &amdgpu_bo_driver, adev->dev,  in amdgpu_ttm_init()
1876 	adev->mman.initialized = true;  in amdgpu_ttm_init()
1890 		adev->mman.aper_base_kaddr = ioremap_cache(adev->gmc.aper_base,  in amdgpu_ttm_init()
1898 		adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base,  in amdgpu_ttm_init()
1923 	if (adev->mman.discovery_bin) {  in amdgpu_ttm_init()
1936 					       adev->mman.stolen_vga_size,  in amdgpu_ttm_init()
1937 					       &adev->mman.stolen_vga_memory,  in amdgpu_ttm_init()
1942 		r = amdgpu_bo_create_kernel_at(adev, adev->mman.stolen_vga_size,  in amdgpu_ttm_init()
1943 					       adev->mman.stolen_extended_size,  in amdgpu_ttm_init()
1944 					       &adev->mman.stolen_extended_memory,  in amdgpu_ttm_init()
1951 					       adev->mman.stolen_reserved_offset,  in amdgpu_ttm_init()
1952 					       adev->mman.stolen_reserved_size,  in amdgpu_ttm_init()
1953 					       &adev->mman.stolen_reserved_memory,  in amdgpu_ttm_init()
2022 				&adev->mman.sdma_access_bo, NULL,  in amdgpu_ttm_init()
2023 				&adev->mman.sdma_access_ptr))  in amdgpu_ttm_init()
2036 	if (!adev->mman.initialized)  in amdgpu_ttm_fini()
2044 		amdgpu_bo_free_kernel(&adev->mman.stolen_vga_memory, NULL, NULL);  in amdgpu_ttm_fini()
2045 		amdgpu_bo_free_kernel(&adev->mman.stolen_extended_memory, NULL, NULL);  in amdgpu_ttm_fini()
2047 		amdgpu_bo_free_kernel(&adev->mman.fw_reserved_memory, NULL,  in amdgpu_ttm_fini()
2049 		if (adev->mman.stolen_reserved_size)  in amdgpu_ttm_fini()
2050 			amdgpu_bo_free_kernel(&adev->mman.stolen_reserved_memory,  in amdgpu_ttm_fini()
2053 	amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,  in amdgpu_ttm_fini()
2054 					&adev->mman.sdma_access_ptr);  in amdgpu_ttm_fini()
2060 		if (adev->mman.aper_base_kaddr)  in amdgpu_ttm_fini()
2061 			iounmap(adev->mman.aper_base_kaddr);  in amdgpu_ttm_fini()
2062 		adev->mman.aper_base_kaddr = NULL;  in amdgpu_ttm_fini()
2070 	ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS);  in amdgpu_ttm_fini()
2071 	ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GWS);  in amdgpu_ttm_fini()
2072 	ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_OA);  in amdgpu_ttm_fini()
2073 	ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_DOORBELL);  in amdgpu_ttm_fini()
2074 	ttm_device_fini(&adev->mman.bdev);  in amdgpu_ttm_fini()
2075 	adev->mman.initialized = false;  in amdgpu_ttm_fini()
2090 	struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);  in amdgpu_ttm_set_buffer_funcs_status()
2094 	if (!adev->mman.initialized || amdgpu_in_reset(adev) ||  in amdgpu_ttm_set_buffer_funcs_status()
2095 	    adev->mman.buffer_funcs_enabled == enable || adev->gmc.is_app_apu)  in amdgpu_ttm_set_buffer_funcs_status()
2102 		ring = adev->mman.buffer_funcs_ring;  in amdgpu_ttm_set_buffer_funcs_status()
2104 		r = drm_sched_entity_init(&adev->mman.high_pr,  in amdgpu_ttm_set_buffer_funcs_status()
2113 		r = drm_sched_entity_init(&adev->mman.low_pr,  in amdgpu_ttm_set_buffer_funcs_status()
2122 		drm_sched_entity_destroy(&adev->mman.high_pr);  in amdgpu_ttm_set_buffer_funcs_status()
2123 		drm_sched_entity_destroy(&adev->mman.low_pr);  in amdgpu_ttm_set_buffer_funcs_status()
2134 	adev->mman.buffer_funcs_enabled = enable;  in amdgpu_ttm_set_buffer_funcs_status()
2139 	drm_sched_entity_destroy(&adev->mman.high_pr);  in amdgpu_ttm_set_buffer_funcs_status()
2154 	struct drm_sched_entity *entity = delayed ? &adev->mman.low_pr :  in amdgpu_ttm_prepare_job()
2155 						    &adev->mman.high_pr;  in amdgpu_ttm_prepare_job()
2193 	max_bytes = adev->mman.buffer_funcs->copy_max_bytes;  in amdgpu_copy_buffer()
2195 	num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->copy_num_dw, 8);  in amdgpu_copy_buffer()
2241 	max_bytes = adev->mman.buffer_funcs->fill_max_bytes;  in amdgpu_ttm_fill_mem()
2243 	num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->fill_num_dw, 8);  in amdgpu_ttm_fill_mem()
2281 	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;  in amdgpu_ttm_clear_buffer()
2286 	if (!adev->mman.buffer_funcs_enabled)  in amdgpu_ttm_clear_buffer()
2296 	mutex_lock(&adev->mman.gtt_window_lock);  in amdgpu_ttm_clear_buffer()
2325 	mutex_unlock(&adev->mman.gtt_window_lock);  in amdgpu_ttm_clear_buffer()
2337 	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;  in amdgpu_fill_buffer()
2342 	if (!adev->mman.buffer_funcs_enabled) {  in amdgpu_fill_buffer()
2349 	mutex_lock(&adev->mman.gtt_window_lock);  in amdgpu_fill_buffer()
2373 	mutex_unlock(&adev->mman.gtt_window_lock);  in amdgpu_fill_buffer()
2400 		man = ttm_manager_type(&adev->mman.bdev, mem_type);  in amdgpu_ttm_evict_resources()
2407 	return ttm_resource_manager_evict_all(&adev->mman.bdev, man);  in amdgpu_ttm_evict_resources()
2416 	return ttm_pool_debugfs(&adev->mman.bdev.pool, m);  in amdgpu_ttm_page_pool_show()
2541 		if (p->mapping != adev->mman.bdev.dev_mapping)  in amdgpu_iomem_read()
2592 		if (p->mapping != adev->mman.bdev.dev_mapping)  in amdgpu_iomem_write()
2630 	ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,  in amdgpu_ttm_debugfs_init()
2633 	ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,  in amdgpu_ttm_debugfs_init()
2636 	ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,  in amdgpu_ttm_debugfs_init()
2639 	ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,  in amdgpu_ttm_debugfs_init()
2642 	ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,  in amdgpu_ttm_debugfs_init()