81 	return ttm_range_man_init(&adev->mman.bdev, type,  in amdgpu_ttm_init_on_chip()
134 		if (!adev->mman.buffer_funcs_enabled) {  in amdgpu_evict_flags()
174 	ring = adev->mman.buffer_funcs_ring;  in amdgpu_ttm_job_submit()
212 	BUG_ON(adev->mman.buffer_funcs->copy_max_bytes <  in amdgpu_ttm_map_buffer()
240 	num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);  in amdgpu_ttm_map_buffer()
311 	if (!adev->mman.buffer_funcs_enabled) {  in amdgpu_ttm_copy_mem_to_mem()
406 	e = atomic_inc_return(&adev->mman.next_move_entity) %  in amdgpu_move_blit()
407 			      adev->mman.num_move_entities;  in amdgpu_move_blit()
408 	entity = &adev->mman.move_entities[e];  in amdgpu_move_blit()
578 	if (adev->mman.buffer_funcs_enabled &&  in amdgpu_bo_move()
591 	if (adev->mman.buffer_funcs_enabled)  in amdgpu_bo_move()
636 		if (adev->mman.aper_base_kaddr &&  in amdgpu_ttm_io_mem_reserve()
638 			mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr +  in amdgpu_ttm_io_mem_reserve()
1223 	if (adev->mman.ttm_pools && gtt->pool_id >= 0)  in amdgpu_ttm_tt_populate()
1224 		pool = &adev->mman.ttm_pools[gtt->pool_id];  in amdgpu_ttm_tt_populate()
1226 		pool = &adev->mman.bdev.pool;  in amdgpu_ttm_tt_populate()
1268 	if (adev->mman.ttm_pools && gtt->pool_id >= 0)  in amdgpu_ttm_tt_unpopulate()
1269 		pool = &adev->mman.ttm_pools[gtt->pool_id];  in amdgpu_ttm_tt_unpopulate()
1271 		pool = &adev->mman.bdev.pool;  in amdgpu_ttm_tt_unpopulate()
1563 	if (!adev->mman.sdma_access_ptr)  in amdgpu_ttm_access_memory_sdma()
1570 		memcpy(adev->mman.sdma_access_ptr, buf, len);  in amdgpu_ttm_access_memory_sdma()
1572 	num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);  in amdgpu_ttm_access_memory_sdma()
1573 	r = amdgpu_job_alloc_with_ib(adev, &adev->mman.default_entity.base,  in amdgpu_ttm_access_memory_sdma()
1581 	mutex_lock(&adev->mman.default_entity.lock);  in amdgpu_ttm_access_memory_sdma()
1585 	dst_addr = amdgpu_bo_gpu_offset(adev->mman.sdma_access_bo);  in amdgpu_ttm_access_memory_sdma()
1592 	fence = amdgpu_ttm_job_submit(adev, &adev->mman.default_entity, job, num_dw);  in amdgpu_ttm_access_memory_sdma()
1593 	mutex_unlock(&adev->mman.default_entity.lock);  in amdgpu_ttm_access_memory_sdma()
1600 		memcpy(buf, adev->mman.sdma_access_ptr, len);  in amdgpu_ttm_access_memory_sdma()
1687 	resv = &adev->mman.resv_region[id];  in amdgpu_ttm_init_vram_resv()
1744 	reserve_size = adev->mman.resv_region[AMDGPU_RESV_FW].size;  in amdgpu_ttm_init_mem_train_resv_region()
1765 	if (adev->mman.resv_region[AMDGPU_RESV_FW_VRAM_USAGE].size > vram_size)  in amdgpu_ttm_init_vram_resv_regions()
1766 		adev->mman.resv_region[AMDGPU_RESV_FW_VRAM_USAGE].size = 0;  in amdgpu_ttm_init_vram_resv_regions()
1768 	if (adev->mman.resv_region[AMDGPU_RESV_DRV_VRAM_USAGE].size > vram_size)  in amdgpu_ttm_init_vram_resv_regions()
1769 		adev->mman.resv_region[AMDGPU_RESV_DRV_VRAM_USAGE].size = 0;  in amdgpu_ttm_init_vram_resv_regions()
1781 	resv = &adev->mman.resv_region[id];  in amdgpu_ttm_mark_vram_reserved()
1806 	resv = &adev->mman.resv_region[id];  in amdgpu_ttm_unmark_vram_reserved()
1858 			&adev->mman.resv_region[AMDGPU_RESV_MEM_TRAIN];  in amdgpu_ttm_training_data_block_init()
1880 	adev->mman.ttm_pools = kzalloc_objs(*adev->mman.ttm_pools,  in amdgpu_ttm_pools_init()
1882 	if (!adev->mman.ttm_pools)  in amdgpu_ttm_pools_init()
1886 		ttm_pool_init(&adev->mman.ttm_pools[i], adev->dev,  in amdgpu_ttm_pools_init()
1897 	if (!adev->gmc.is_app_apu || !adev->mman.ttm_pools)  in amdgpu_ttm_pools_fini()
1901 		ttm_pool_fini(&adev->mman.ttm_pools[i]);  in amdgpu_ttm_pools_fini()
1903 	kfree(adev->mman.ttm_pools);  in amdgpu_ttm_pools_fini()
1904 	adev->mman.ttm_pools = NULL;  in amdgpu_ttm_pools_fini()
2073 	r = ttm_device_init(&adev->mman.bdev, &amdgpu_bo_driver, adev->dev,  in amdgpu_ttm_init()
2092 	adev->mman.initialized = true;  in amdgpu_ttm_init()
2108 		adev->mman.aper_base_kaddr = ioremap_cache(adev->gmc.aper_base,  in amdgpu_ttm_init()
2116 		adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base,  in amdgpu_ttm_init()
2126 	if (adev->mman.resv_region[AMDGPU_RESV_MEM_TRAIN].size) {  in amdgpu_ttm_init()
2221 				&adev->mman.sdma_access_bo, NULL,  in amdgpu_ttm_init()
2222 				&adev->mman.sdma_access_ptr))  in amdgpu_ttm_init()
2236 	if (!adev->mman.initialized)  in amdgpu_ttm_fini()
2251 	amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,  in amdgpu_ttm_fini()
2252 					&adev->mman.sdma_access_ptr);  in amdgpu_ttm_fini()
2260 		if (adev->mman.aper_base_kaddr)  in amdgpu_ttm_fini()
2261 			iounmap(adev->mman.aper_base_kaddr);  in amdgpu_ttm_fini()
2262 		adev->mman.aper_base_kaddr = NULL;  in amdgpu_ttm_fini()
2273 	ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS);  in amdgpu_ttm_fini()
2274 	ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GWS);  in amdgpu_ttm_fini()
2275 	ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_OA);  in amdgpu_ttm_fini()
2276 	ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_DOORBELL);  in amdgpu_ttm_fini()
2277 	ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_MMIO_REMAP);  in amdgpu_ttm_fini()
2278 	ttm_device_fini(&adev->mman.bdev);  in amdgpu_ttm_fini()
2279 	adev->mman.initialized = false;  in amdgpu_ttm_fini()
2294 	struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);  in amdgpu_ttm_set_buffer_funcs_status()
2299 	if (!adev->mman.initialized || amdgpu_in_reset(adev) ||  in amdgpu_ttm_set_buffer_funcs_status()
2300 	    adev->mman.buffer_funcs_enabled == enable || adev->gmc.is_app_apu)  in amdgpu_ttm_set_buffer_funcs_status()
2307 		if (!adev->mman.buffer_funcs_ring || !adev->mman.buffer_funcs_ring->sched.ready) {  in amdgpu_ttm_set_buffer_funcs_status()
2314 		ring = adev->mman.buffer_funcs_ring;  in amdgpu_ttm_set_buffer_funcs_status()
2316 		r = amdgpu_ttm_buffer_entity_init(&adev->mman.gtt_mgr,  in amdgpu_ttm_set_buffer_funcs_status()
2317 						  &adev->mman.default_entity,  in amdgpu_ttm_set_buffer_funcs_status()
2326 		adev->mman.clear_entities = kcalloc(num_clear_entities,  in amdgpu_ttm_set_buffer_funcs_status()
2329 		atomic_set(&adev->mman.next_clear_entity, 0);  in amdgpu_ttm_set_buffer_funcs_status()
2330 		if (!adev->mman.clear_entities)  in amdgpu_ttm_set_buffer_funcs_status()
2333 		adev->mman.num_clear_entities = num_clear_entities;  in amdgpu_ttm_set_buffer_funcs_status()
2337 				&adev->mman.gtt_mgr, &adev->mman.clear_entities[i],  in amdgpu_ttm_set_buffer_funcs_status()
2343 						&adev->mman.gtt_mgr, &adev->mman.clear_entities[j]);  in amdgpu_ttm_set_buffer_funcs_status()
2344 				kfree(adev->mman.clear_entities);  in amdgpu_ttm_set_buffer_funcs_status()
2345 				adev->mman.num_clear_entities = 0;  in amdgpu_ttm_set_buffer_funcs_status()
2346 				adev->mman.clear_entities = NULL;  in amdgpu_ttm_set_buffer_funcs_status()
2351 		adev->mman.num_move_entities = num_move_entities;  in amdgpu_ttm_set_buffer_funcs_status()
2352 		atomic_set(&adev->mman.next_move_entity, 0);  in amdgpu_ttm_set_buffer_funcs_status()
2355 				&adev->mman.gtt_mgr,  in amdgpu_ttm_set_buffer_funcs_status()
2356 				&adev->mman.move_entities[i],  in amdgpu_ttm_set_buffer_funcs_status()
2362 						&adev->mman.gtt_mgr, &adev->mman.move_entities[j]);  in amdgpu_ttm_set_buffer_funcs_status()
2363 				adev->mman.num_move_entities = 0;  in amdgpu_ttm_set_buffer_funcs_status()
2368 		amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,  in amdgpu_ttm_set_buffer_funcs_status()
2369 					      &adev->mman.default_entity);  in amdgpu_ttm_set_buffer_funcs_status()
2370 		for (i = 0; i < adev->mman.num_clear_entities; i++)  in amdgpu_ttm_set_buffer_funcs_status()
2371 			amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,  in amdgpu_ttm_set_buffer_funcs_status()
2372 						      &adev->mman.clear_entities[i]);  in amdgpu_ttm_set_buffer_funcs_status()
2373 		for (i = 0; i < adev->mman.num_move_entities; i++)  in amdgpu_ttm_set_buffer_funcs_status()
2374 			amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,  in amdgpu_ttm_set_buffer_funcs_status()
2375 						      &adev->mman.move_entities[i]);  in amdgpu_ttm_set_buffer_funcs_status()
2380 		kfree(adev->mman.clear_entities);  in amdgpu_ttm_set_buffer_funcs_status()
2381 		adev->mman.clear_entities = NULL;  in amdgpu_ttm_set_buffer_funcs_status()
2382 		adev->mman.num_clear_entities = 0;  in amdgpu_ttm_set_buffer_funcs_status()
2383 		adev->mman.num_move_entities = 0;  in amdgpu_ttm_set_buffer_funcs_status()
2392 	adev->mman.buffer_funcs_enabled = enable;  in amdgpu_ttm_set_buffer_funcs_status()
2397 	for (i = 0; i < adev->mman.num_clear_entities; i++)  in amdgpu_ttm_set_buffer_funcs_status()
2398 		amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,  in amdgpu_ttm_set_buffer_funcs_status()
2399 					      &adev->mman.clear_entities[i]);  in amdgpu_ttm_set_buffer_funcs_status()
2400 	kfree(adev->mman.clear_entities);  in amdgpu_ttm_set_buffer_funcs_status()
2401 	adev->mman.clear_entities = NULL;  in amdgpu_ttm_set_buffer_funcs_status()
2402 	adev->mman.num_clear_entities = 0;  in amdgpu_ttm_set_buffer_funcs_status()
2404 	amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,  in amdgpu_ttm_set_buffer_funcs_status()
2405 				      &adev->mman.default_entity);  in amdgpu_ttm_set_buffer_funcs_status()
2452 	ring = adev->mman.buffer_funcs_ring;  in amdgpu_copy_buffer()
2460 	max_bytes = adev->mman.buffer_funcs->copy_max_bytes;  in amdgpu_copy_buffer()
2462 	num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->copy_num_dw, 8);  in amdgpu_copy_buffer()
2504 	max_bytes = adev->mman.buffer_funcs->fill_max_bytes;  in amdgpu_ttm_fill_mem()
2506 	num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->fill_num_dw, 8);  in amdgpu_ttm_fill_mem()
2547 	if (!adev->mman.buffer_funcs_enabled)  in amdgpu_ttm_clear_buffer()
2552 	entity = &adev->mman.clear_entities[0];  in amdgpu_ttm_clear_buffer()
2644 	struct amdgpu_mman *mman = &adev->mman;  in amdgpu_ttm_next_clear_entity()  local
2647 	if (mman->num_clear_entities == 0)  in amdgpu_ttm_next_clear_entity()
2650 	i = atomic_inc_return(&mman->next_clear_entity) %  in amdgpu_ttm_next_clear_entity()
2651 			      mman->num_clear_entities;  in amdgpu_ttm_next_clear_entity()
2652 	return &mman->clear_entities[i];  in amdgpu_ttm_next_clear_entity()
2675 		man = ttm_manager_type(&adev->mman.bdev, mem_type);  in amdgpu_ttm_evict_resources()
2682 	return ttm_resource_manager_evict_all(&adev->mman.bdev, man);  in amdgpu_ttm_evict_resources()
2691 	return ttm_pool_debugfs(&adev->mman.bdev.pool, m);  in amdgpu_ttm_page_pool_show()
2816 		if (p->mapping != adev->mman.bdev.dev_mapping)  in amdgpu_iomem_read()
2867 		if (p->mapping != adev->mman.bdev.dev_mapping)  in amdgpu_iomem_write()
2905 	ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,  in amdgpu_ttm_debugfs_init()
2908 	ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,  in amdgpu_ttm_debugfs_init()
2911 	ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,  in amdgpu_ttm_debugfs_init()
2914 	ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,  in amdgpu_ttm_debugfs_init()
2917 	ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,  in amdgpu_ttm_debugfs_init()