19  * struct xe_mem_pool - DRM MM pool for sub-allocating memory from a BO on an
22  * The XE memory pool is a DRM MM manager that provides sub-allocation of memory
27  * The memory pool maintains a primary BO that is pinned in the GGTT and mapped
31  * The API provided by the memory pool allows clients to allocate and free memory
38 	/** @bo: Active pool BO (GGTT-pinned, CPU-mapped). */
55 static struct xe_tile *pool_to_tile(struct xe_mem_pool *pool)  in pool_to_tile()  argument
57 	return pool->bo->tile;  in pool_to_tile()
62 	struct xe_mem_pool *pool = arg;  in fini_pool_action()  local
64 	if (pool->is_iomem)  in fini_pool_action()
65 		kvfree(pool->cpu_addr);  in fini_pool_action()
67 	drm_mm_takedown(&pool->base);  in fini_pool_action()
70 static int pool_shadow_init(struct xe_mem_pool *pool)  in pool_shadow_init()  argument
72 	struct xe_tile *tile = pool->bo->tile;  in pool_shadow_init()
77 	xe_assert(xe, !pool->shadow);  in pool_shadow_init()
79 	ret = drmm_mutex_init(&xe->drm, &pool->swap_guard);  in pool_shadow_init()
85 		might_lock(&pool->swap_guard);  in pool_shadow_init()
89 					      xe_bo_size(pool->bo),  in pool_shadow_init()
97 	pool->shadow = shadow;  in pool_shadow_init()
103  * xe_mem_pool_init() - Initialize memory pool.
108  * @flags: flags to use to create shadow pool.
110  * Initializes a memory pool for sub-allocating memory from a backing BO on the
121 	struct xe_mem_pool *pool;  in xe_mem_pool_init()  local
129 	pool = drmm_kzalloc(&xe->drm, sizeof(*pool), GFP_KERNEL);  in xe_mem_pool_init()
130 	if (!pool)  in xe_mem_pool_init()
139 		xe_tile_err(tile, "Failed to prepare %uKiB BO for mem pool (%pe)\n",  in xe_mem_pool_init()
143 	pool->bo = bo;  in xe_mem_pool_init()
144 	pool->is_iomem = bo->vmap.is_iomem;  in xe_mem_pool_init()
146 	if (pool->is_iomem) {  in xe_mem_pool_init()
147 		pool->cpu_addr = kvzalloc(size, GFP_KERNEL);  in xe_mem_pool_init()
148 		if (!pool->cpu_addr)  in xe_mem_pool_init()
151 		pool->cpu_addr = bo->vmap.vaddr;  in xe_mem_pool_init()
155 		ret = pool_shadow_init(pool);  in xe_mem_pool_init()
161 	drm_mm_init(&pool->base, 0, managed_size);  in xe_mem_pool_init()
162 	ret = drmm_add_action_or_reset(&xe->drm, fini_pool_action, pool);  in xe_mem_pool_init()
166 	return pool;  in xe_mem_pool_init()
171 			    "Failed to initialize shadow BO for mem pool (%d)\n", ret);  in xe_mem_pool_init()
173 		kvfree(pool->cpu_addr);  in xe_mem_pool_init()
178  * xe_mem_pool_sync() - Copy the entire contents of the main pool to shadow pool.
179  * @pool: the memory pool containing the primary and shadow BOs.
181  * Copies the entire contents of the primary pool to the shadow pool. This must
183  * flag to ensure that the shadow pool has the same initial contents as the primary
184  * pool. After this initial synchronization, clients can choose to synchronize the
185  * shadow pool with the primary pool on a node  basis using
190 void xe_mem_pool_sync(struct xe_mem_pool *pool)  in xe_mem_pool_sync()  argument
192 	struct xe_tile *tile = pool_to_tile(pool);  in xe_mem_pool_sync()
195 	xe_tile_assert(tile, pool->shadow);  in xe_mem_pool_sync()
197 	xe_map_memcpy_to(xe, &pool->shadow->vmap, 0,  in xe_mem_pool_sync()
198 			 pool->cpu_addr, xe_bo_size(pool->bo));  in xe_mem_pool_sync()
203  * @pool: the memory pool containing the primary and shadow BOs.
206  * pool. This allows for atomic updates to the contents of the primary BO
213 void xe_mem_pool_swap_shadow_locked(struct xe_mem_pool *pool)  in xe_mem_pool_swap_shadow_locked()  argument
215 	struct xe_tile *tile = pool_to_tile(pool);  in xe_mem_pool_swap_shadow_locked()
217 	xe_tile_assert(tile, pool->shadow);  in xe_mem_pool_swap_shadow_locked()
218 	lockdep_assert_held(&pool->swap_guard);  in xe_mem_pool_swap_shadow_locked()
220 	swap(pool->bo, pool->shadow);  in xe_mem_pool_swap_shadow_locked()
221 	if (!pool->bo->vmap.is_iomem)  in xe_mem_pool_swap_shadow_locked()
222 		pool->cpu_addr = pool->bo->vmap.vaddr;  in xe_mem_pool_swap_shadow_locked()
226  * xe_mem_pool_sync_shadow_locked() - Copy node from primary pool to shadow pool.
227  * @node: the node allocated in the memory pool.
229  * Copies the specified batch buffer from the primary pool to the shadow pool.
237 	struct xe_mem_pool *pool = node_to_pool(node);  in xe_mem_pool_sync_shadow_locked()  local
238 	struct xe_tile *tile = pool_to_tile(pool);  in xe_mem_pool_sync_shadow_locked()
242 	xe_tile_assert(tile, pool->shadow);  in xe_mem_pool_sync_shadow_locked()
243 	lockdep_assert_held(&pool->swap_guard);  in xe_mem_pool_sync_shadow_locked()
245 	xe_map_memcpy_to(xe, &pool->shadow->vmap,  in xe_mem_pool_sync_shadow_locked()
247 			 pool->cpu_addr + sa_node->start,  in xe_mem_pool_sync_shadow_locked()
252  * xe_mem_pool_gpu_addr() - Retrieve GPU address of memory pool.
253  * @pool: the memory pool
255  * Returns: GGTT address of the memory pool.
257 u64 xe_mem_pool_gpu_addr(struct xe_mem_pool *pool)  in xe_mem_pool_gpu_addr()  argument
259 	return xe_bo_ggtt_addr(pool->bo);  in xe_mem_pool_gpu_addr()
263  * xe_mem_pool_cpu_addr() - Retrieve CPU address of manager pool.
264  * @pool: the memory pool
266  * Returns: CPU virtual address of memory pool.
268 void *xe_mem_pool_cpu_addr(struct xe_mem_pool *pool)  in xe_mem_pool_cpu_addr()  argument
270 	return pool->cpu_addr;  in xe_mem_pool_cpu_addr()
275  * operations on a memory pool.
276  * @pool: the memory pool
278  * Returns: Swap guard mutex or NULL if shadow pool is not created.
280 struct mutex *xe_mem_pool_bo_swap_guard(struct xe_mem_pool *pool)  in xe_mem_pool_bo_swap_guard()  argument
282 	if (!pool->shadow)  in xe_mem_pool_bo_swap_guard()
285 	return &pool->swap_guard;  in xe_mem_pool_bo_swap_guard()
291  * @node: the node allocated in the memory pool to flush.
295 	struct xe_mem_pool *pool = node_to_pool(node);  in xe_mem_pool_bo_flush_write()  local
296 	struct xe_tile *tile = pool_to_tile(pool);  in xe_mem_pool_bo_flush_write()
300 	if (!pool->bo->vmap.is_iomem)  in xe_mem_pool_bo_flush_write()
303 	xe_map_memcpy_to(xe, &pool->bo->vmap, sa_node->start,  in xe_mem_pool_bo_flush_write()
304 			 pool->cpu_addr + sa_node->start,  in xe_mem_pool_bo_flush_write()
311  * @node: the node allocated in the memory pool to read back.
315 	struct xe_mem_pool *pool = node_to_pool(node);  in xe_mem_pool_bo_sync_read()  local
316 	struct xe_tile *tile = pool_to_tile(pool);  in xe_mem_pool_bo_sync_read()
320 	if (!pool->bo->vmap.is_iomem)  in xe_mem_pool_bo_sync_read()
323 	xe_map_memcpy_from(xe, pool->cpu_addr + sa_node->start,  in xe_mem_pool_bo_sync_read()
324 			   &pool->bo->vmap, sa_node->start, sa_node->size);  in xe_mem_pool_bo_sync_read()
343  * xe_mem_pool_insert_node() - Insert a node into the memory pool.
344  * @pool: the memory pool to insert into
348  * Inserts a node into the specified memory pool using drm_mm for
353 int xe_mem_pool_insert_node(struct xe_mem_pool *pool,  in xe_mem_pool_insert_node()  argument
356 	if (!pool)  in xe_mem_pool_insert_node()
359 	return drm_mm_insert_node(&pool->base, &node->sa_node, size);  in xe_mem_pool_insert_node()
363  * xe_mem_pool_free_node() - Free a node allocated from the memory pool.
379  * @node: the node allocated in the memory pool
385 	struct xe_mem_pool *pool = node_to_pool(node);  in xe_mem_pool_node_cpu_addr()  local
387 	return xe_mem_pool_cpu_addr(pool) + node->sa_node.start;  in xe_mem_pool_node_cpu_addr()
392  * @pool: the memory pool info be dumped.
396  * pool information.
400 void xe_mem_pool_dump(struct xe_mem_pool *pool, struct drm_printer *p)  in xe_mem_pool_dump()  argument
402 	drm_mm_print(&pool->base, p);  in xe_mem_pool_dump()