xref: /linux/drivers/gpu/drm/xe/xe_bo.c (revision b50ecc5aca4d18f1f0c4942f5c797bc85edef144)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2021 Intel Corporation
4  */
5 
6 #include "xe_bo.h"
7 
8 #include <linux/dma-buf.h>
9 
10 #include <drm/drm_drv.h>
11 #include <drm/drm_gem_ttm_helper.h>
12 #include <drm/drm_managed.h>
13 #include <drm/ttm/ttm_device.h>
14 #include <drm/ttm/ttm_placement.h>
15 #include <drm/ttm/ttm_tt.h>
16 #include <uapi/drm/xe_drm.h>
17 
18 #include "xe_device.h"
19 #include "xe_dma_buf.h"
20 #include "xe_drm_client.h"
21 #include "xe_ggtt.h"
22 #include "xe_gt.h"
23 #include "xe_map.h"
24 #include "xe_migrate.h"
25 #include "xe_pm.h"
26 #include "xe_preempt_fence.h"
27 #include "xe_res_cursor.h"
28 #include "xe_trace_bo.h"
29 #include "xe_ttm_stolen_mgr.h"
30 #include "xe_vm.h"
31 
32 const char *const xe_mem_type_to_name[TTM_NUM_MEM_TYPES]  = {
33 	[XE_PL_SYSTEM] = "system",
34 	[XE_PL_TT] = "gtt",
35 	[XE_PL_VRAM0] = "vram0",
36 	[XE_PL_VRAM1] = "vram1",
37 	[XE_PL_STOLEN] = "stolen"
38 };
39 
40 static const struct ttm_place sys_placement_flags = {
41 	.fpfn = 0,
42 	.lpfn = 0,
43 	.mem_type = XE_PL_SYSTEM,
44 	.flags = 0,
45 };
46 
47 static struct ttm_placement sys_placement = {
48 	.num_placement = 1,
49 	.placement = &sys_placement_flags,
50 };
51 
52 static const struct ttm_place tt_placement_flags[] = {
53 	{
54 		.fpfn = 0,
55 		.lpfn = 0,
56 		.mem_type = XE_PL_TT,
57 		.flags = TTM_PL_FLAG_DESIRED,
58 	},
59 	{
60 		.fpfn = 0,
61 		.lpfn = 0,
62 		.mem_type = XE_PL_SYSTEM,
63 		.flags = TTM_PL_FLAG_FALLBACK,
64 	}
65 };
66 
67 static struct ttm_placement tt_placement = {
68 	.num_placement = 2,
69 	.placement = tt_placement_flags,
70 };
71 
72 bool mem_type_is_vram(u32 mem_type)
73 {
74 	return mem_type >= XE_PL_VRAM0 && mem_type != XE_PL_STOLEN;
75 }
76 
77 static bool resource_is_stolen_vram(struct xe_device *xe, struct ttm_resource *res)
78 {
79 	return res->mem_type == XE_PL_STOLEN && IS_DGFX(xe);
80 }
81 
82 static bool resource_is_vram(struct ttm_resource *res)
83 {
84 	return mem_type_is_vram(res->mem_type);
85 }
86 
87 bool xe_bo_is_vram(struct xe_bo *bo)
88 {
89 	return resource_is_vram(bo->ttm.resource) ||
90 		resource_is_stolen_vram(xe_bo_device(bo), bo->ttm.resource);
91 }
92 
93 bool xe_bo_is_stolen(struct xe_bo *bo)
94 {
95 	return bo->ttm.resource->mem_type == XE_PL_STOLEN;
96 }
97 
98 /**
99  * xe_bo_has_single_placement - check if BO is placed only in one memory location
100  * @bo: The BO
101  *
102  * This function checks whether a given BO is placed in only one memory location.
103  *
104  * Returns: true if the BO is placed in a single memory location, false otherwise.
105  *
106  */
107 bool xe_bo_has_single_placement(struct xe_bo *bo)
108 {
109 	return bo->placement.num_placement == 1;
110 }
111 
112 /**
113  * xe_bo_is_stolen_devmem - check if BO is of stolen type accessed via PCI BAR
114  * @bo: The BO
115  *
116  * The stolen memory is accessed through the PCI BAR for both DGFX and some
117  * integrated platforms that have a dedicated bit in the PTE for devmem (DM).
118  *
119  * Returns: true if it's stolen memory accessed via PCI BAR, false otherwise.
120  */
121 bool xe_bo_is_stolen_devmem(struct xe_bo *bo)
122 {
123 	return xe_bo_is_stolen(bo) &&
124 		GRAPHICS_VERx100(xe_bo_device(bo)) >= 1270;
125 }
126 
127 static bool xe_bo_is_user(struct xe_bo *bo)
128 {
129 	return bo->flags & XE_BO_FLAG_USER;
130 }
131 
132 static struct xe_migrate *
133 mem_type_to_migrate(struct xe_device *xe, u32 mem_type)
134 {
135 	struct xe_tile *tile;
136 
137 	xe_assert(xe, mem_type == XE_PL_STOLEN || mem_type_is_vram(mem_type));
138 	tile = &xe->tiles[mem_type == XE_PL_STOLEN ? 0 : (mem_type - XE_PL_VRAM0)];
139 	return tile->migrate;
140 }
141 
142 static struct xe_mem_region *res_to_mem_region(struct ttm_resource *res)
143 {
144 	struct xe_device *xe = ttm_to_xe_device(res->bo->bdev);
145 	struct ttm_resource_manager *mgr;
146 
147 	xe_assert(xe, resource_is_vram(res));
148 	mgr = ttm_manager_type(&xe->ttm, res->mem_type);
149 	return to_xe_ttm_vram_mgr(mgr)->vram;
150 }
151 
152 static void try_add_system(struct xe_device *xe, struct xe_bo *bo,
153 			   u32 bo_flags, u32 *c)
154 {
155 	if (bo_flags & XE_BO_FLAG_SYSTEM) {
156 		xe_assert(xe, *c < ARRAY_SIZE(bo->placements));
157 
158 		bo->placements[*c] = (struct ttm_place) {
159 			.mem_type = XE_PL_TT,
160 		};
161 		*c += 1;
162 	}
163 }
164 
165 static void add_vram(struct xe_device *xe, struct xe_bo *bo,
166 		     struct ttm_place *places, u32 bo_flags, u32 mem_type, u32 *c)
167 {
168 	struct ttm_place place = { .mem_type = mem_type };
169 	struct xe_mem_region *vram;
170 	u64 io_size;
171 
172 	xe_assert(xe, *c < ARRAY_SIZE(bo->placements));
173 
174 	vram = to_xe_ttm_vram_mgr(ttm_manager_type(&xe->ttm, mem_type))->vram;
175 	xe_assert(xe, vram && vram->usable_size);
176 	io_size = vram->io_size;
177 
178 	/*
179 	 * For eviction / restore on suspend / resume objects
180 	 * pinned in VRAM must be contiguous
181 	 */
182 	if (bo_flags & (XE_BO_FLAG_PINNED |
183 			XE_BO_FLAG_GGTT))
184 		place.flags |= TTM_PL_FLAG_CONTIGUOUS;
185 
186 	if (io_size < vram->usable_size) {
187 		if (bo_flags & XE_BO_FLAG_NEEDS_CPU_ACCESS) {
188 			place.fpfn = 0;
189 			place.lpfn = io_size >> PAGE_SHIFT;
190 		} else {
191 			place.flags |= TTM_PL_FLAG_TOPDOWN;
192 		}
193 	}
194 	places[*c] = place;
195 	*c += 1;
196 }
197 
198 static void try_add_vram(struct xe_device *xe, struct xe_bo *bo,
199 			 u32 bo_flags, u32 *c)
200 {
201 	if (bo_flags & XE_BO_FLAG_VRAM0)
202 		add_vram(xe, bo, bo->placements, bo_flags, XE_PL_VRAM0, c);
203 	if (bo_flags & XE_BO_FLAG_VRAM1)
204 		add_vram(xe, bo, bo->placements, bo_flags, XE_PL_VRAM1, c);
205 }
206 
207 static void try_add_stolen(struct xe_device *xe, struct xe_bo *bo,
208 			   u32 bo_flags, u32 *c)
209 {
210 	if (bo_flags & XE_BO_FLAG_STOLEN) {
211 		xe_assert(xe, *c < ARRAY_SIZE(bo->placements));
212 
213 		bo->placements[*c] = (struct ttm_place) {
214 			.mem_type = XE_PL_STOLEN,
215 			.flags = bo_flags & (XE_BO_FLAG_PINNED |
216 					     XE_BO_FLAG_GGTT) ?
217 				TTM_PL_FLAG_CONTIGUOUS : 0,
218 		};
219 		*c += 1;
220 	}
221 }
222 
223 static int __xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo,
224 				       u32 bo_flags)
225 {
226 	u32 c = 0;
227 
228 	try_add_vram(xe, bo, bo_flags, &c);
229 	try_add_system(xe, bo, bo_flags, &c);
230 	try_add_stolen(xe, bo, bo_flags, &c);
231 
232 	if (!c)
233 		return -EINVAL;
234 
235 	bo->placement = (struct ttm_placement) {
236 		.num_placement = c,
237 		.placement = bo->placements,
238 	};
239 
240 	return 0;
241 }
242 
243 int xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo,
244 			      u32 bo_flags)
245 {
246 	xe_bo_assert_held(bo);
247 	return __xe_bo_placement_for_flags(xe, bo, bo_flags);
248 }
249 
250 static void xe_evict_flags(struct ttm_buffer_object *tbo,
251 			   struct ttm_placement *placement)
252 {
253 	if (!xe_bo_is_xe_bo(tbo)) {
254 		/* Don't handle scatter gather BOs */
255 		if (tbo->type == ttm_bo_type_sg) {
256 			placement->num_placement = 0;
257 			return;
258 		}
259 
260 		*placement = sys_placement;
261 		return;
262 	}
263 
264 	/*
265 	 * For xe, sg bos that are evicted to system just triggers a
266 	 * rebind of the sg list upon subsequent validation to XE_PL_TT.
267 	 */
268 	switch (tbo->resource->mem_type) {
269 	case XE_PL_VRAM0:
270 	case XE_PL_VRAM1:
271 	case XE_PL_STOLEN:
272 		*placement = tt_placement;
273 		break;
274 	case XE_PL_TT:
275 	default:
276 		*placement = sys_placement;
277 		break;
278 	}
279 }
280 
281 struct xe_ttm_tt {
282 	struct ttm_tt ttm;
283 	struct device *dev;
284 	struct sg_table sgt;
285 	struct sg_table *sg;
286 	/** @purgeable: Whether the content of the pages of @ttm is purgeable. */
287 	bool purgeable;
288 };
289 
290 static int xe_tt_map_sg(struct ttm_tt *tt)
291 {
292 	struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);
293 	unsigned long num_pages = tt->num_pages;
294 	int ret;
295 
296 	XE_WARN_ON(tt->page_flags & TTM_TT_FLAG_EXTERNAL);
297 
298 	if (xe_tt->sg)
299 		return 0;
300 
301 	ret = sg_alloc_table_from_pages_segment(&xe_tt->sgt, tt->pages,
302 						num_pages, 0,
303 						(u64)num_pages << PAGE_SHIFT,
304 						xe_sg_segment_size(xe_tt->dev),
305 						GFP_KERNEL);
306 	if (ret)
307 		return ret;
308 
309 	xe_tt->sg = &xe_tt->sgt;
310 	ret = dma_map_sgtable(xe_tt->dev, xe_tt->sg, DMA_BIDIRECTIONAL,
311 			      DMA_ATTR_SKIP_CPU_SYNC);
312 	if (ret) {
313 		sg_free_table(xe_tt->sg);
314 		xe_tt->sg = NULL;
315 		return ret;
316 	}
317 
318 	return 0;
319 }
320 
321 static void xe_tt_unmap_sg(struct ttm_tt *tt)
322 {
323 	struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);
324 
325 	if (xe_tt->sg) {
326 		dma_unmap_sgtable(xe_tt->dev, xe_tt->sg,
327 				  DMA_BIDIRECTIONAL, 0);
328 		sg_free_table(xe_tt->sg);
329 		xe_tt->sg = NULL;
330 	}
331 }
332 
333 struct sg_table *xe_bo_sg(struct xe_bo *bo)
334 {
335 	struct ttm_tt *tt = bo->ttm.ttm;
336 	struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);
337 
338 	return xe_tt->sg;
339 }
340 
341 static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo,
342 				       u32 page_flags)
343 {
344 	struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
345 	struct xe_device *xe = xe_bo_device(bo);
346 	struct xe_ttm_tt *tt;
347 	unsigned long extra_pages;
348 	enum ttm_caching caching = ttm_cached;
349 	int err;
350 
351 	tt = kzalloc(sizeof(*tt), GFP_KERNEL);
352 	if (!tt)
353 		return NULL;
354 
355 	tt->dev = xe->drm.dev;
356 
357 	extra_pages = 0;
358 	if (xe_bo_needs_ccs_pages(bo))
359 		extra_pages = DIV_ROUND_UP(xe_device_ccs_bytes(xe, bo->size),
360 					   PAGE_SIZE);
361 
362 	/*
363 	 * DGFX system memory is always WB / ttm_cached, since
364 	 * other caching modes are only supported on x86. DGFX
365 	 * GPU system memory accesses are always coherent with the
366 	 * CPU.
367 	 */
368 	if (!IS_DGFX(xe)) {
369 		switch (bo->cpu_caching) {
370 		case DRM_XE_GEM_CPU_CACHING_WC:
371 			caching = ttm_write_combined;
372 			break;
373 		default:
374 			caching = ttm_cached;
375 			break;
376 		}
377 
378 		WARN_ON((bo->flags & XE_BO_FLAG_USER) && !bo->cpu_caching);
379 
380 		/*
381 		 * Display scanout is always non-coherent with the CPU cache.
382 		 *
383 		 * For Xe_LPG and beyond, PPGTT PTE lookups are also
384 		 * non-coherent and require a CPU:WC mapping.
385 		 */
386 		if ((!bo->cpu_caching && bo->flags & XE_BO_FLAG_SCANOUT) ||
387 		    (xe->info.graphics_verx100 >= 1270 &&
388 		     bo->flags & XE_BO_FLAG_PAGETABLE))
389 			caching = ttm_write_combined;
390 	}
391 
392 	if (bo->flags & XE_BO_FLAG_NEEDS_UC) {
393 		/*
394 		 * Valid only for internally-created buffers only, for
395 		 * which cpu_caching is never initialized.
396 		 */
397 		xe_assert(xe, bo->cpu_caching == 0);
398 		caching = ttm_uncached;
399 	}
400 
401 	err = ttm_tt_init(&tt->ttm, &bo->ttm, page_flags, caching, extra_pages);
402 	if (err) {
403 		kfree(tt);
404 		return NULL;
405 	}
406 
407 	return &tt->ttm;
408 }
409 
410 static int xe_ttm_tt_populate(struct ttm_device *ttm_dev, struct ttm_tt *tt,
411 			      struct ttm_operation_ctx *ctx)
412 {
413 	int err;
414 
415 	/*
416 	 * dma-bufs are not populated with pages, and the dma-
417 	 * addresses are set up when moved to XE_PL_TT.
418 	 */
419 	if (tt->page_flags & TTM_TT_FLAG_EXTERNAL)
420 		return 0;
421 
422 	err = ttm_pool_alloc(&ttm_dev->pool, tt, ctx);
423 	if (err)
424 		return err;
425 
426 	return err;
427 }
428 
429 static void xe_ttm_tt_unpopulate(struct ttm_device *ttm_dev, struct ttm_tt *tt)
430 {
431 	if (tt->page_flags & TTM_TT_FLAG_EXTERNAL)
432 		return;
433 
434 	xe_tt_unmap_sg(tt);
435 
436 	return ttm_pool_free(&ttm_dev->pool, tt);
437 }
438 
439 static void xe_ttm_tt_destroy(struct ttm_device *ttm_dev, struct ttm_tt *tt)
440 {
441 	ttm_tt_fini(tt);
442 	kfree(tt);
443 }
444 
445 static int xe_ttm_io_mem_reserve(struct ttm_device *bdev,
446 				 struct ttm_resource *mem)
447 {
448 	struct xe_device *xe = ttm_to_xe_device(bdev);
449 
450 	switch (mem->mem_type) {
451 	case XE_PL_SYSTEM:
452 	case XE_PL_TT:
453 		return 0;
454 	case XE_PL_VRAM0:
455 	case XE_PL_VRAM1: {
456 		struct xe_ttm_vram_mgr_resource *vres =
457 			to_xe_ttm_vram_mgr_resource(mem);
458 		struct xe_mem_region *vram = res_to_mem_region(mem);
459 
460 		if (vres->used_visible_size < mem->size)
461 			return -EINVAL;
462 
463 		mem->bus.offset = mem->start << PAGE_SHIFT;
464 
465 		if (vram->mapping &&
466 		    mem->placement & TTM_PL_FLAG_CONTIGUOUS)
467 			mem->bus.addr = (u8 __force *)vram->mapping +
468 				mem->bus.offset;
469 
470 		mem->bus.offset += vram->io_start;
471 		mem->bus.is_iomem = true;
472 
473 #if  !IS_ENABLED(CONFIG_X86)
474 		mem->bus.caching = ttm_write_combined;
475 #endif
476 		return 0;
477 	} case XE_PL_STOLEN:
478 		return xe_ttm_stolen_io_mem_reserve(xe, mem);
479 	default:
480 		return -EINVAL;
481 	}
482 }
483 
484 static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
485 				const struct ttm_operation_ctx *ctx)
486 {
487 	struct dma_resv_iter cursor;
488 	struct dma_fence *fence;
489 	struct drm_gem_object *obj = &bo->ttm.base;
490 	struct drm_gpuvm_bo *vm_bo;
491 	bool idle = false;
492 	int ret = 0;
493 
494 	dma_resv_assert_held(bo->ttm.base.resv);
495 
496 	if (!list_empty(&bo->ttm.base.gpuva.list)) {
497 		dma_resv_iter_begin(&cursor, bo->ttm.base.resv,
498 				    DMA_RESV_USAGE_BOOKKEEP);
499 		dma_resv_for_each_fence_unlocked(&cursor, fence)
500 			dma_fence_enable_sw_signaling(fence);
501 		dma_resv_iter_end(&cursor);
502 	}
503 
504 	drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
505 		struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm);
506 		struct drm_gpuva *gpuva;
507 
508 		if (!xe_vm_in_fault_mode(vm)) {
509 			drm_gpuvm_bo_evict(vm_bo, true);
510 			continue;
511 		}
512 
513 		if (!idle) {
514 			long timeout;
515 
516 			if (ctx->no_wait_gpu &&
517 			    !dma_resv_test_signaled(bo->ttm.base.resv,
518 						    DMA_RESV_USAGE_BOOKKEEP))
519 				return -EBUSY;
520 
521 			timeout = dma_resv_wait_timeout(bo->ttm.base.resv,
522 							DMA_RESV_USAGE_BOOKKEEP,
523 							ctx->interruptible,
524 							MAX_SCHEDULE_TIMEOUT);
525 			if (!timeout)
526 				return -ETIME;
527 			if (timeout < 0)
528 				return timeout;
529 
530 			idle = true;
531 		}
532 
533 		drm_gpuvm_bo_for_each_va(gpuva, vm_bo) {
534 			struct xe_vma *vma = gpuva_to_vma(gpuva);
535 
536 			trace_xe_vma_evict(vma);
537 			ret = xe_vm_invalidate_vma(vma);
538 			if (XE_WARN_ON(ret))
539 				return ret;
540 		}
541 	}
542 
543 	return ret;
544 }
545 
546 /*
547  * The dma-buf map_attachment() / unmap_attachment() is hooked up here.
548  * Note that unmapping the attachment is deferred to the next
549  * map_attachment time, or to bo destroy (after idling) whichever comes first.
550  * This is to avoid syncing before unmap_attachment(), assuming that the
551  * caller relies on idling the reservation object before moving the
552  * backing store out. Should that assumption not hold, then we will be able
553  * to unconditionally call unmap_attachment() when moving out to system.
554  */
555 static int xe_bo_move_dmabuf(struct ttm_buffer_object *ttm_bo,
556 			     struct ttm_resource *new_res)
557 {
558 	struct dma_buf_attachment *attach = ttm_bo->base.import_attach;
559 	struct xe_ttm_tt *xe_tt = container_of(ttm_bo->ttm, struct xe_ttm_tt,
560 					       ttm);
561 	struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
562 	struct sg_table *sg;
563 
564 	xe_assert(xe, attach);
565 	xe_assert(xe, ttm_bo->ttm);
566 
567 	if (new_res->mem_type == XE_PL_SYSTEM)
568 		goto out;
569 
570 	if (ttm_bo->sg) {
571 		dma_buf_unmap_attachment(attach, ttm_bo->sg, DMA_BIDIRECTIONAL);
572 		ttm_bo->sg = NULL;
573 	}
574 
575 	sg = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
576 	if (IS_ERR(sg))
577 		return PTR_ERR(sg);
578 
579 	ttm_bo->sg = sg;
580 	xe_tt->sg = sg;
581 
582 out:
583 	ttm_bo_move_null(ttm_bo, new_res);
584 
585 	return 0;
586 }
587 
588 /**
589  * xe_bo_move_notify - Notify subsystems of a pending move
590  * @bo: The buffer object
591  * @ctx: The struct ttm_operation_ctx controlling locking and waits.
592  *
593  * This function notifies subsystems of an upcoming buffer move.
594  * Upon receiving such a notification, subsystems should schedule
595  * halting access to the underlying pages and optionally add a fence
596  * to the buffer object's dma_resv object, that signals when access is
597  * stopped. The caller will wait on all dma_resv fences before
598  * starting the move.
599  *
600  * A subsystem may commence access to the object after obtaining
601  * bindings to the new backing memory under the object lock.
602  *
603  * Return: 0 on success, -EINTR or -ERESTARTSYS if interrupted in fault mode,
604  * negative error code on error.
605  */
606 static int xe_bo_move_notify(struct xe_bo *bo,
607 			     const struct ttm_operation_ctx *ctx)
608 {
609 	struct ttm_buffer_object *ttm_bo = &bo->ttm;
610 	struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
611 	struct ttm_resource *old_mem = ttm_bo->resource;
612 	u32 old_mem_type = old_mem ? old_mem->mem_type : XE_PL_SYSTEM;
613 	int ret;
614 
615 	/*
616 	 * If this starts to call into many components, consider
617 	 * using a notification chain here.
618 	 */
619 
620 	if (xe_bo_is_pinned(bo))
621 		return -EINVAL;
622 
623 	xe_bo_vunmap(bo);
624 	ret = xe_bo_trigger_rebind(xe, bo, ctx);
625 	if (ret)
626 		return ret;
627 
628 	/* Don't call move_notify() for imported dma-bufs. */
629 	if (ttm_bo->base.dma_buf && !ttm_bo->base.import_attach)
630 		dma_buf_move_notify(ttm_bo->base.dma_buf);
631 
632 	/*
633 	 * TTM has already nuked the mmap for us (see ttm_bo_unmap_virtual),
634 	 * so if we moved from VRAM make sure to unlink this from the userfault
635 	 * tracking.
636 	 */
637 	if (mem_type_is_vram(old_mem_type)) {
638 		mutex_lock(&xe->mem_access.vram_userfault.lock);
639 		if (!list_empty(&bo->vram_userfault_link))
640 			list_del_init(&bo->vram_userfault_link);
641 		mutex_unlock(&xe->mem_access.vram_userfault.lock);
642 	}
643 
644 	return 0;
645 }
646 
647 static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,
648 		      struct ttm_operation_ctx *ctx,
649 		      struct ttm_resource *new_mem,
650 		      struct ttm_place *hop)
651 {
652 	struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
653 	struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
654 	struct ttm_resource *old_mem = ttm_bo->resource;
655 	u32 old_mem_type = old_mem ? old_mem->mem_type : XE_PL_SYSTEM;
656 	struct ttm_tt *ttm = ttm_bo->ttm;
657 	struct xe_migrate *migrate = NULL;
658 	struct dma_fence *fence;
659 	bool move_lacks_source;
660 	bool tt_has_data;
661 	bool needs_clear;
662 	bool handle_system_ccs = (!IS_DGFX(xe) && xe_bo_needs_ccs_pages(bo) &&
663 				  ttm && ttm_tt_is_populated(ttm)) ? true : false;
664 	int ret = 0;
665 
666 	/* Bo creation path, moving to system or TT. */
667 	if ((!old_mem && ttm) && !handle_system_ccs) {
668 		if (new_mem->mem_type == XE_PL_TT)
669 			ret = xe_tt_map_sg(ttm);
670 		if (!ret)
671 			ttm_bo_move_null(ttm_bo, new_mem);
672 		goto out;
673 	}
674 
675 	if (ttm_bo->type == ttm_bo_type_sg) {
676 		ret = xe_bo_move_notify(bo, ctx);
677 		if (!ret)
678 			ret = xe_bo_move_dmabuf(ttm_bo, new_mem);
679 		return ret;
680 	}
681 
682 	tt_has_data = ttm && (ttm_tt_is_populated(ttm) ||
683 			      (ttm->page_flags & TTM_TT_FLAG_SWAPPED));
684 
685 	move_lacks_source = !old_mem || (handle_system_ccs ? (!bo->ccs_cleared) :
686 					 (!mem_type_is_vram(old_mem_type) && !tt_has_data));
687 
688 	needs_clear = (ttm && ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC) ||
689 		(!ttm && ttm_bo->type == ttm_bo_type_device);
690 
691 	if (new_mem->mem_type == XE_PL_TT) {
692 		ret = xe_tt_map_sg(ttm);
693 		if (ret)
694 			goto out;
695 	}
696 
697 	if ((move_lacks_source && !needs_clear)) {
698 		ttm_bo_move_null(ttm_bo, new_mem);
699 		goto out;
700 	}
701 
702 	if (old_mem_type == XE_PL_SYSTEM && new_mem->mem_type == XE_PL_TT && !handle_system_ccs) {
703 		ttm_bo_move_null(ttm_bo, new_mem);
704 		goto out;
705 	}
706 
707 	/*
708 	 * Failed multi-hop where the old_mem is still marked as
709 	 * TTM_PL_FLAG_TEMPORARY, should just be a dummy move.
710 	 */
711 	if (old_mem_type == XE_PL_TT &&
712 	    new_mem->mem_type == XE_PL_TT) {
713 		ttm_bo_move_null(ttm_bo, new_mem);
714 		goto out;
715 	}
716 
717 	if (!move_lacks_source && !xe_bo_is_pinned(bo)) {
718 		ret = xe_bo_move_notify(bo, ctx);
719 		if (ret)
720 			goto out;
721 	}
722 
723 	if (old_mem_type == XE_PL_TT &&
724 	    new_mem->mem_type == XE_PL_SYSTEM) {
725 		long timeout = dma_resv_wait_timeout(ttm_bo->base.resv,
726 						     DMA_RESV_USAGE_BOOKKEEP,
727 						     true,
728 						     MAX_SCHEDULE_TIMEOUT);
729 		if (timeout < 0) {
730 			ret = timeout;
731 			goto out;
732 		}
733 
734 		if (!handle_system_ccs) {
735 			ttm_bo_move_null(ttm_bo, new_mem);
736 			goto out;
737 		}
738 	}
739 
740 	if (!move_lacks_source &&
741 	    ((old_mem_type == XE_PL_SYSTEM && resource_is_vram(new_mem)) ||
742 	     (mem_type_is_vram(old_mem_type) &&
743 	      new_mem->mem_type == XE_PL_SYSTEM))) {
744 		hop->fpfn = 0;
745 		hop->lpfn = 0;
746 		hop->mem_type = XE_PL_TT;
747 		hop->flags = TTM_PL_FLAG_TEMPORARY;
748 		ret = -EMULTIHOP;
749 		goto out;
750 	}
751 
752 	if (bo->tile)
753 		migrate = bo->tile->migrate;
754 	else if (resource_is_vram(new_mem))
755 		migrate = mem_type_to_migrate(xe, new_mem->mem_type);
756 	else if (mem_type_is_vram(old_mem_type))
757 		migrate = mem_type_to_migrate(xe, old_mem_type);
758 	else
759 		migrate = xe->tiles[0].migrate;
760 
761 	xe_assert(xe, migrate);
762 	trace_xe_bo_move(bo, new_mem->mem_type, old_mem_type, move_lacks_source);
763 	if (xe_rpm_reclaim_safe(xe)) {
764 		/*
765 		 * We might be called through swapout in the validation path of
766 		 * another TTM device, so acquire rpm here.
767 		 */
768 		xe_pm_runtime_get(xe);
769 	} else {
770 		drm_WARN_ON(&xe->drm, handle_system_ccs);
771 		xe_pm_runtime_get_noresume(xe);
772 	}
773 
774 	if (xe_bo_is_pinned(bo) && !xe_bo_is_user(bo)) {
775 		/*
776 		 * Kernel memory that is pinned should only be moved on suspend
777 		 * / resume, some of the pinned memory is required for the
778 		 * device to resume / use the GPU to move other evicted memory
779 		 * (user memory) around. This likely could be optimized a bit
780 		 * futher where we find the minimum set of pinned memory
781 		 * required for resume but for simplity doing a memcpy for all
782 		 * pinned memory.
783 		 */
784 		ret = xe_bo_vmap(bo);
785 		if (!ret) {
786 			ret = ttm_bo_move_memcpy(ttm_bo, ctx, new_mem);
787 
788 			/* Create a new VMAP once kernel BO back in VRAM */
789 			if (!ret && resource_is_vram(new_mem)) {
790 				struct xe_mem_region *vram = res_to_mem_region(new_mem);
791 				void __iomem *new_addr = vram->mapping +
792 					(new_mem->start << PAGE_SHIFT);
793 
794 				if (XE_WARN_ON(new_mem->start == XE_BO_INVALID_OFFSET)) {
795 					ret = -EINVAL;
796 					xe_pm_runtime_put(xe);
797 					goto out;
798 				}
799 
800 				xe_assert(xe, new_mem->start ==
801 					  bo->placements->fpfn);
802 
803 				iosys_map_set_vaddr_iomem(&bo->vmap, new_addr);
804 			}
805 		}
806 	} else {
807 		if (move_lacks_source) {
808 			u32 flags = 0;
809 
810 			if (mem_type_is_vram(new_mem->mem_type))
811 				flags |= XE_MIGRATE_CLEAR_FLAG_FULL;
812 			else if (handle_system_ccs)
813 				flags |= XE_MIGRATE_CLEAR_FLAG_CCS_DATA;
814 
815 			fence = xe_migrate_clear(migrate, bo, new_mem, flags);
816 		}
817 		else
818 			fence = xe_migrate_copy(migrate, bo, bo, old_mem,
819 						new_mem, handle_system_ccs);
820 		if (IS_ERR(fence)) {
821 			ret = PTR_ERR(fence);
822 			xe_pm_runtime_put(xe);
823 			goto out;
824 		}
825 		if (!move_lacks_source) {
826 			ret = ttm_bo_move_accel_cleanup(ttm_bo, fence, evict,
827 							true, new_mem);
828 			if (ret) {
829 				dma_fence_wait(fence, false);
830 				ttm_bo_move_null(ttm_bo, new_mem);
831 				ret = 0;
832 			}
833 		} else {
834 			/*
835 			 * ttm_bo_move_accel_cleanup() may blow up if
836 			 * bo->resource == NULL, so just attach the
837 			 * fence and set the new resource.
838 			 */
839 			dma_resv_add_fence(ttm_bo->base.resv, fence,
840 					   DMA_RESV_USAGE_KERNEL);
841 			ttm_bo_move_null(ttm_bo, new_mem);
842 		}
843 
844 		dma_fence_put(fence);
845 	}
846 
847 	xe_pm_runtime_put(xe);
848 
849 out:
850 	if ((!ttm_bo->resource || ttm_bo->resource->mem_type == XE_PL_SYSTEM) &&
851 	    ttm_bo->ttm)
852 		xe_tt_unmap_sg(ttm_bo->ttm);
853 
854 	return ret;
855 }
856 
857 /**
858  * xe_bo_evict_pinned() - Evict a pinned VRAM object to system memory
859  * @bo: The buffer object to move.
860  *
861  * On successful completion, the object memory will be moved to sytem memory.
862  *
863  * This is needed to for special handling of pinned VRAM object during
864  * suspend-resume.
865  *
866  * Return: 0 on success. Negative error code on failure.
867  */
868 int xe_bo_evict_pinned(struct xe_bo *bo)
869 {
870 	struct ttm_place place = {
871 		.mem_type = XE_PL_TT,
872 	};
873 	struct ttm_placement placement = {
874 		.placement = &place,
875 		.num_placement = 1,
876 	};
877 	struct ttm_operation_ctx ctx = {
878 		.interruptible = false,
879 	};
880 	struct ttm_resource *new_mem;
881 	int ret;
882 
883 	xe_bo_assert_held(bo);
884 
885 	if (WARN_ON(!bo->ttm.resource))
886 		return -EINVAL;
887 
888 	if (WARN_ON(!xe_bo_is_pinned(bo)))
889 		return -EINVAL;
890 
891 	if (!xe_bo_is_vram(bo))
892 		return 0;
893 
894 	ret = ttm_bo_mem_space(&bo->ttm, &placement, &new_mem, &ctx);
895 	if (ret)
896 		return ret;
897 
898 	if (!bo->ttm.ttm) {
899 		bo->ttm.ttm = xe_ttm_tt_create(&bo->ttm, 0);
900 		if (!bo->ttm.ttm) {
901 			ret = -ENOMEM;
902 			goto err_res_free;
903 		}
904 	}
905 
906 	ret = ttm_bo_populate(&bo->ttm, &ctx);
907 	if (ret)
908 		goto err_res_free;
909 
910 	ret = dma_resv_reserve_fences(bo->ttm.base.resv, 1);
911 	if (ret)
912 		goto err_res_free;
913 
914 	ret = xe_bo_move(&bo->ttm, false, &ctx, new_mem, NULL);
915 	if (ret)
916 		goto err_res_free;
917 
918 	return 0;
919 
920 err_res_free:
921 	ttm_resource_free(&bo->ttm, &new_mem);
922 	return ret;
923 }
924 
925 /**
926  * xe_bo_restore_pinned() - Restore a pinned VRAM object
927  * @bo: The buffer object to move.
928  *
929  * On successful completion, the object memory will be moved back to VRAM.
930  *
931  * This is needed to for special handling of pinned VRAM object during
932  * suspend-resume.
933  *
934  * Return: 0 on success. Negative error code on failure.
935  */
936 int xe_bo_restore_pinned(struct xe_bo *bo)
937 {
938 	struct ttm_operation_ctx ctx = {
939 		.interruptible = false,
940 	};
941 	struct ttm_resource *new_mem;
942 	struct ttm_place *place = &bo->placements[0];
943 	int ret;
944 
945 	xe_bo_assert_held(bo);
946 
947 	if (WARN_ON(!bo->ttm.resource))
948 		return -EINVAL;
949 
950 	if (WARN_ON(!xe_bo_is_pinned(bo)))
951 		return -EINVAL;
952 
953 	if (WARN_ON(xe_bo_is_vram(bo)))
954 		return -EINVAL;
955 
956 	if (WARN_ON(!bo->ttm.ttm && !xe_bo_is_stolen(bo)))
957 		return -EINVAL;
958 
959 	if (!mem_type_is_vram(place->mem_type))
960 		return 0;
961 
962 	ret = ttm_bo_mem_space(&bo->ttm, &bo->placement, &new_mem, &ctx);
963 	if (ret)
964 		return ret;
965 
966 	ret = ttm_bo_populate(&bo->ttm, &ctx);
967 	if (ret)
968 		goto err_res_free;
969 
970 	ret = dma_resv_reserve_fences(bo->ttm.base.resv, 1);
971 	if (ret)
972 		goto err_res_free;
973 
974 	ret = xe_bo_move(&bo->ttm, false, &ctx, new_mem, NULL);
975 	if (ret)
976 		goto err_res_free;
977 
978 	return 0;
979 
980 err_res_free:
981 	ttm_resource_free(&bo->ttm, &new_mem);
982 	return ret;
983 }
984 
985 static unsigned long xe_ttm_io_mem_pfn(struct ttm_buffer_object *ttm_bo,
986 				       unsigned long page_offset)
987 {
988 	struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
989 	struct xe_res_cursor cursor;
990 	struct xe_mem_region *vram;
991 
992 	if (ttm_bo->resource->mem_type == XE_PL_STOLEN)
993 		return xe_ttm_stolen_io_offset(bo, page_offset << PAGE_SHIFT) >> PAGE_SHIFT;
994 
995 	vram = res_to_mem_region(ttm_bo->resource);
996 	xe_res_first(ttm_bo->resource, (u64)page_offset << PAGE_SHIFT, 0, &cursor);
997 	return (vram->io_start + cursor.start) >> PAGE_SHIFT;
998 }
999 
1000 static void __xe_bo_vunmap(struct xe_bo *bo);
1001 
1002 /*
1003  * TODO: Move this function to TTM so we don't rely on how TTM does its
1004  * locking, thereby abusing TTM internals.
1005  */
1006 static bool xe_ttm_bo_lock_in_destructor(struct ttm_buffer_object *ttm_bo)
1007 {
1008 	struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
1009 	bool locked;
1010 
1011 	xe_assert(xe, !kref_read(&ttm_bo->kref));
1012 
1013 	/*
1014 	 * We can typically only race with TTM trylocking under the
1015 	 * lru_lock, which will immediately be unlocked again since
1016 	 * the ttm_bo refcount is zero at this point. So trylocking *should*
1017 	 * always succeed here, as long as we hold the lru lock.
1018 	 */
1019 	spin_lock(&ttm_bo->bdev->lru_lock);
1020 	locked = dma_resv_trylock(ttm_bo->base.resv);
1021 	spin_unlock(&ttm_bo->bdev->lru_lock);
1022 	xe_assert(xe, locked);
1023 
1024 	return locked;
1025 }
1026 
1027 static void xe_ttm_bo_release_notify(struct ttm_buffer_object *ttm_bo)
1028 {
1029 	struct dma_resv_iter cursor;
1030 	struct dma_fence *fence;
1031 	struct dma_fence *replacement = NULL;
1032 	struct xe_bo *bo;
1033 
1034 	if (!xe_bo_is_xe_bo(ttm_bo))
1035 		return;
1036 
1037 	bo = ttm_to_xe_bo(ttm_bo);
1038 	xe_assert(xe_bo_device(bo), !(bo->created && kref_read(&ttm_bo->base.refcount)));
1039 
1040 	/*
1041 	 * Corner case where TTM fails to allocate memory and this BOs resv
1042 	 * still points the VMs resv
1043 	 */
1044 	if (ttm_bo->base.resv != &ttm_bo->base._resv)
1045 		return;
1046 
1047 	if (!xe_ttm_bo_lock_in_destructor(ttm_bo))
1048 		return;
1049 
1050 	/*
1051 	 * Scrub the preempt fences if any. The unbind fence is already
1052 	 * attached to the resv.
1053 	 * TODO: Don't do this for external bos once we scrub them after
1054 	 * unbind.
1055 	 */
1056 	dma_resv_for_each_fence(&cursor, ttm_bo->base.resv,
1057 				DMA_RESV_USAGE_BOOKKEEP, fence) {
1058 		if (xe_fence_is_xe_preempt(fence) &&
1059 		    !dma_fence_is_signaled(fence)) {
1060 			if (!replacement)
1061 				replacement = dma_fence_get_stub();
1062 
1063 			dma_resv_replace_fences(ttm_bo->base.resv,
1064 						fence->context,
1065 						replacement,
1066 						DMA_RESV_USAGE_BOOKKEEP);
1067 		}
1068 	}
1069 	dma_fence_put(replacement);
1070 
1071 	dma_resv_unlock(ttm_bo->base.resv);
1072 }
1073 
1074 static void xe_ttm_bo_delete_mem_notify(struct ttm_buffer_object *ttm_bo)
1075 {
1076 	if (!xe_bo_is_xe_bo(ttm_bo))
1077 		return;
1078 
1079 	/*
1080 	 * Object is idle and about to be destroyed. Release the
1081 	 * dma-buf attachment.
1082 	 */
1083 	if (ttm_bo->type == ttm_bo_type_sg && ttm_bo->sg) {
1084 		struct xe_ttm_tt *xe_tt = container_of(ttm_bo->ttm,
1085 						       struct xe_ttm_tt, ttm);
1086 
1087 		dma_buf_unmap_attachment(ttm_bo->base.import_attach, ttm_bo->sg,
1088 					 DMA_BIDIRECTIONAL);
1089 		ttm_bo->sg = NULL;
1090 		xe_tt->sg = NULL;
1091 	}
1092 }
1093 
1094 static void xe_ttm_bo_purge(struct ttm_buffer_object *ttm_bo, struct ttm_operation_ctx *ctx)
1095 {
1096 	struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
1097 
1098 	if (ttm_bo->ttm) {
1099 		struct ttm_placement place = {};
1100 		int ret = ttm_bo_validate(ttm_bo, &place, ctx);
1101 
1102 		drm_WARN_ON(&xe->drm, ret);
1103 	}
1104 }
1105 
1106 static void xe_ttm_bo_swap_notify(struct ttm_buffer_object *ttm_bo)
1107 {
1108 	struct ttm_operation_ctx ctx = {
1109 		.interruptible = false
1110 	};
1111 
1112 	if (ttm_bo->ttm) {
1113 		struct xe_ttm_tt *xe_tt =
1114 			container_of(ttm_bo->ttm, struct xe_ttm_tt, ttm);
1115 
1116 		if (xe_tt->purgeable)
1117 			xe_ttm_bo_purge(ttm_bo, &ctx);
1118 	}
1119 }
1120 
1121 const struct ttm_device_funcs xe_ttm_funcs = {
1122 	.ttm_tt_create = xe_ttm_tt_create,
1123 	.ttm_tt_populate = xe_ttm_tt_populate,
1124 	.ttm_tt_unpopulate = xe_ttm_tt_unpopulate,
1125 	.ttm_tt_destroy = xe_ttm_tt_destroy,
1126 	.evict_flags = xe_evict_flags,
1127 	.move = xe_bo_move,
1128 	.io_mem_reserve = xe_ttm_io_mem_reserve,
1129 	.io_mem_pfn = xe_ttm_io_mem_pfn,
1130 	.release_notify = xe_ttm_bo_release_notify,
1131 	.eviction_valuable = ttm_bo_eviction_valuable,
1132 	.delete_mem_notify = xe_ttm_bo_delete_mem_notify,
1133 	.swap_notify = xe_ttm_bo_swap_notify,
1134 };
1135 
1136 static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo)
1137 {
1138 	struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
1139 	struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
1140 
1141 	if (bo->ttm.base.import_attach)
1142 		drm_prime_gem_destroy(&bo->ttm.base, NULL);
1143 	drm_gem_object_release(&bo->ttm.base);
1144 
1145 	xe_assert(xe, list_empty(&ttm_bo->base.gpuva.list));
1146 
1147 	if (bo->ggtt_node && bo->ggtt_node->base.size)
1148 		xe_ggtt_remove_bo(bo->tile->mem.ggtt, bo);
1149 
1150 #ifdef CONFIG_PROC_FS
1151 	if (bo->client)
1152 		xe_drm_client_remove_bo(bo);
1153 #endif
1154 
1155 	if (bo->vm && xe_bo_is_user(bo))
1156 		xe_vm_put(bo->vm);
1157 
1158 	mutex_lock(&xe->mem_access.vram_userfault.lock);
1159 	if (!list_empty(&bo->vram_userfault_link))
1160 		list_del(&bo->vram_userfault_link);
1161 	mutex_unlock(&xe->mem_access.vram_userfault.lock);
1162 
1163 	kfree(bo);
1164 }
1165 
1166 static void xe_gem_object_free(struct drm_gem_object *obj)
1167 {
1168 	/* Our BO reference counting scheme works as follows:
1169 	 *
1170 	 * The gem object kref is typically used throughout the driver,
1171 	 * and the gem object holds a ttm_buffer_object refcount, so
1172 	 * that when the last gem object reference is put, which is when
1173 	 * we end up in this function, we put also that ttm_buffer_object
1174 	 * refcount. Anything using gem interfaces is then no longer
1175 	 * allowed to access the object in a way that requires a gem
1176 	 * refcount, including locking the object.
1177 	 *
1178 	 * driver ttm callbacks is allowed to use the ttm_buffer_object
1179 	 * refcount directly if needed.
1180 	 */
1181 	__xe_bo_vunmap(gem_to_xe_bo(obj));
1182 	ttm_bo_put(container_of(obj, struct ttm_buffer_object, base));
1183 }
1184 
1185 static void xe_gem_object_close(struct drm_gem_object *obj,
1186 				struct drm_file *file_priv)
1187 {
1188 	struct xe_bo *bo = gem_to_xe_bo(obj);
1189 
1190 	if (bo->vm && !xe_vm_in_fault_mode(bo->vm)) {
1191 		xe_assert(xe_bo_device(bo), xe_bo_is_user(bo));
1192 
1193 		xe_bo_lock(bo, false);
1194 		ttm_bo_set_bulk_move(&bo->ttm, NULL);
1195 		xe_bo_unlock(bo);
1196 	}
1197 }
1198 
1199 static vm_fault_t xe_gem_fault(struct vm_fault *vmf)
1200 {
1201 	struct ttm_buffer_object *tbo = vmf->vma->vm_private_data;
1202 	struct drm_device *ddev = tbo->base.dev;
1203 	struct xe_device *xe = to_xe_device(ddev);
1204 	struct xe_bo *bo = ttm_to_xe_bo(tbo);
1205 	bool needs_rpm = bo->flags & XE_BO_FLAG_VRAM_MASK;
1206 	vm_fault_t ret;
1207 	int idx;
1208 
1209 	if (needs_rpm)
1210 		xe_pm_runtime_get(xe);
1211 
1212 	ret = ttm_bo_vm_reserve(tbo, vmf);
1213 	if (ret)
1214 		goto out;
1215 
1216 	if (drm_dev_enter(ddev, &idx)) {
1217 		trace_xe_bo_cpu_fault(bo);
1218 
1219 		ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot,
1220 					       TTM_BO_VM_NUM_PREFAULT);
1221 		drm_dev_exit(idx);
1222 	} else {
1223 		ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot);
1224 	}
1225 
1226 	if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
1227 		goto out;
1228 	/*
1229 	 * ttm_bo_vm_reserve() already has dma_resv_lock.
1230 	 */
1231 	if (ret == VM_FAULT_NOPAGE && mem_type_is_vram(tbo->resource->mem_type)) {
1232 		mutex_lock(&xe->mem_access.vram_userfault.lock);
1233 		if (list_empty(&bo->vram_userfault_link))
1234 			list_add(&bo->vram_userfault_link, &xe->mem_access.vram_userfault.list);
1235 		mutex_unlock(&xe->mem_access.vram_userfault.lock);
1236 	}
1237 
1238 	dma_resv_unlock(tbo->base.resv);
1239 out:
1240 	if (needs_rpm)
1241 		xe_pm_runtime_put(xe);
1242 
1243 	return ret;
1244 }
1245 
1246 static const struct vm_operations_struct xe_gem_vm_ops = {
1247 	.fault = xe_gem_fault,
1248 	.open = ttm_bo_vm_open,
1249 	.close = ttm_bo_vm_close,
1250 	.access = ttm_bo_vm_access
1251 };
1252 
1253 static const struct drm_gem_object_funcs xe_gem_object_funcs = {
1254 	.free = xe_gem_object_free,
1255 	.close = xe_gem_object_close,
1256 	.mmap = drm_gem_ttm_mmap,
1257 	.export = xe_gem_prime_export,
1258 	.vm_ops = &xe_gem_vm_ops,
1259 };
1260 
1261 /**
1262  * xe_bo_alloc - Allocate storage for a struct xe_bo
1263  *
1264  * This funcition is intended to allocate storage to be used for input
1265  * to __xe_bo_create_locked(), in the case a pointer to the bo to be
1266  * created is needed before the call to __xe_bo_create_locked().
1267  * If __xe_bo_create_locked ends up never to be called, then the
1268  * storage allocated with this function needs to be freed using
1269  * xe_bo_free().
1270  *
1271  * Return: A pointer to an uninitialized struct xe_bo on success,
1272  * ERR_PTR(-ENOMEM) on error.
1273  */
1274 struct xe_bo *xe_bo_alloc(void)
1275 {
1276 	struct xe_bo *bo = kzalloc(sizeof(*bo), GFP_KERNEL);
1277 
1278 	if (!bo)
1279 		return ERR_PTR(-ENOMEM);
1280 
1281 	return bo;
1282 }
1283 
1284 /**
1285  * xe_bo_free - Free storage allocated using xe_bo_alloc()
1286  * @bo: The buffer object storage.
1287  *
1288  * Refer to xe_bo_alloc() documentation for valid use-cases.
1289  */
1290 void xe_bo_free(struct xe_bo *bo)
1291 {
1292 	kfree(bo);
1293 }
1294 
1295 struct xe_bo *___xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo,
1296 				     struct xe_tile *tile, struct dma_resv *resv,
1297 				     struct ttm_lru_bulk_move *bulk, size_t size,
1298 				     u16 cpu_caching, enum ttm_bo_type type,
1299 				     u32 flags)
1300 {
1301 	struct ttm_operation_ctx ctx = {
1302 		.interruptible = true,
1303 		.no_wait_gpu = false,
1304 	};
1305 	struct ttm_placement *placement;
1306 	uint32_t alignment;
1307 	size_t aligned_size;
1308 	int err;
1309 
1310 	/* Only kernel objects should set GT */
1311 	xe_assert(xe, !tile || type == ttm_bo_type_kernel);
1312 
1313 	if (XE_WARN_ON(!size)) {
1314 		xe_bo_free(bo);
1315 		return ERR_PTR(-EINVAL);
1316 	}
1317 
1318 	if (flags & (XE_BO_FLAG_VRAM_MASK | XE_BO_FLAG_STOLEN) &&
1319 	    !(flags & XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE) &&
1320 	    ((xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) ||
1321 	     (flags & (XE_BO_FLAG_NEEDS_64K | XE_BO_FLAG_NEEDS_2M)))) {
1322 		size_t align = flags & XE_BO_FLAG_NEEDS_2M ? SZ_2M : SZ_64K;
1323 
1324 		aligned_size = ALIGN(size, align);
1325 		if (type != ttm_bo_type_device)
1326 			size = ALIGN(size, align);
1327 		flags |= XE_BO_FLAG_INTERNAL_64K;
1328 		alignment = align >> PAGE_SHIFT;
1329 	} else {
1330 		aligned_size = ALIGN(size, SZ_4K);
1331 		flags &= ~XE_BO_FLAG_INTERNAL_64K;
1332 		alignment = SZ_4K >> PAGE_SHIFT;
1333 	}
1334 
1335 	if (type == ttm_bo_type_device && aligned_size != size)
1336 		return ERR_PTR(-EINVAL);
1337 
1338 	if (!bo) {
1339 		bo = xe_bo_alloc();
1340 		if (IS_ERR(bo))
1341 			return bo;
1342 	}
1343 
1344 	bo->ccs_cleared = false;
1345 	bo->tile = tile;
1346 	bo->size = size;
1347 	bo->flags = flags;
1348 	bo->cpu_caching = cpu_caching;
1349 	bo->ttm.base.funcs = &xe_gem_object_funcs;
1350 	bo->ttm.priority = XE_BO_PRIORITY_NORMAL;
1351 	INIT_LIST_HEAD(&bo->pinned_link);
1352 #ifdef CONFIG_PROC_FS
1353 	INIT_LIST_HEAD(&bo->client_link);
1354 #endif
1355 	INIT_LIST_HEAD(&bo->vram_userfault_link);
1356 
1357 	drm_gem_private_object_init(&xe->drm, &bo->ttm.base, size);
1358 
1359 	if (resv) {
1360 		ctx.allow_res_evict = !(flags & XE_BO_FLAG_NO_RESV_EVICT);
1361 		ctx.resv = resv;
1362 	}
1363 
1364 	if (!(flags & XE_BO_FLAG_FIXED_PLACEMENT)) {
1365 		err = __xe_bo_placement_for_flags(xe, bo, bo->flags);
1366 		if (WARN_ON(err)) {
1367 			xe_ttm_bo_destroy(&bo->ttm);
1368 			return ERR_PTR(err);
1369 		}
1370 	}
1371 
1372 	/* Defer populating type_sg bos */
1373 	placement = (type == ttm_bo_type_sg ||
1374 		     bo->flags & XE_BO_FLAG_DEFER_BACKING) ? &sys_placement :
1375 		&bo->placement;
1376 	err = ttm_bo_init_reserved(&xe->ttm, &bo->ttm, type,
1377 				   placement, alignment,
1378 				   &ctx, NULL, resv, xe_ttm_bo_destroy);
1379 	if (err)
1380 		return ERR_PTR(err);
1381 
1382 	/*
1383 	 * The VRAM pages underneath are potentially still being accessed by the
1384 	 * GPU, as per async GPU clearing and async evictions. However TTM makes
1385 	 * sure to add any corresponding move/clear fences into the objects
1386 	 * dma-resv using the DMA_RESV_USAGE_KERNEL slot.
1387 	 *
1388 	 * For KMD internal buffers we don't care about GPU clearing, however we
1389 	 * still need to handle async evictions, where the VRAM is still being
1390 	 * accessed by the GPU. Most internal callers are not expecting this,
1391 	 * since they are missing the required synchronisation before accessing
1392 	 * the memory. To keep things simple just sync wait any kernel fences
1393 	 * here, if the buffer is designated KMD internal.
1394 	 *
1395 	 * For normal userspace objects we should already have the required
1396 	 * pipelining or sync waiting elsewhere, since we already have to deal
1397 	 * with things like async GPU clearing.
1398 	 */
1399 	if (type == ttm_bo_type_kernel) {
1400 		long timeout = dma_resv_wait_timeout(bo->ttm.base.resv,
1401 						     DMA_RESV_USAGE_KERNEL,
1402 						     ctx.interruptible,
1403 						     MAX_SCHEDULE_TIMEOUT);
1404 
1405 		if (timeout < 0) {
1406 			if (!resv)
1407 				dma_resv_unlock(bo->ttm.base.resv);
1408 			xe_bo_put(bo);
1409 			return ERR_PTR(timeout);
1410 		}
1411 	}
1412 
1413 	bo->created = true;
1414 	if (bulk)
1415 		ttm_bo_set_bulk_move(&bo->ttm, bulk);
1416 	else
1417 		ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);
1418 
1419 	return bo;
1420 }
1421 
1422 static int __xe_bo_fixed_placement(struct xe_device *xe,
1423 				   struct xe_bo *bo,
1424 				   u32 flags,
1425 				   u64 start, u64 end, u64 size)
1426 {
1427 	struct ttm_place *place = bo->placements;
1428 
1429 	if (flags & (XE_BO_FLAG_USER | XE_BO_FLAG_SYSTEM))
1430 		return -EINVAL;
1431 
1432 	place->flags = TTM_PL_FLAG_CONTIGUOUS;
1433 	place->fpfn = start >> PAGE_SHIFT;
1434 	place->lpfn = end >> PAGE_SHIFT;
1435 
1436 	switch (flags & (XE_BO_FLAG_STOLEN | XE_BO_FLAG_VRAM_MASK)) {
1437 	case XE_BO_FLAG_VRAM0:
1438 		place->mem_type = XE_PL_VRAM0;
1439 		break;
1440 	case XE_BO_FLAG_VRAM1:
1441 		place->mem_type = XE_PL_VRAM1;
1442 		break;
1443 	case XE_BO_FLAG_STOLEN:
1444 		place->mem_type = XE_PL_STOLEN;
1445 		break;
1446 
1447 	default:
1448 		/* 0 or multiple of the above set */
1449 		return -EINVAL;
1450 	}
1451 
1452 	bo->placement = (struct ttm_placement) {
1453 		.num_placement = 1,
1454 		.placement = place,
1455 	};
1456 
1457 	return 0;
1458 }
1459 
1460 static struct xe_bo *
1461 __xe_bo_create_locked(struct xe_device *xe,
1462 		      struct xe_tile *tile, struct xe_vm *vm,
1463 		      size_t size, u64 start, u64 end,
1464 		      u16 cpu_caching, enum ttm_bo_type type, u32 flags,
1465 		      u64 alignment)
1466 {
1467 	struct xe_bo *bo = NULL;
1468 	int err;
1469 
1470 	if (vm)
1471 		xe_vm_assert_held(vm);
1472 
1473 	if (start || end != ~0ULL) {
1474 		bo = xe_bo_alloc();
1475 		if (IS_ERR(bo))
1476 			return bo;
1477 
1478 		flags |= XE_BO_FLAG_FIXED_PLACEMENT;
1479 		err = __xe_bo_fixed_placement(xe, bo, flags, start, end, size);
1480 		if (err) {
1481 			xe_bo_free(bo);
1482 			return ERR_PTR(err);
1483 		}
1484 	}
1485 
1486 	bo = ___xe_bo_create_locked(xe, bo, tile, vm ? xe_vm_resv(vm) : NULL,
1487 				    vm && !xe_vm_in_fault_mode(vm) &&
1488 				    flags & XE_BO_FLAG_USER ?
1489 				    &vm->lru_bulk_move : NULL, size,
1490 				    cpu_caching, type, flags);
1491 	if (IS_ERR(bo))
1492 		return bo;
1493 
1494 	bo->min_align = alignment;
1495 
1496 	/*
1497 	 * Note that instead of taking a reference no the drm_gpuvm_resv_bo(),
1498 	 * to ensure the shared resv doesn't disappear under the bo, the bo
1499 	 * will keep a reference to the vm, and avoid circular references
1500 	 * by having all the vm's bo refereferences released at vm close
1501 	 * time.
1502 	 */
1503 	if (vm && xe_bo_is_user(bo))
1504 		xe_vm_get(vm);
1505 	bo->vm = vm;
1506 
1507 	if (bo->flags & XE_BO_FLAG_GGTT) {
1508 		if (!tile && flags & XE_BO_FLAG_STOLEN)
1509 			tile = xe_device_get_root_tile(xe);
1510 
1511 		xe_assert(xe, tile);
1512 
1513 		if (flags & XE_BO_FLAG_FIXED_PLACEMENT) {
1514 			err = xe_ggtt_insert_bo_at(tile->mem.ggtt, bo,
1515 						   start + bo->size, U64_MAX);
1516 		} else {
1517 			err = xe_ggtt_insert_bo(tile->mem.ggtt, bo);
1518 		}
1519 		if (err)
1520 			goto err_unlock_put_bo;
1521 	}
1522 
1523 	return bo;
1524 
1525 err_unlock_put_bo:
1526 	__xe_bo_unset_bulk_move(bo);
1527 	xe_bo_unlock_vm_held(bo);
1528 	xe_bo_put(bo);
1529 	return ERR_PTR(err);
1530 }
1531 
1532 struct xe_bo *
1533 xe_bo_create_locked_range(struct xe_device *xe,
1534 			  struct xe_tile *tile, struct xe_vm *vm,
1535 			  size_t size, u64 start, u64 end,
1536 			  enum ttm_bo_type type, u32 flags, u64 alignment)
1537 {
1538 	return __xe_bo_create_locked(xe, tile, vm, size, start, end, 0, type,
1539 				     flags, alignment);
1540 }
1541 
1542 struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile,
1543 				  struct xe_vm *vm, size_t size,
1544 				  enum ttm_bo_type type, u32 flags)
1545 {
1546 	return __xe_bo_create_locked(xe, tile, vm, size, 0, ~0ULL, 0, type,
1547 				     flags, 0);
1548 }
1549 
1550 struct xe_bo *xe_bo_create_user(struct xe_device *xe, struct xe_tile *tile,
1551 				struct xe_vm *vm, size_t size,
1552 				u16 cpu_caching,
1553 				u32 flags)
1554 {
1555 	struct xe_bo *bo = __xe_bo_create_locked(xe, tile, vm, size, 0, ~0ULL,
1556 						 cpu_caching, ttm_bo_type_device,
1557 						 flags | XE_BO_FLAG_USER, 0);
1558 	if (!IS_ERR(bo))
1559 		xe_bo_unlock_vm_held(bo);
1560 
1561 	return bo;
1562 }
1563 
1564 struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_tile *tile,
1565 			   struct xe_vm *vm, size_t size,
1566 			   enum ttm_bo_type type, u32 flags)
1567 {
1568 	struct xe_bo *bo = xe_bo_create_locked(xe, tile, vm, size, type, flags);
1569 
1570 	if (!IS_ERR(bo))
1571 		xe_bo_unlock_vm_held(bo);
1572 
1573 	return bo;
1574 }
1575 
1576 struct xe_bo *xe_bo_create_pin_map_at(struct xe_device *xe, struct xe_tile *tile,
1577 				      struct xe_vm *vm,
1578 				      size_t size, u64 offset,
1579 				      enum ttm_bo_type type, u32 flags)
1580 {
1581 	return xe_bo_create_pin_map_at_aligned(xe, tile, vm, size, offset,
1582 					       type, flags, 0);
1583 }
1584 
1585 struct xe_bo *xe_bo_create_pin_map_at_aligned(struct xe_device *xe,
1586 					      struct xe_tile *tile,
1587 					      struct xe_vm *vm,
1588 					      size_t size, u64 offset,
1589 					      enum ttm_bo_type type, u32 flags,
1590 					      u64 alignment)
1591 {
1592 	struct xe_bo *bo;
1593 	int err;
1594 	u64 start = offset == ~0ull ? 0 : offset;
1595 	u64 end = offset == ~0ull ? offset : start + size;
1596 
1597 	if (flags & XE_BO_FLAG_STOLEN &&
1598 	    xe_ttm_stolen_cpu_access_needs_ggtt(xe))
1599 		flags |= XE_BO_FLAG_GGTT;
1600 
1601 	bo = xe_bo_create_locked_range(xe, tile, vm, size, start, end, type,
1602 				       flags | XE_BO_FLAG_NEEDS_CPU_ACCESS,
1603 				       alignment);
1604 	if (IS_ERR(bo))
1605 		return bo;
1606 
1607 	err = xe_bo_pin(bo);
1608 	if (err)
1609 		goto err_put;
1610 
1611 	err = xe_bo_vmap(bo);
1612 	if (err)
1613 		goto err_unpin;
1614 
1615 	xe_bo_unlock_vm_held(bo);
1616 
1617 	return bo;
1618 
1619 err_unpin:
1620 	xe_bo_unpin(bo);
1621 err_put:
1622 	xe_bo_unlock_vm_held(bo);
1623 	xe_bo_put(bo);
1624 	return ERR_PTR(err);
1625 }
1626 
1627 struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile,
1628 				   struct xe_vm *vm, size_t size,
1629 				   enum ttm_bo_type type, u32 flags)
1630 {
1631 	return xe_bo_create_pin_map_at(xe, tile, vm, size, ~0ull, type, flags);
1632 }
1633 
1634 struct xe_bo *xe_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile,
1635 				     const void *data, size_t size,
1636 				     enum ttm_bo_type type, u32 flags)
1637 {
1638 	struct xe_bo *bo = xe_bo_create_pin_map(xe, tile, NULL,
1639 						ALIGN(size, PAGE_SIZE),
1640 						type, flags);
1641 	if (IS_ERR(bo))
1642 		return bo;
1643 
1644 	xe_map_memcpy_to(xe, &bo->vmap, 0, data, size);
1645 
1646 	return bo;
1647 }
1648 
1649 static void __xe_bo_unpin_map_no_vm(void *arg)
1650 {
1651 	xe_bo_unpin_map_no_vm(arg);
1652 }
1653 
1654 struct xe_bo *xe_managed_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile,
1655 					   size_t size, u32 flags)
1656 {
1657 	struct xe_bo *bo;
1658 	int ret;
1659 
1660 	bo = xe_bo_create_pin_map(xe, tile, NULL, size, ttm_bo_type_kernel, flags);
1661 	if (IS_ERR(bo))
1662 		return bo;
1663 
1664 	ret = devm_add_action_or_reset(xe->drm.dev, __xe_bo_unpin_map_no_vm, bo);
1665 	if (ret)
1666 		return ERR_PTR(ret);
1667 
1668 	return bo;
1669 }
1670 
1671 struct xe_bo *xe_managed_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile,
1672 					     const void *data, size_t size, u32 flags)
1673 {
1674 	struct xe_bo *bo = xe_managed_bo_create_pin_map(xe, tile, ALIGN(size, PAGE_SIZE), flags);
1675 
1676 	if (IS_ERR(bo))
1677 		return bo;
1678 
1679 	xe_map_memcpy_to(xe, &bo->vmap, 0, data, size);
1680 
1681 	return bo;
1682 }
1683 
1684 /**
1685  * xe_managed_bo_reinit_in_vram
1686  * @xe: xe device
1687  * @tile: Tile where the new buffer will be created
1688  * @src: Managed buffer object allocated in system memory
1689  *
1690  * Replace a managed src buffer object allocated in system memory with a new
1691  * one allocated in vram, copying the data between them.
1692  * Buffer object in VRAM is not going to have the same GGTT address, the caller
1693  * is responsible for making sure that any old references to it are updated.
1694  *
1695  * Returns 0 for success, negative error code otherwise.
1696  */
1697 int xe_managed_bo_reinit_in_vram(struct xe_device *xe, struct xe_tile *tile, struct xe_bo **src)
1698 {
1699 	struct xe_bo *bo;
1700 	u32 dst_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | XE_BO_FLAG_GGTT;
1701 
1702 	dst_flags |= (*src)->flags & XE_BO_FLAG_GGTT_INVALIDATE;
1703 
1704 	xe_assert(xe, IS_DGFX(xe));
1705 	xe_assert(xe, !(*src)->vmap.is_iomem);
1706 
1707 	bo = xe_managed_bo_create_from_data(xe, tile, (*src)->vmap.vaddr,
1708 					    (*src)->size, dst_flags);
1709 	if (IS_ERR(bo))
1710 		return PTR_ERR(bo);
1711 
1712 	devm_release_action(xe->drm.dev, __xe_bo_unpin_map_no_vm, *src);
1713 	*src = bo;
1714 
1715 	return 0;
1716 }
1717 
1718 /*
1719  * XXX: This is in the VM bind data path, likely should calculate this once and
1720  * store, with a recalculation if the BO is moved.
1721  */
1722 uint64_t vram_region_gpu_offset(struct ttm_resource *res)
1723 {
1724 	struct xe_device *xe = ttm_to_xe_device(res->bo->bdev);
1725 
1726 	if (res->mem_type == XE_PL_STOLEN)
1727 		return xe_ttm_stolen_gpu_offset(xe);
1728 
1729 	return res_to_mem_region(res)->dpa_base;
1730 }
1731 
1732 /**
1733  * xe_bo_pin_external - pin an external BO
1734  * @bo: buffer object to be pinned
1735  *
1736  * Pin an external (not tied to a VM, can be exported via dma-buf / prime FD)
1737  * BO. Unique call compared to xe_bo_pin as this function has it own set of
1738  * asserts and code to ensure evict / restore on suspend / resume.
1739  *
1740  * Returns 0 for success, negative error code otherwise.
1741  */
1742 int xe_bo_pin_external(struct xe_bo *bo)
1743 {
1744 	struct xe_device *xe = xe_bo_device(bo);
1745 	int err;
1746 
1747 	xe_assert(xe, !bo->vm);
1748 	xe_assert(xe, xe_bo_is_user(bo));
1749 
1750 	if (!xe_bo_is_pinned(bo)) {
1751 		err = xe_bo_validate(bo, NULL, false);
1752 		if (err)
1753 			return err;
1754 
1755 		if (xe_bo_is_vram(bo)) {
1756 			spin_lock(&xe->pinned.lock);
1757 			list_add_tail(&bo->pinned_link,
1758 				      &xe->pinned.external_vram);
1759 			spin_unlock(&xe->pinned.lock);
1760 		}
1761 	}
1762 
1763 	ttm_bo_pin(&bo->ttm);
1764 
1765 	/*
1766 	 * FIXME: If we always use the reserve / unreserve functions for locking
1767 	 * we do not need this.
1768 	 */
1769 	ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);
1770 
1771 	return 0;
1772 }
1773 
1774 int xe_bo_pin(struct xe_bo *bo)
1775 {
1776 	struct ttm_place *place = &bo->placements[0];
1777 	struct xe_device *xe = xe_bo_device(bo);
1778 	int err;
1779 
1780 	/* We currently don't expect user BO to be pinned */
1781 	xe_assert(xe, !xe_bo_is_user(bo));
1782 
1783 	/* Pinned object must be in GGTT or have pinned flag */
1784 	xe_assert(xe, bo->flags & (XE_BO_FLAG_PINNED |
1785 				   XE_BO_FLAG_GGTT));
1786 
1787 	/*
1788 	 * No reason we can't support pinning imported dma-bufs we just don't
1789 	 * expect to pin an imported dma-buf.
1790 	 */
1791 	xe_assert(xe, !bo->ttm.base.import_attach);
1792 
1793 	/* We only expect at most 1 pin */
1794 	xe_assert(xe, !xe_bo_is_pinned(bo));
1795 
1796 	err = xe_bo_validate(bo, NULL, false);
1797 	if (err)
1798 		return err;
1799 
1800 	/*
1801 	 * For pinned objects in on DGFX, which are also in vram, we expect
1802 	 * these to be in contiguous VRAM memory. Required eviction / restore
1803 	 * during suspend / resume (force restore to same physical address).
1804 	 */
1805 	if (IS_DGFX(xe) && !(IS_ENABLED(CONFIG_DRM_XE_DEBUG) &&
1806 	    bo->flags & XE_BO_FLAG_INTERNAL_TEST)) {
1807 		if (mem_type_is_vram(place->mem_type)) {
1808 			xe_assert(xe, place->flags & TTM_PL_FLAG_CONTIGUOUS);
1809 
1810 			place->fpfn = (xe_bo_addr(bo, 0, PAGE_SIZE) -
1811 				       vram_region_gpu_offset(bo->ttm.resource)) >> PAGE_SHIFT;
1812 			place->lpfn = place->fpfn + (bo->size >> PAGE_SHIFT);
1813 		}
1814 	}
1815 
1816 	if (mem_type_is_vram(place->mem_type) || bo->flags & XE_BO_FLAG_GGTT) {
1817 		spin_lock(&xe->pinned.lock);
1818 		list_add_tail(&bo->pinned_link, &xe->pinned.kernel_bo_present);
1819 		spin_unlock(&xe->pinned.lock);
1820 	}
1821 
1822 	ttm_bo_pin(&bo->ttm);
1823 
1824 	/*
1825 	 * FIXME: If we always use the reserve / unreserve functions for locking
1826 	 * we do not need this.
1827 	 */
1828 	ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);
1829 
1830 	return 0;
1831 }
1832 
1833 /**
1834  * xe_bo_unpin_external - unpin an external BO
1835  * @bo: buffer object to be unpinned
1836  *
1837  * Unpin an external (not tied to a VM, can be exported via dma-buf / prime FD)
1838  * BO. Unique call compared to xe_bo_unpin as this function has it own set of
1839  * asserts and code to ensure evict / restore on suspend / resume.
1840  *
1841  * Returns 0 for success, negative error code otherwise.
1842  */
1843 void xe_bo_unpin_external(struct xe_bo *bo)
1844 {
1845 	struct xe_device *xe = xe_bo_device(bo);
1846 
1847 	xe_assert(xe, !bo->vm);
1848 	xe_assert(xe, xe_bo_is_pinned(bo));
1849 	xe_assert(xe, xe_bo_is_user(bo));
1850 
1851 	spin_lock(&xe->pinned.lock);
1852 	if (bo->ttm.pin_count == 1 && !list_empty(&bo->pinned_link))
1853 		list_del_init(&bo->pinned_link);
1854 	spin_unlock(&xe->pinned.lock);
1855 
1856 	ttm_bo_unpin(&bo->ttm);
1857 
1858 	/*
1859 	 * FIXME: If we always use the reserve / unreserve functions for locking
1860 	 * we do not need this.
1861 	 */
1862 	ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);
1863 }
1864 
1865 void xe_bo_unpin(struct xe_bo *bo)
1866 {
1867 	struct ttm_place *place = &bo->placements[0];
1868 	struct xe_device *xe = xe_bo_device(bo);
1869 
1870 	xe_assert(xe, !bo->ttm.base.import_attach);
1871 	xe_assert(xe, xe_bo_is_pinned(bo));
1872 
1873 	if (mem_type_is_vram(place->mem_type) || bo->flags & XE_BO_FLAG_GGTT) {
1874 		spin_lock(&xe->pinned.lock);
1875 		xe_assert(xe, !list_empty(&bo->pinned_link));
1876 		list_del_init(&bo->pinned_link);
1877 		spin_unlock(&xe->pinned.lock);
1878 	}
1879 	ttm_bo_unpin(&bo->ttm);
1880 }
1881 
1882 /**
1883  * xe_bo_validate() - Make sure the bo is in an allowed placement
1884  * @bo: The bo,
1885  * @vm: Pointer to a the vm the bo shares a locked dma_resv object with, or
1886  *      NULL. Used together with @allow_res_evict.
1887  * @allow_res_evict: Whether it's allowed to evict bos sharing @vm's
1888  *                   reservation object.
1889  *
1890  * Make sure the bo is in allowed placement, migrating it if necessary. If
1891  * needed, other bos will be evicted. If bos selected for eviction shares
1892  * the @vm's reservation object, they can be evicted iff @allow_res_evict is
1893  * set to true, otherwise they will be bypassed.
1894  *
1895  * Return: 0 on success, negative error code on failure. May return
1896  * -EINTR or -ERESTARTSYS if internal waits are interrupted by a signal.
1897  */
1898 int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict)
1899 {
1900 	struct ttm_operation_ctx ctx = {
1901 		.interruptible = true,
1902 		.no_wait_gpu = false,
1903 	};
1904 
1905 	if (vm) {
1906 		lockdep_assert_held(&vm->lock);
1907 		xe_vm_assert_held(vm);
1908 
1909 		ctx.allow_res_evict = allow_res_evict;
1910 		ctx.resv = xe_vm_resv(vm);
1911 	}
1912 
1913 	return ttm_bo_validate(&bo->ttm, &bo->placement, &ctx);
1914 }
1915 
1916 bool xe_bo_is_xe_bo(struct ttm_buffer_object *bo)
1917 {
1918 	if (bo->destroy == &xe_ttm_bo_destroy)
1919 		return true;
1920 
1921 	return false;
1922 }
1923 
1924 /*
1925  * Resolve a BO address. There is no assert to check if the proper lock is held
1926  * so it should only be used in cases where it is not fatal to get the wrong
1927  * address, such as printing debug information, but not in cases where memory is
1928  * written based on this result.
1929  */
1930 dma_addr_t __xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size)
1931 {
1932 	struct xe_device *xe = xe_bo_device(bo);
1933 	struct xe_res_cursor cur;
1934 	u64 page;
1935 
1936 	xe_assert(xe, page_size <= PAGE_SIZE);
1937 	page = offset >> PAGE_SHIFT;
1938 	offset &= (PAGE_SIZE - 1);
1939 
1940 	if (!xe_bo_is_vram(bo) && !xe_bo_is_stolen(bo)) {
1941 		xe_assert(xe, bo->ttm.ttm);
1942 
1943 		xe_res_first_sg(xe_bo_sg(bo), page << PAGE_SHIFT,
1944 				page_size, &cur);
1945 		return xe_res_dma(&cur) + offset;
1946 	} else {
1947 		struct xe_res_cursor cur;
1948 
1949 		xe_res_first(bo->ttm.resource, page << PAGE_SHIFT,
1950 			     page_size, &cur);
1951 		return cur.start + offset + vram_region_gpu_offset(bo->ttm.resource);
1952 	}
1953 }
1954 
1955 dma_addr_t xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size)
1956 {
1957 	if (!READ_ONCE(bo->ttm.pin_count))
1958 		xe_bo_assert_held(bo);
1959 	return __xe_bo_addr(bo, offset, page_size);
1960 }
1961 
1962 int xe_bo_vmap(struct xe_bo *bo)
1963 {
1964 	void *virtual;
1965 	bool is_iomem;
1966 	int ret;
1967 
1968 	xe_bo_assert_held(bo);
1969 
1970 	if (!(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS))
1971 		return -EINVAL;
1972 
1973 	if (!iosys_map_is_null(&bo->vmap))
1974 		return 0;
1975 
1976 	/*
1977 	 * We use this more or less deprecated interface for now since
1978 	 * ttm_bo_vmap() doesn't offer the optimization of kmapping
1979 	 * single page bos, which is done here.
1980 	 * TODO: Fix up ttm_bo_vmap to do that, or fix up ttm_bo_kmap
1981 	 * to use struct iosys_map.
1982 	 */
1983 	ret = ttm_bo_kmap(&bo->ttm, 0, bo->size >> PAGE_SHIFT, &bo->kmap);
1984 	if (ret)
1985 		return ret;
1986 
1987 	virtual = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
1988 	if (is_iomem)
1989 		iosys_map_set_vaddr_iomem(&bo->vmap, (void __iomem *)virtual);
1990 	else
1991 		iosys_map_set_vaddr(&bo->vmap, virtual);
1992 
1993 	return 0;
1994 }
1995 
1996 static void __xe_bo_vunmap(struct xe_bo *bo)
1997 {
1998 	if (!iosys_map_is_null(&bo->vmap)) {
1999 		iosys_map_clear(&bo->vmap);
2000 		ttm_bo_kunmap(&bo->kmap);
2001 	}
2002 }
2003 
2004 void xe_bo_vunmap(struct xe_bo *bo)
2005 {
2006 	xe_bo_assert_held(bo);
2007 	__xe_bo_vunmap(bo);
2008 }
2009 
2010 int xe_gem_create_ioctl(struct drm_device *dev, void *data,
2011 			struct drm_file *file)
2012 {
2013 	struct xe_device *xe = to_xe_device(dev);
2014 	struct xe_file *xef = to_xe_file(file);
2015 	struct drm_xe_gem_create *args = data;
2016 	struct xe_vm *vm = NULL;
2017 	struct xe_bo *bo;
2018 	unsigned int bo_flags;
2019 	u32 handle;
2020 	int err;
2021 
2022 	if (XE_IOCTL_DBG(xe, args->extensions) ||
2023 	    XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) ||
2024 	    XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
2025 		return -EINVAL;
2026 
2027 	/* at least one valid memory placement must be specified */
2028 	if (XE_IOCTL_DBG(xe, (args->placement & ~xe->info.mem_region_mask) ||
2029 			 !args->placement))
2030 		return -EINVAL;
2031 
2032 	if (XE_IOCTL_DBG(xe, args->flags &
2033 			 ~(DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING |
2034 			   DRM_XE_GEM_CREATE_FLAG_SCANOUT |
2035 			   DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM)))
2036 		return -EINVAL;
2037 
2038 	if (XE_IOCTL_DBG(xe, args->handle))
2039 		return -EINVAL;
2040 
2041 	if (XE_IOCTL_DBG(xe, !args->size))
2042 		return -EINVAL;
2043 
2044 	if (XE_IOCTL_DBG(xe, args->size > SIZE_MAX))
2045 		return -EINVAL;
2046 
2047 	if (XE_IOCTL_DBG(xe, args->size & ~PAGE_MASK))
2048 		return -EINVAL;
2049 
2050 	bo_flags = 0;
2051 	if (args->flags & DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING)
2052 		bo_flags |= XE_BO_FLAG_DEFER_BACKING;
2053 
2054 	if (args->flags & DRM_XE_GEM_CREATE_FLAG_SCANOUT)
2055 		bo_flags |= XE_BO_FLAG_SCANOUT;
2056 
2057 	bo_flags |= args->placement << (ffs(XE_BO_FLAG_SYSTEM) - 1);
2058 
2059 	/* CCS formats need physical placement at a 64K alignment in VRAM. */
2060 	if ((bo_flags & XE_BO_FLAG_VRAM_MASK) &&
2061 	    (bo_flags & XE_BO_FLAG_SCANOUT) &&
2062 	    !(xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) &&
2063 	    IS_ALIGNED(args->size, SZ_64K))
2064 		bo_flags |= XE_BO_FLAG_NEEDS_64K;
2065 
2066 	if (args->flags & DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM) {
2067 		if (XE_IOCTL_DBG(xe, !(bo_flags & XE_BO_FLAG_VRAM_MASK)))
2068 			return -EINVAL;
2069 
2070 		bo_flags |= XE_BO_FLAG_NEEDS_CPU_ACCESS;
2071 	}
2072 
2073 	if (XE_IOCTL_DBG(xe, !args->cpu_caching ||
2074 			 args->cpu_caching > DRM_XE_GEM_CPU_CACHING_WC))
2075 		return -EINVAL;
2076 
2077 	if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_FLAG_VRAM_MASK &&
2078 			 args->cpu_caching != DRM_XE_GEM_CPU_CACHING_WC))
2079 		return -EINVAL;
2080 
2081 	if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_FLAG_SCANOUT &&
2082 			 args->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB))
2083 		return -EINVAL;
2084 
2085 	if (args->vm_id) {
2086 		vm = xe_vm_lookup(xef, args->vm_id);
2087 		if (XE_IOCTL_DBG(xe, !vm))
2088 			return -ENOENT;
2089 		err = xe_vm_lock(vm, true);
2090 		if (err)
2091 			goto out_vm;
2092 	}
2093 
2094 	bo = xe_bo_create_user(xe, NULL, vm, args->size, args->cpu_caching,
2095 			       bo_flags);
2096 
2097 	if (vm)
2098 		xe_vm_unlock(vm);
2099 
2100 	if (IS_ERR(bo)) {
2101 		err = PTR_ERR(bo);
2102 		goto out_vm;
2103 	}
2104 
2105 	err = drm_gem_handle_create(file, &bo->ttm.base, &handle);
2106 	if (err)
2107 		goto out_bulk;
2108 
2109 	args->handle = handle;
2110 	goto out_put;
2111 
2112 out_bulk:
2113 	if (vm && !xe_vm_in_fault_mode(vm)) {
2114 		xe_vm_lock(vm, false);
2115 		__xe_bo_unset_bulk_move(bo);
2116 		xe_vm_unlock(vm);
2117 	}
2118 out_put:
2119 	xe_bo_put(bo);
2120 out_vm:
2121 	if (vm)
2122 		xe_vm_put(vm);
2123 
2124 	return err;
2125 }
2126 
2127 int xe_gem_mmap_offset_ioctl(struct drm_device *dev, void *data,
2128 			     struct drm_file *file)
2129 {
2130 	struct xe_device *xe = to_xe_device(dev);
2131 	struct drm_xe_gem_mmap_offset *args = data;
2132 	struct drm_gem_object *gem_obj;
2133 
2134 	if (XE_IOCTL_DBG(xe, args->extensions) ||
2135 	    XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
2136 		return -EINVAL;
2137 
2138 	if (XE_IOCTL_DBG(xe, args->flags))
2139 		return -EINVAL;
2140 
2141 	gem_obj = drm_gem_object_lookup(file, args->handle);
2142 	if (XE_IOCTL_DBG(xe, !gem_obj))
2143 		return -ENOENT;
2144 
2145 	/* The mmap offset was set up at BO allocation time. */
2146 	args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node);
2147 
2148 	xe_bo_put(gem_to_xe_bo(gem_obj));
2149 	return 0;
2150 }
2151 
2152 /**
2153  * xe_bo_lock() - Lock the buffer object's dma_resv object
2154  * @bo: The struct xe_bo whose lock is to be taken
2155  * @intr: Whether to perform any wait interruptible
2156  *
2157  * Locks the buffer object's dma_resv object. If the buffer object is
2158  * pointing to a shared dma_resv object, that shared lock is locked.
2159  *
2160  * Return: 0 on success, -EINTR if @intr is true and the wait for a
2161  * contended lock was interrupted. If @intr is set to false, the
2162  * function always returns 0.
2163  */
2164 int xe_bo_lock(struct xe_bo *bo, bool intr)
2165 {
2166 	if (intr)
2167 		return dma_resv_lock_interruptible(bo->ttm.base.resv, NULL);
2168 
2169 	dma_resv_lock(bo->ttm.base.resv, NULL);
2170 
2171 	return 0;
2172 }
2173 
2174 /**
2175  * xe_bo_unlock() - Unlock the buffer object's dma_resv object
2176  * @bo: The struct xe_bo whose lock is to be released.
2177  *
2178  * Unlock a buffer object lock that was locked by xe_bo_lock().
2179  */
2180 void xe_bo_unlock(struct xe_bo *bo)
2181 {
2182 	dma_resv_unlock(bo->ttm.base.resv);
2183 }
2184 
2185 /**
2186  * xe_bo_can_migrate - Whether a buffer object likely can be migrated
2187  * @bo: The buffer object to migrate
2188  * @mem_type: The TTM memory type intended to migrate to
2189  *
2190  * Check whether the buffer object supports migration to the
2191  * given memory type. Note that pinning may affect the ability to migrate as
2192  * returned by this function.
2193  *
2194  * This function is primarily intended as a helper for checking the
2195  * possibility to migrate buffer objects and can be called without
2196  * the object lock held.
2197  *
2198  * Return: true if migration is possible, false otherwise.
2199  */
2200 bool xe_bo_can_migrate(struct xe_bo *bo, u32 mem_type)
2201 {
2202 	unsigned int cur_place;
2203 
2204 	if (bo->ttm.type == ttm_bo_type_kernel)
2205 		return true;
2206 
2207 	if (bo->ttm.type == ttm_bo_type_sg)
2208 		return false;
2209 
2210 	for (cur_place = 0; cur_place < bo->placement.num_placement;
2211 	     cur_place++) {
2212 		if (bo->placements[cur_place].mem_type == mem_type)
2213 			return true;
2214 	}
2215 
2216 	return false;
2217 }
2218 
2219 static void xe_place_from_ttm_type(u32 mem_type, struct ttm_place *place)
2220 {
2221 	memset(place, 0, sizeof(*place));
2222 	place->mem_type = mem_type;
2223 }
2224 
2225 /**
2226  * xe_bo_migrate - Migrate an object to the desired region id
2227  * @bo: The buffer object to migrate.
2228  * @mem_type: The TTM region type to migrate to.
2229  *
2230  * Attempt to migrate the buffer object to the desired memory region. The
2231  * buffer object may not be pinned, and must be locked.
2232  * On successful completion, the object memory type will be updated,
2233  * but an async migration task may not have completed yet, and to
2234  * accomplish that, the object's kernel fences must be signaled with
2235  * the object lock held.
2236  *
2237  * Return: 0 on success. Negative error code on failure. In particular may
2238  * return -EINTR or -ERESTARTSYS if signal pending.
2239  */
2240 int xe_bo_migrate(struct xe_bo *bo, u32 mem_type)
2241 {
2242 	struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev);
2243 	struct ttm_operation_ctx ctx = {
2244 		.interruptible = true,
2245 		.no_wait_gpu = false,
2246 	};
2247 	struct ttm_placement placement;
2248 	struct ttm_place requested;
2249 
2250 	xe_bo_assert_held(bo);
2251 
2252 	if (bo->ttm.resource->mem_type == mem_type)
2253 		return 0;
2254 
2255 	if (xe_bo_is_pinned(bo))
2256 		return -EBUSY;
2257 
2258 	if (!xe_bo_can_migrate(bo, mem_type))
2259 		return -EINVAL;
2260 
2261 	xe_place_from_ttm_type(mem_type, &requested);
2262 	placement.num_placement = 1;
2263 	placement.placement = &requested;
2264 
2265 	/*
2266 	 * Stolen needs to be handled like below VRAM handling if we ever need
2267 	 * to support it.
2268 	 */
2269 	drm_WARN_ON(&xe->drm, mem_type == XE_PL_STOLEN);
2270 
2271 	if (mem_type_is_vram(mem_type)) {
2272 		u32 c = 0;
2273 
2274 		add_vram(xe, bo, &requested, bo->flags, mem_type, &c);
2275 	}
2276 
2277 	return ttm_bo_validate(&bo->ttm, &placement, &ctx);
2278 }
2279 
2280 /**
2281  * xe_bo_evict - Evict an object to evict placement
2282  * @bo: The buffer object to migrate.
2283  * @force_alloc: Set force_alloc in ttm_operation_ctx
2284  *
2285  * On successful completion, the object memory will be moved to evict
2286  * placement. Ths function blocks until the object has been fully moved.
2287  *
2288  * Return: 0 on success. Negative error code on failure.
2289  */
2290 int xe_bo_evict(struct xe_bo *bo, bool force_alloc)
2291 {
2292 	struct ttm_operation_ctx ctx = {
2293 		.interruptible = false,
2294 		.no_wait_gpu = false,
2295 		.force_alloc = force_alloc,
2296 	};
2297 	struct ttm_placement placement;
2298 	int ret;
2299 
2300 	xe_evict_flags(&bo->ttm, &placement);
2301 	ret = ttm_bo_validate(&bo->ttm, &placement, &ctx);
2302 	if (ret)
2303 		return ret;
2304 
2305 	dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL,
2306 			      false, MAX_SCHEDULE_TIMEOUT);
2307 
2308 	return 0;
2309 }
2310 
2311 /**
2312  * xe_bo_needs_ccs_pages - Whether a bo needs to back up CCS pages when
2313  * placed in system memory.
2314  * @bo: The xe_bo
2315  *
2316  * Return: true if extra pages need to be allocated, false otherwise.
2317  */
2318 bool xe_bo_needs_ccs_pages(struct xe_bo *bo)
2319 {
2320 	struct xe_device *xe = xe_bo_device(bo);
2321 
2322 	if (GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe))
2323 		return false;
2324 
2325 	if (!xe_device_has_flat_ccs(xe) || bo->ttm.type != ttm_bo_type_device)
2326 		return false;
2327 
2328 	/* On discrete GPUs, if the GPU can access this buffer from
2329 	 * system memory (i.e., it allows XE_PL_TT placement), FlatCCS
2330 	 * can't be used since there's no CCS storage associated with
2331 	 * non-VRAM addresses.
2332 	 */
2333 	if (IS_DGFX(xe) && (bo->flags & XE_BO_FLAG_SYSTEM))
2334 		return false;
2335 
2336 	return true;
2337 }
2338 
2339 /**
2340  * __xe_bo_release_dummy() - Dummy kref release function
2341  * @kref: The embedded struct kref.
2342  *
2343  * Dummy release function for xe_bo_put_deferred(). Keep off.
2344  */
2345 void __xe_bo_release_dummy(struct kref *kref)
2346 {
2347 }
2348 
2349 /**
2350  * xe_bo_put_commit() - Put bos whose put was deferred by xe_bo_put_deferred().
2351  * @deferred: The lockless list used for the call to xe_bo_put_deferred().
2352  *
2353  * Puts all bos whose put was deferred by xe_bo_put_deferred().
2354  * The @deferred list can be either an onstack local list or a global
2355  * shared list used by a workqueue.
2356  */
2357 void xe_bo_put_commit(struct llist_head *deferred)
2358 {
2359 	struct llist_node *freed;
2360 	struct xe_bo *bo, *next;
2361 
2362 	if (!deferred)
2363 		return;
2364 
2365 	freed = llist_del_all(deferred);
2366 	if (!freed)
2367 		return;
2368 
2369 	llist_for_each_entry_safe(bo, next, freed, freed)
2370 		drm_gem_object_free(&bo->ttm.base.refcount);
2371 }
2372 
2373 void xe_bo_put(struct xe_bo *bo)
2374 {
2375 	might_sleep();
2376 	if (bo) {
2377 #ifdef CONFIG_PROC_FS
2378 		if (bo->client)
2379 			might_lock(&bo->client->bos_lock);
2380 #endif
2381 		if (bo->ggtt_node && bo->ggtt_node->ggtt)
2382 			might_lock(&bo->ggtt_node->ggtt->lock);
2383 		drm_gem_object_put(&bo->ttm.base);
2384 	}
2385 }
2386 
2387 /**
2388  * xe_bo_dumb_create - Create a dumb bo as backing for a fb
2389  * @file_priv: ...
2390  * @dev: ...
2391  * @args: ...
2392  *
2393  * See dumb_create() hook in include/drm/drm_drv.h
2394  *
2395  * Return: ...
2396  */
2397 int xe_bo_dumb_create(struct drm_file *file_priv,
2398 		      struct drm_device *dev,
2399 		      struct drm_mode_create_dumb *args)
2400 {
2401 	struct xe_device *xe = to_xe_device(dev);
2402 	struct xe_bo *bo;
2403 	uint32_t handle;
2404 	int cpp = DIV_ROUND_UP(args->bpp, 8);
2405 	int err;
2406 	u32 page_size = max_t(u32, PAGE_SIZE,
2407 		xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K);
2408 
2409 	args->pitch = ALIGN(args->width * cpp, 64);
2410 	args->size = ALIGN(mul_u32_u32(args->pitch, args->height),
2411 			   page_size);
2412 
2413 	bo = xe_bo_create_user(xe, NULL, NULL, args->size,
2414 			       DRM_XE_GEM_CPU_CACHING_WC,
2415 			       XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) |
2416 			       XE_BO_FLAG_SCANOUT |
2417 			       XE_BO_FLAG_NEEDS_CPU_ACCESS);
2418 	if (IS_ERR(bo))
2419 		return PTR_ERR(bo);
2420 
2421 	err = drm_gem_handle_create(file_priv, &bo->ttm.base, &handle);
2422 	/* drop reference from allocate - handle holds it now */
2423 	drm_gem_object_put(&bo->ttm.base);
2424 	if (!err)
2425 		args->handle = handle;
2426 	return err;
2427 }
2428 
2429 void xe_bo_runtime_pm_release_mmap_offset(struct xe_bo *bo)
2430 {
2431 	struct ttm_buffer_object *tbo = &bo->ttm;
2432 	struct ttm_device *bdev = tbo->bdev;
2433 
2434 	drm_vma_node_unmap(&tbo->base.vma_node, bdev->dev_mapping);
2435 
2436 	list_del_init(&bo->vram_userfault_link);
2437 }
2438 
2439 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
2440 #include "tests/xe_bo.c"
2441 #endif
2442