xref: /linux/drivers/gpu/drm/xe/xe_migrate.c (revision bf36793fa260cb68cc817f311f1f683788261796)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2020 Intel Corporation
4  */
5 
6 #include "xe_migrate.h"
7 
8 #include <linux/bitfield.h>
9 #include <linux/sizes.h>
10 
11 #include <drm/drm_managed.h>
12 #include <drm/ttm/ttm_tt.h>
13 #include <drm/xe_drm.h>
14 
15 #include <generated/xe_wa_oob.h>
16 
17 #include "instructions/xe_gpu_commands.h"
18 #include "instructions/xe_mi_commands.h"
19 #include "regs/xe_gtt_defs.h"
20 #include "tests/xe_test.h"
21 #include "xe_assert.h"
22 #include "xe_bb.h"
23 #include "xe_bo.h"
24 #include "xe_exec_queue.h"
25 #include "xe_ggtt.h"
26 #include "xe_gt.h"
27 #include "xe_hw_engine.h"
28 #include "xe_lrc.h"
29 #include "xe_map.h"
30 #include "xe_mocs.h"
31 #include "xe_pt.h"
32 #include "xe_res_cursor.h"
33 #include "xe_sched_job.h"
34 #include "xe_sync.h"
35 #include "xe_trace_bo.h"
36 #include "xe_vm.h"
37 
38 /**
39  * struct xe_migrate - migrate context.
40  */
41 struct xe_migrate {
42 	/** @q: Default exec queue used for migration */
43 	struct xe_exec_queue *q;
44 	/** @tile: Backpointer to the tile this struct xe_migrate belongs to. */
45 	struct xe_tile *tile;
46 	/** @job_mutex: Timeline mutex for @eng. */
47 	struct mutex job_mutex;
48 	/** @pt_bo: Page-table buffer object. */
49 	struct xe_bo *pt_bo;
50 	/** @batch_base_ofs: VM offset of the migration batch buffer */
51 	u64 batch_base_ofs;
52 	/** @usm_batch_base_ofs: VM offset of the usm batch buffer */
53 	u64 usm_batch_base_ofs;
54 	/** @cleared_mem_ofs: VM offset of @cleared_bo. */
55 	u64 cleared_mem_ofs;
56 	/**
57 	 * @fence: dma-fence representing the last migration job batch.
58 	 * Protected by @job_mutex.
59 	 */
60 	struct dma_fence *fence;
61 	/**
62 	 * @vm_update_sa: For integrated, used to suballocate page-tables
63 	 * out of the pt_bo.
64 	 */
65 	struct drm_suballoc_manager vm_update_sa;
66 	/** @min_chunk_size: For dgfx, Minimum chunk size */
67 	u64 min_chunk_size;
68 };
69 
70 #define MAX_PREEMPTDISABLE_TRANSFER SZ_8M /* Around 1ms. */
71 #define MAX_CCS_LIMITED_TRANSFER SZ_4M /* XE_PAGE_SIZE * (FIELD_MAX(XE2_CCS_SIZE_MASK) + 1) */
72 #define NUM_KERNEL_PDE 15
73 #define NUM_PT_SLOTS 32
74 #define LEVEL0_PAGE_TABLE_ENCODE_SIZE SZ_2M
75 #define MAX_NUM_PTE 512
76 
77 /*
78  * Although MI_STORE_DATA_IMM's "length" field is 10-bits, 0x3FE is the largest
79  * legal value accepted.  Since that instruction field is always stored in
80  * (val-2) format, this translates to 0x400 dwords for the true maximum length
81  * of the instruction.  Subtracting the instruction header (1 dword) and
82  * address (2 dwords), that leaves 0x3FD dwords (0x1FE qwords) for PTE values.
83  */
84 #define MAX_PTE_PER_SDI 0x1FE
85 
86 /**
87  * xe_tile_migrate_engine() - Get this tile's migrate engine.
88  * @tile: The tile.
89  *
90  * Returns the default migrate engine of this tile.
91  * TODO: Perhaps this function is slightly misplaced, and even unneeded?
92  *
93  * Return: The default migrate engine
94  */
95 struct xe_exec_queue *xe_tile_migrate_engine(struct xe_tile *tile)
96 {
97 	return tile->migrate->q;
98 }
99 
100 static void xe_migrate_fini(struct drm_device *dev, void *arg)
101 {
102 	struct xe_migrate *m = arg;
103 
104 	xe_vm_lock(m->q->vm, false);
105 	xe_bo_unpin(m->pt_bo);
106 	xe_vm_unlock(m->q->vm);
107 
108 	dma_fence_put(m->fence);
109 	xe_bo_put(m->pt_bo);
110 	drm_suballoc_manager_fini(&m->vm_update_sa);
111 	mutex_destroy(&m->job_mutex);
112 	xe_vm_close_and_put(m->q->vm);
113 	xe_exec_queue_put(m->q);
114 }
115 
116 static u64 xe_migrate_vm_addr(u64 slot, u32 level)
117 {
118 	XE_WARN_ON(slot >= NUM_PT_SLOTS);
119 
120 	/* First slot is reserved for mapping of PT bo and bb, start from 1 */
121 	return (slot + 1ULL) << xe_pt_shift(level + 1);
122 }
123 
124 static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr)
125 {
126 	/*
127 	 * Remove the DPA to get a correct offset into identity table for the
128 	 * migrate offset
129 	 */
130 	addr -= xe->mem.vram.dpa_base;
131 	return addr + (256ULL << xe_pt_shift(2));
132 }
133 
134 static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
135 				 struct xe_vm *vm)
136 {
137 	struct xe_device *xe = tile_to_xe(tile);
138 	u16 pat_index = xe->pat.idx[XE_CACHE_WB];
139 	u8 id = tile->id;
140 	u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level,
141 	    num_setup = num_level + 1;
142 	u32 map_ofs, level, i;
143 	struct xe_bo *bo, *batch = tile->mem.kernel_bb_pool->bo;
144 	u64 entry, pt30_ofs;
145 
146 	/* Can't bump NUM_PT_SLOTS too high */
147 	BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/XE_PAGE_SIZE);
148 	/* Must be a multiple of 64K to support all platforms */
149 	BUILD_BUG_ON(NUM_PT_SLOTS * XE_PAGE_SIZE % SZ_64K);
150 	/* And one slot reserved for the 4KiB page table updates */
151 	BUILD_BUG_ON(!(NUM_KERNEL_PDE & 1));
152 
153 	/* Need to be sure everything fits in the first PT, or create more */
154 	xe_tile_assert(tile, m->batch_base_ofs + batch->size < SZ_2M);
155 
156 	bo = xe_bo_create_pin_map(vm->xe, tile, vm,
157 				  num_entries * XE_PAGE_SIZE,
158 				  ttm_bo_type_kernel,
159 				  XE_BO_FLAG_VRAM_IF_DGFX(tile) |
160 				  XE_BO_FLAG_PINNED);
161 	if (IS_ERR(bo))
162 		return PTR_ERR(bo);
163 
164 	/* PT31 reserved for 2M identity map */
165 	pt30_ofs = bo->size - 2 * XE_PAGE_SIZE;
166 	entry = vm->pt_ops->pde_encode_bo(bo, pt30_ofs, pat_index);
167 	xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry);
168 
169 	map_ofs = (num_entries - num_setup) * XE_PAGE_SIZE;
170 
171 	/* Map the entire BO in our level 0 pt */
172 	for (i = 0, level = 0; i < num_entries; level++) {
173 		entry = vm->pt_ops->pte_encode_bo(bo, i * XE_PAGE_SIZE,
174 						  pat_index, 0);
175 
176 		xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, entry);
177 
178 		if (vm->flags & XE_VM_FLAG_64K)
179 			i += 16;
180 		else
181 			i += 1;
182 	}
183 
184 	if (!IS_DGFX(xe)) {
185 		/* Write out batch too */
186 		m->batch_base_ofs = NUM_PT_SLOTS * XE_PAGE_SIZE;
187 		for (i = 0; i < batch->size;
188 		     i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE :
189 		     XE_PAGE_SIZE) {
190 			entry = vm->pt_ops->pte_encode_bo(batch, i,
191 							  pat_index, 0);
192 
193 			xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64,
194 				  entry);
195 			level++;
196 		}
197 		if (xe->info.has_usm) {
198 			xe_tile_assert(tile, batch->size == SZ_1M);
199 
200 			batch = tile->primary_gt->usm.bb_pool->bo;
201 			m->usm_batch_base_ofs = m->batch_base_ofs + SZ_1M;
202 			xe_tile_assert(tile, batch->size == SZ_512K);
203 
204 			for (i = 0; i < batch->size;
205 			     i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE :
206 			     XE_PAGE_SIZE) {
207 				entry = vm->pt_ops->pte_encode_bo(batch, i,
208 								  pat_index, 0);
209 
210 				xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64,
211 					  entry);
212 				level++;
213 			}
214 		}
215 	} else {
216 		u64 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE);
217 
218 		m->batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr);
219 
220 		if (xe->info.has_usm) {
221 			batch = tile->primary_gt->usm.bb_pool->bo;
222 			batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE);
223 			m->usm_batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr);
224 		}
225 	}
226 
227 	for (level = 1; level < num_level; level++) {
228 		u32 flags = 0;
229 
230 		if (vm->flags & XE_VM_FLAG_64K && level == 1)
231 			flags = XE_PDE_64K;
232 
233 		entry = vm->pt_ops->pde_encode_bo(bo, map_ofs + (u64)(level - 1) *
234 						  XE_PAGE_SIZE, pat_index);
235 		xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level, u64,
236 			  entry | flags);
237 	}
238 
239 	/* Write PDE's that point to our BO. */
240 	for (i = 0; i < map_ofs / PAGE_SIZE; i++) {
241 		entry = vm->pt_ops->pde_encode_bo(bo, (u64)i * XE_PAGE_SIZE,
242 						  pat_index);
243 
244 		xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE +
245 			  (i + 1) * 8, u64, entry);
246 	}
247 
248 	/* Set up a 1GiB NULL mapping at 255GiB offset. */
249 	level = 2;
250 	xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level + 255 * 8, u64,
251 		  vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0)
252 		  | XE_PTE_NULL);
253 	m->cleared_mem_ofs = (255ULL << xe_pt_shift(level));
254 
255 	/* Identity map the entire vram at 256GiB offset */
256 	if (IS_DGFX(xe)) {
257 		u64 pos, ofs, flags;
258 		/* XXX: Unclear if this should be usable_size? */
259 		u64 vram_limit =  xe->mem.vram.actual_physical_size +
260 			xe->mem.vram.dpa_base;
261 
262 		level = 2;
263 		ofs = map_ofs + XE_PAGE_SIZE * level + 256 * 8;
264 		flags = vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level,
265 						    true, 0);
266 
267 		xe_assert(xe, IS_ALIGNED(xe->mem.vram.usable_size, SZ_2M));
268 
269 		/*
270 		 * Use 1GB pages when possible, last chunk always use 2M
271 		 * pages as mixing reserved memory (stolen, WOCPM) with a single
272 		 * mapping is not allowed on certain platforms.
273 		 */
274 		for (pos = xe->mem.vram.dpa_base; pos < vram_limit;
275 		     pos += SZ_1G, ofs += 8) {
276 			if (pos + SZ_1G >= vram_limit) {
277 				u64 pt31_ofs = bo->size - XE_PAGE_SIZE;
278 
279 				entry = vm->pt_ops->pde_encode_bo(bo, pt31_ofs,
280 								  pat_index);
281 				xe_map_wr(xe, &bo->vmap, ofs, u64, entry);
282 
283 				flags = vm->pt_ops->pte_encode_addr(xe, 0,
284 								    pat_index,
285 								    level - 1,
286 								    true, 0);
287 
288 				for (ofs = pt31_ofs; pos < vram_limit;
289 				     pos += SZ_2M, ofs += 8)
290 					xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags);
291 				break;	/* Ensure pos == vram_limit assert correct */
292 			}
293 
294 			xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags);
295 		}
296 
297 		xe_assert(xe, pos == vram_limit);
298 	}
299 
300 	/*
301 	 * Example layout created above, with root level = 3:
302 	 * [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's
303 	 * [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's
304 	 * [PT9...PT27]: Userspace PT's for VM_BIND, 4 KiB PTE's
305 	 * [PT28 = PDE 0] [PT29 = PDE 1] [PT30 = PDE 2] [PT31 = 2M vram identity map]
306 	 *
307 	 * This makes the lowest part of the VM point to the pagetables.
308 	 * Hence the lowest 2M in the vm should point to itself, with a few writes
309 	 * and flushes, other parts of the VM can be used either for copying and
310 	 * clearing.
311 	 *
312 	 * For performance, the kernel reserves PDE's, so about 20 are left
313 	 * for async VM updates.
314 	 *
315 	 * To make it easier to work, each scratch PT is put in slot (1 + PT #)
316 	 * everywhere, this allows lockless updates to scratch pages by using
317 	 * the different addresses in VM.
318 	 */
319 #define NUM_VMUSA_UNIT_PER_PAGE	32
320 #define VM_SA_UPDATE_UNIT_SIZE		(XE_PAGE_SIZE / NUM_VMUSA_UNIT_PER_PAGE)
321 #define NUM_VMUSA_WRITES_PER_UNIT	(VM_SA_UPDATE_UNIT_SIZE / sizeof(u64))
322 	drm_suballoc_manager_init(&m->vm_update_sa,
323 				  (size_t)(map_ofs / XE_PAGE_SIZE - NUM_KERNEL_PDE) *
324 				  NUM_VMUSA_UNIT_PER_PAGE, 0);
325 
326 	m->pt_bo = bo;
327 	return 0;
328 }
329 
330 /*
331  * Including the reserved copy engine is required to avoid deadlocks due to
332  * migrate jobs servicing the faults gets stuck behind the job that faulted.
333  */
334 static u32 xe_migrate_usm_logical_mask(struct xe_gt *gt)
335 {
336 	u32 logical_mask = 0;
337 	struct xe_hw_engine *hwe;
338 	enum xe_hw_engine_id id;
339 
340 	for_each_hw_engine(hwe, gt, id) {
341 		if (hwe->class != XE_ENGINE_CLASS_COPY)
342 			continue;
343 
344 		if (xe_gt_is_usm_hwe(gt, hwe))
345 			logical_mask |= BIT(hwe->logical_instance);
346 	}
347 
348 	return logical_mask;
349 }
350 
351 /**
352  * xe_migrate_init() - Initialize a migrate context
353  * @tile: Back-pointer to the tile we're initializing for.
354  *
355  * Return: Pointer to a migrate context on success. Error pointer on error.
356  */
357 struct xe_migrate *xe_migrate_init(struct xe_tile *tile)
358 {
359 	struct xe_device *xe = tile_to_xe(tile);
360 	struct xe_gt *primary_gt = tile->primary_gt;
361 	struct xe_migrate *m;
362 	struct xe_vm *vm;
363 	int err;
364 
365 	m = drmm_kzalloc(&xe->drm, sizeof(*m), GFP_KERNEL);
366 	if (!m)
367 		return ERR_PTR(-ENOMEM);
368 
369 	m->tile = tile;
370 
371 	/* Special layout, prepared below.. */
372 	vm = xe_vm_create(xe, XE_VM_FLAG_MIGRATION |
373 			  XE_VM_FLAG_SET_TILE_ID(tile));
374 	if (IS_ERR(vm))
375 		return ERR_CAST(vm);
376 
377 	xe_vm_lock(vm, false);
378 	err = xe_migrate_prepare_vm(tile, m, vm);
379 	xe_vm_unlock(vm);
380 	if (err) {
381 		xe_vm_close_and_put(vm);
382 		return ERR_PTR(err);
383 	}
384 
385 	if (xe->info.has_usm) {
386 		struct xe_hw_engine *hwe = xe_gt_hw_engine(primary_gt,
387 							   XE_ENGINE_CLASS_COPY,
388 							   primary_gt->usm.reserved_bcs_instance,
389 							   false);
390 		u32 logical_mask = xe_migrate_usm_logical_mask(primary_gt);
391 
392 		if (!hwe || !logical_mask)
393 			return ERR_PTR(-EINVAL);
394 
395 		/*
396 		 * XXX: Currently only reserving 1 (likely slow) BCS instance on
397 		 * PVC, may want to revisit if performance is needed.
398 		 */
399 		m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe,
400 					    EXEC_QUEUE_FLAG_KERNEL |
401 					    EXEC_QUEUE_FLAG_PERMANENT |
402 					    EXEC_QUEUE_FLAG_HIGH_PRIORITY, 0);
403 	} else {
404 		m->q = xe_exec_queue_create_class(xe, primary_gt, vm,
405 						  XE_ENGINE_CLASS_COPY,
406 						  EXEC_QUEUE_FLAG_KERNEL |
407 						  EXEC_QUEUE_FLAG_PERMANENT);
408 	}
409 	if (IS_ERR(m->q)) {
410 		xe_vm_close_and_put(vm);
411 		return ERR_CAST(m->q);
412 	}
413 
414 	mutex_init(&m->job_mutex);
415 	fs_reclaim_acquire(GFP_KERNEL);
416 	might_lock(&m->job_mutex);
417 	fs_reclaim_release(GFP_KERNEL);
418 
419 	err = drmm_add_action_or_reset(&xe->drm, xe_migrate_fini, m);
420 	if (err)
421 		return ERR_PTR(err);
422 
423 	if (IS_DGFX(xe)) {
424 		if (xe_device_has_flat_ccs(xe))
425 			/* min chunk size corresponds to 4K of CCS Metadata */
426 			m->min_chunk_size = SZ_4K * SZ_64K /
427 				xe_device_ccs_bytes(xe, SZ_64K);
428 		else
429 			/* Somewhat arbitrary to avoid a huge amount of blits */
430 			m->min_chunk_size = SZ_64K;
431 		m->min_chunk_size = roundup_pow_of_two(m->min_chunk_size);
432 		drm_dbg(&xe->drm, "Migrate min chunk size is 0x%08llx\n",
433 			(unsigned long long)m->min_chunk_size);
434 	}
435 
436 	return m;
437 }
438 
439 static u64 max_mem_transfer_per_pass(struct xe_device *xe)
440 {
441 	if (!IS_DGFX(xe) && xe_device_has_flat_ccs(xe))
442 		return MAX_CCS_LIMITED_TRANSFER;
443 
444 	return MAX_PREEMPTDISABLE_TRANSFER;
445 }
446 
447 static u64 xe_migrate_res_sizes(struct xe_migrate *m, struct xe_res_cursor *cur)
448 {
449 	struct xe_device *xe = tile_to_xe(m->tile);
450 	u64 size = min_t(u64, max_mem_transfer_per_pass(xe), cur->remaining);
451 
452 	if (mem_type_is_vram(cur->mem_type)) {
453 		/*
454 		 * VRAM we want to blit in chunks with sizes aligned to
455 		 * min_chunk_size in order for the offset to CCS metadata to be
456 		 * page-aligned. If it's the last chunk it may be smaller.
457 		 *
458 		 * Another constraint is that we need to limit the blit to
459 		 * the VRAM block size, unless size is smaller than
460 		 * min_chunk_size.
461 		 */
462 		u64 chunk = max_t(u64, cur->size, m->min_chunk_size);
463 
464 		size = min_t(u64, size, chunk);
465 		if (size > m->min_chunk_size)
466 			size = round_down(size, m->min_chunk_size);
467 	}
468 
469 	return size;
470 }
471 
472 static bool xe_migrate_allow_identity(u64 size, const struct xe_res_cursor *cur)
473 {
474 	/* If the chunk is not fragmented, allow identity map. */
475 	return cur->size >= size;
476 }
477 
478 static u32 pte_update_size(struct xe_migrate *m,
479 			   bool is_vram,
480 			   struct ttm_resource *res,
481 			   struct xe_res_cursor *cur,
482 			   u64 *L0, u64 *L0_ofs, u32 *L0_pt,
483 			   u32 cmd_size, u32 pt_ofs, u32 avail_pts)
484 {
485 	u32 cmds = 0;
486 
487 	*L0_pt = pt_ofs;
488 	if (is_vram && xe_migrate_allow_identity(*L0, cur)) {
489 		/* Offset into identity map. */
490 		*L0_ofs = xe_migrate_vram_ofs(tile_to_xe(m->tile),
491 					      cur->start + vram_region_gpu_offset(res));
492 		cmds += cmd_size;
493 	} else {
494 		/* Clip L0 to available size */
495 		u64 size = min(*L0, (u64)avail_pts * SZ_2M);
496 		u32 num_4k_pages = (size + XE_PAGE_SIZE - 1) >> XE_PTE_SHIFT;
497 
498 		*L0 = size;
499 		*L0_ofs = xe_migrate_vm_addr(pt_ofs, 0);
500 
501 		/* MI_STORE_DATA_IMM */
502 		cmds += 3 * DIV_ROUND_UP(num_4k_pages, MAX_PTE_PER_SDI);
503 
504 		/* PDE qwords */
505 		cmds += num_4k_pages * 2;
506 
507 		/* Each chunk has a single blit command */
508 		cmds += cmd_size;
509 	}
510 
511 	return cmds;
512 }
513 
514 static void emit_pte(struct xe_migrate *m,
515 		     struct xe_bb *bb, u32 at_pt,
516 		     bool is_vram, bool is_comp_pte,
517 		     struct xe_res_cursor *cur,
518 		     u32 size, struct ttm_resource *res)
519 {
520 	struct xe_device *xe = tile_to_xe(m->tile);
521 	struct xe_vm *vm = m->q->vm;
522 	u16 pat_index;
523 	u32 ptes;
524 	u64 ofs = (u64)at_pt * XE_PAGE_SIZE;
525 	u64 cur_ofs;
526 
527 	/* Indirect access needs compression enabled uncached PAT index */
528 	if (GRAPHICS_VERx100(xe) >= 2000)
529 		pat_index = is_comp_pte ? xe->pat.idx[XE_CACHE_NONE_COMPRESSION] :
530 					  xe->pat.idx[XE_CACHE_WB];
531 	else
532 		pat_index = xe->pat.idx[XE_CACHE_WB];
533 
534 	ptes = DIV_ROUND_UP(size, XE_PAGE_SIZE);
535 
536 	while (ptes) {
537 		u32 chunk = min(MAX_PTE_PER_SDI, ptes);
538 
539 		bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk);
540 		bb->cs[bb->len++] = ofs;
541 		bb->cs[bb->len++] = 0;
542 
543 		cur_ofs = ofs;
544 		ofs += chunk * 8;
545 		ptes -= chunk;
546 
547 		while (chunk--) {
548 			u64 addr, flags = 0;
549 			bool devmem = false;
550 
551 			addr = xe_res_dma(cur) & PAGE_MASK;
552 			if (is_vram) {
553 				if (vm->flags & XE_VM_FLAG_64K) {
554 					u64 va = cur_ofs * XE_PAGE_SIZE / 8;
555 
556 					xe_assert(xe, (va & (SZ_64K - 1)) ==
557 						  (addr & (SZ_64K - 1)));
558 
559 					flags |= XE_PTE_PS64;
560 				}
561 
562 				addr += vram_region_gpu_offset(res);
563 				devmem = true;
564 			}
565 
566 			addr = vm->pt_ops->pte_encode_addr(m->tile->xe,
567 							   addr, pat_index,
568 							   0, devmem, flags);
569 			bb->cs[bb->len++] = lower_32_bits(addr);
570 			bb->cs[bb->len++] = upper_32_bits(addr);
571 
572 			xe_res_next(cur, min_t(u32, size, PAGE_SIZE));
573 			cur_ofs += 8;
574 		}
575 	}
576 }
577 
578 #define EMIT_COPY_CCS_DW 5
579 static void emit_copy_ccs(struct xe_gt *gt, struct xe_bb *bb,
580 			  u64 dst_ofs, bool dst_is_indirect,
581 			  u64 src_ofs, bool src_is_indirect,
582 			  u32 size)
583 {
584 	struct xe_device *xe = gt_to_xe(gt);
585 	u32 *cs = bb->cs + bb->len;
586 	u32 num_ccs_blks;
587 	u32 num_pages;
588 	u32 ccs_copy_size;
589 	u32 mocs;
590 
591 	if (GRAPHICS_VERx100(xe) >= 2000) {
592 		num_pages = DIV_ROUND_UP(size, XE_PAGE_SIZE);
593 		xe_gt_assert(gt, FIELD_FIT(XE2_CCS_SIZE_MASK, num_pages - 1));
594 
595 		ccs_copy_size = REG_FIELD_PREP(XE2_CCS_SIZE_MASK, num_pages - 1);
596 		mocs = FIELD_PREP(XE2_XY_CTRL_SURF_MOCS_INDEX_MASK, gt->mocs.uc_index);
597 
598 	} else {
599 		num_ccs_blks = DIV_ROUND_UP(xe_device_ccs_bytes(gt_to_xe(gt), size),
600 					    NUM_CCS_BYTES_PER_BLOCK);
601 		xe_gt_assert(gt, FIELD_FIT(CCS_SIZE_MASK, num_ccs_blks - 1));
602 
603 		ccs_copy_size = REG_FIELD_PREP(CCS_SIZE_MASK, num_ccs_blks - 1);
604 		mocs = FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, gt->mocs.uc_index);
605 	}
606 
607 	*cs++ = XY_CTRL_SURF_COPY_BLT |
608 		(src_is_indirect ? 0x0 : 0x1) << SRC_ACCESS_TYPE_SHIFT |
609 		(dst_is_indirect ? 0x0 : 0x1) << DST_ACCESS_TYPE_SHIFT |
610 		ccs_copy_size;
611 	*cs++ = lower_32_bits(src_ofs);
612 	*cs++ = upper_32_bits(src_ofs) | mocs;
613 	*cs++ = lower_32_bits(dst_ofs);
614 	*cs++ = upper_32_bits(dst_ofs) | mocs;
615 
616 	bb->len = cs - bb->cs;
617 }
618 
619 #define EMIT_COPY_DW 10
620 static void emit_copy(struct xe_gt *gt, struct xe_bb *bb,
621 		      u64 src_ofs, u64 dst_ofs, unsigned int size,
622 		      unsigned int pitch)
623 {
624 	struct xe_device *xe = gt_to_xe(gt);
625 	u32 mocs = 0;
626 	u32 tile_y = 0;
627 
628 	xe_gt_assert(gt, size / pitch <= S16_MAX);
629 	xe_gt_assert(gt, pitch / 4 <= S16_MAX);
630 	xe_gt_assert(gt, pitch <= U16_MAX);
631 
632 	if (GRAPHICS_VER(xe) >= 20)
633 		mocs = FIELD_PREP(XE2_XY_FAST_COPY_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index);
634 
635 	if (GRAPHICS_VERx100(xe) >= 1250)
636 		tile_y = XY_FAST_COPY_BLT_D1_SRC_TILE4 | XY_FAST_COPY_BLT_D1_DST_TILE4;
637 
638 	bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (10 - 2);
639 	bb->cs[bb->len++] = XY_FAST_COPY_BLT_DEPTH_32 | pitch | tile_y | mocs;
640 	bb->cs[bb->len++] = 0;
641 	bb->cs[bb->len++] = (size / pitch) << 16 | pitch / 4;
642 	bb->cs[bb->len++] = lower_32_bits(dst_ofs);
643 	bb->cs[bb->len++] = upper_32_bits(dst_ofs);
644 	bb->cs[bb->len++] = 0;
645 	bb->cs[bb->len++] = pitch | mocs;
646 	bb->cs[bb->len++] = lower_32_bits(src_ofs);
647 	bb->cs[bb->len++] = upper_32_bits(src_ofs);
648 }
649 
650 static u64 xe_migrate_batch_base(struct xe_migrate *m, bool usm)
651 {
652 	return usm ? m->usm_batch_base_ofs : m->batch_base_ofs;
653 }
654 
655 static u32 xe_migrate_ccs_copy(struct xe_migrate *m,
656 			       struct xe_bb *bb,
657 			       u64 src_ofs, bool src_is_indirect,
658 			       u64 dst_ofs, bool dst_is_indirect, u32 dst_size,
659 			       u64 ccs_ofs, bool copy_ccs)
660 {
661 	struct xe_gt *gt = m->tile->primary_gt;
662 	u32 flush_flags = 0;
663 
664 	if (xe_device_has_flat_ccs(gt_to_xe(gt)) && !copy_ccs && dst_is_indirect) {
665 		/*
666 		 * If the src is already in vram, then it should already
667 		 * have been cleared by us, or has been populated by the
668 		 * user. Make sure we copy the CCS aux state as-is.
669 		 *
670 		 * Otherwise if the bo doesn't have any CCS metadata attached,
671 		 * we still need to clear it for security reasons.
672 		 */
673 		u64 ccs_src_ofs =  src_is_indirect ? src_ofs : m->cleared_mem_ofs;
674 
675 		emit_copy_ccs(gt, bb,
676 			      dst_ofs, true,
677 			      ccs_src_ofs, src_is_indirect, dst_size);
678 
679 		flush_flags = MI_FLUSH_DW_CCS;
680 	} else if (copy_ccs) {
681 		if (!src_is_indirect)
682 			src_ofs = ccs_ofs;
683 		else if (!dst_is_indirect)
684 			dst_ofs = ccs_ofs;
685 
686 		xe_gt_assert(gt, src_is_indirect || dst_is_indirect);
687 
688 		emit_copy_ccs(gt, bb, dst_ofs, dst_is_indirect, src_ofs,
689 			      src_is_indirect, dst_size);
690 		if (dst_is_indirect)
691 			flush_flags = MI_FLUSH_DW_CCS;
692 	}
693 
694 	return flush_flags;
695 }
696 
697 /**
698  * xe_migrate_copy() - Copy content of TTM resources.
699  * @m: The migration context.
700  * @src_bo: The buffer object @src is currently bound to.
701  * @dst_bo: If copying between resources created for the same bo, set this to
702  * the same value as @src_bo. If copying between buffer objects, set it to
703  * the buffer object @dst is currently bound to.
704  * @src: The source TTM resource.
705  * @dst: The dst TTM resource.
706  * @copy_only_ccs: If true copy only CCS metadata
707  *
708  * Copies the contents of @src to @dst: On flat CCS devices,
709  * the CCS metadata is copied as well if needed, or if not present,
710  * the CCS metadata of @dst is cleared for security reasons.
711  *
712  * Return: Pointer to a dma_fence representing the last copy batch, or
713  * an error pointer on failure. If there is a failure, any copy operation
714  * started by the function call has been synced.
715  */
716 struct dma_fence *xe_migrate_copy(struct xe_migrate *m,
717 				  struct xe_bo *src_bo,
718 				  struct xe_bo *dst_bo,
719 				  struct ttm_resource *src,
720 				  struct ttm_resource *dst,
721 				  bool copy_only_ccs)
722 {
723 	struct xe_gt *gt = m->tile->primary_gt;
724 	struct xe_device *xe = gt_to_xe(gt);
725 	struct dma_fence *fence = NULL;
726 	u64 size = src_bo->size;
727 	struct xe_res_cursor src_it, dst_it, ccs_it;
728 	u64 src_L0_ofs, dst_L0_ofs;
729 	u32 src_L0_pt, dst_L0_pt;
730 	u64 src_L0, dst_L0;
731 	int pass = 0;
732 	int err;
733 	bool src_is_pltt = src->mem_type == XE_PL_TT;
734 	bool dst_is_pltt = dst->mem_type == XE_PL_TT;
735 	bool src_is_vram = mem_type_is_vram(src->mem_type);
736 	bool dst_is_vram = mem_type_is_vram(dst->mem_type);
737 	bool copy_ccs = xe_device_has_flat_ccs(xe) &&
738 		xe_bo_needs_ccs_pages(src_bo) && xe_bo_needs_ccs_pages(dst_bo);
739 	bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram);
740 
741 	/* Copying CCS between two different BOs is not supported yet. */
742 	if (XE_WARN_ON(copy_ccs && src_bo != dst_bo))
743 		return ERR_PTR(-EINVAL);
744 
745 	if (src_bo != dst_bo && XE_WARN_ON(src_bo->size != dst_bo->size))
746 		return ERR_PTR(-EINVAL);
747 
748 	if (!src_is_vram)
749 		xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it);
750 	else
751 		xe_res_first(src, 0, size, &src_it);
752 	if (!dst_is_vram)
753 		xe_res_first_sg(xe_bo_sg(dst_bo), 0, size, &dst_it);
754 	else
755 		xe_res_first(dst, 0, size, &dst_it);
756 
757 	if (copy_system_ccs)
758 		xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo),
759 				PAGE_ALIGN(xe_device_ccs_bytes(xe, size)),
760 				&ccs_it);
761 
762 	while (size) {
763 		u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */
764 		struct xe_sched_job *job;
765 		struct xe_bb *bb;
766 		u32 flush_flags;
767 		u32 update_idx;
768 		u64 ccs_ofs, ccs_size;
769 		u32 ccs_pt;
770 
771 		bool usm = xe->info.has_usm;
772 		u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;
773 
774 		src_L0 = xe_migrate_res_sizes(m, &src_it);
775 		dst_L0 = xe_migrate_res_sizes(m, &dst_it);
776 
777 		drm_dbg(&xe->drm, "Pass %u, sizes: %llu & %llu\n",
778 			pass++, src_L0, dst_L0);
779 
780 		src_L0 = min(src_L0, dst_L0);
781 
782 		batch_size += pte_update_size(m, src_is_vram, src, &src_it, &src_L0,
783 					      &src_L0_ofs, &src_L0_pt, 0, 0,
784 					      avail_pts);
785 
786 		batch_size += pte_update_size(m, dst_is_vram, dst, &dst_it, &src_L0,
787 					      &dst_L0_ofs, &dst_L0_pt, 0,
788 					      avail_pts, avail_pts);
789 
790 		if (copy_system_ccs) {
791 			ccs_size = xe_device_ccs_bytes(xe, src_L0);
792 			batch_size += pte_update_size(m, false, NULL, &ccs_it, &ccs_size,
793 						      &ccs_ofs, &ccs_pt, 0,
794 						      2 * avail_pts,
795 						      avail_pts);
796 			xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE));
797 		}
798 
799 		/* Add copy commands size here */
800 		batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) +
801 			((xe_device_has_flat_ccs(xe) ? EMIT_COPY_CCS_DW : 0));
802 
803 		bb = xe_bb_new(gt, batch_size, usm);
804 		if (IS_ERR(bb)) {
805 			err = PTR_ERR(bb);
806 			goto err_sync;
807 		}
808 
809 		if (src_is_vram && xe_migrate_allow_identity(src_L0, &src_it))
810 			xe_res_next(&src_it, src_L0);
811 		else
812 			emit_pte(m, bb, src_L0_pt, src_is_vram, copy_system_ccs,
813 				 &src_it, src_L0, src);
814 
815 		if (dst_is_vram && xe_migrate_allow_identity(src_L0, &dst_it))
816 			xe_res_next(&dst_it, src_L0);
817 		else
818 			emit_pte(m, bb, dst_L0_pt, dst_is_vram, copy_system_ccs,
819 				 &dst_it, src_L0, dst);
820 
821 		if (copy_system_ccs)
822 			emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src);
823 
824 		bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
825 		update_idx = bb->len;
826 
827 		if (!copy_only_ccs)
828 			emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE);
829 
830 		flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs,
831 						  IS_DGFX(xe) ? src_is_vram : src_is_pltt,
832 						  dst_L0_ofs,
833 						  IS_DGFX(xe) ? dst_is_vram : dst_is_pltt,
834 						  src_L0, ccs_ofs, copy_ccs);
835 
836 		job = xe_bb_create_migration_job(m->q, bb,
837 						 xe_migrate_batch_base(m, usm),
838 						 update_idx);
839 		if (IS_ERR(job)) {
840 			err = PTR_ERR(job);
841 			goto err;
842 		}
843 
844 		xe_sched_job_add_migrate_flush(job, flush_flags);
845 		if (!fence) {
846 			err = xe_sched_job_add_deps(job, src_bo->ttm.base.resv,
847 						    DMA_RESV_USAGE_BOOKKEEP);
848 			if (!err && src_bo != dst_bo)
849 				err = xe_sched_job_add_deps(job, dst_bo->ttm.base.resv,
850 							    DMA_RESV_USAGE_BOOKKEEP);
851 			if (err)
852 				goto err_job;
853 		}
854 
855 		mutex_lock(&m->job_mutex);
856 		xe_sched_job_arm(job);
857 		dma_fence_put(fence);
858 		fence = dma_fence_get(&job->drm.s_fence->finished);
859 		xe_sched_job_push(job);
860 
861 		dma_fence_put(m->fence);
862 		m->fence = dma_fence_get(fence);
863 
864 		mutex_unlock(&m->job_mutex);
865 
866 		xe_bb_free(bb, fence);
867 		size -= src_L0;
868 		continue;
869 
870 err_job:
871 		xe_sched_job_put(job);
872 err:
873 		xe_bb_free(bb, NULL);
874 
875 err_sync:
876 		/* Sync partial copy if any. FIXME: under job_mutex? */
877 		if (fence) {
878 			dma_fence_wait(fence, false);
879 			dma_fence_put(fence);
880 		}
881 
882 		return ERR_PTR(err);
883 	}
884 
885 	return fence;
886 }
887 
888 static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,
889 				 u32 size, u32 pitch)
890 {
891 	struct xe_device *xe = gt_to_xe(gt);
892 	u32 *cs = bb->cs + bb->len;
893 	u32 len = PVC_MEM_SET_CMD_LEN_DW;
894 
895 	*cs++ = PVC_MEM_SET_CMD | PVC_MEM_SET_MATRIX | (len - 2);
896 	*cs++ = pitch - 1;
897 	*cs++ = (size / pitch) - 1;
898 	*cs++ = pitch - 1;
899 	*cs++ = lower_32_bits(src_ofs);
900 	*cs++ = upper_32_bits(src_ofs);
901 	if (GRAPHICS_VERx100(xe) >= 2000)
902 		*cs++ = FIELD_PREP(XE2_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index);
903 	else
904 		*cs++ = FIELD_PREP(PVC_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index);
905 
906 	xe_gt_assert(gt, cs - bb->cs == len + bb->len);
907 
908 	bb->len += len;
909 }
910 
911 static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb,
912 				 u64 src_ofs, u32 size, u32 pitch, bool is_vram)
913 {
914 	struct xe_device *xe = gt_to_xe(gt);
915 	u32 *cs = bb->cs + bb->len;
916 	u32 len = XY_FAST_COLOR_BLT_DW;
917 
918 	if (GRAPHICS_VERx100(xe) < 1250)
919 		len = 11;
920 
921 	*cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 |
922 		(len - 2);
923 	if (GRAPHICS_VERx100(xe) >= 2000)
924 		*cs++ = FIELD_PREP(XE2_XY_FAST_COLOR_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index) |
925 			(pitch - 1);
926 	else
927 		*cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, gt->mocs.uc_index) |
928 			(pitch - 1);
929 	*cs++ = 0;
930 	*cs++ = (size / pitch) << 16 | pitch / 4;
931 	*cs++ = lower_32_bits(src_ofs);
932 	*cs++ = upper_32_bits(src_ofs);
933 	*cs++ = (is_vram ? 0x0 : 0x1) <<  XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT;
934 	*cs++ = 0;
935 	*cs++ = 0;
936 	*cs++ = 0;
937 	*cs++ = 0;
938 
939 	if (len > 11) {
940 		*cs++ = 0;
941 		*cs++ = 0;
942 		*cs++ = 0;
943 		*cs++ = 0;
944 		*cs++ = 0;
945 	}
946 
947 	xe_gt_assert(gt, cs - bb->cs == len + bb->len);
948 
949 	bb->len += len;
950 }
951 
952 static bool has_service_copy_support(struct xe_gt *gt)
953 {
954 	/*
955 	 * What we care about is whether the architecture was designed with
956 	 * service copy functionality (specifically the new MEM_SET / MEM_COPY
957 	 * instructions) so check the architectural engine list rather than the
958 	 * actual list since these instructions are usable on BCS0 even if
959 	 * all of the actual service copy engines (BCS1-BCS8) have been fused
960 	 * off.
961 	 */
962 	return gt->info.engine_mask & GENMASK(XE_HW_ENGINE_BCS8,
963 					      XE_HW_ENGINE_BCS1);
964 }
965 
966 static u32 emit_clear_cmd_len(struct xe_gt *gt)
967 {
968 	if (has_service_copy_support(gt))
969 		return PVC_MEM_SET_CMD_LEN_DW;
970 	else
971 		return XY_FAST_COLOR_BLT_DW;
972 }
973 
974 static void emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,
975 		       u32 size, u32 pitch, bool is_vram)
976 {
977 	if (has_service_copy_support(gt))
978 		emit_clear_link_copy(gt, bb, src_ofs, size, pitch);
979 	else
980 		emit_clear_main_copy(gt, bb, src_ofs, size, pitch,
981 				     is_vram);
982 }
983 
984 /**
985  * xe_migrate_clear() - Copy content of TTM resources.
986  * @m: The migration context.
987  * @bo: The buffer object @dst is currently bound to.
988  * @dst: The dst TTM resource to be cleared.
989  *
990  * Clear the contents of @dst to zero. On flat CCS devices,
991  * the CCS metadata is cleared to zero as well on VRAM destinations.
992  * TODO: Eliminate the @bo argument.
993  *
994  * Return: Pointer to a dma_fence representing the last clear batch, or
995  * an error pointer on failure. If there is a failure, any clear operation
996  * started by the function call has been synced.
997  */
998 struct dma_fence *xe_migrate_clear(struct xe_migrate *m,
999 				   struct xe_bo *bo,
1000 				   struct ttm_resource *dst)
1001 {
1002 	bool clear_vram = mem_type_is_vram(dst->mem_type);
1003 	struct xe_gt *gt = m->tile->primary_gt;
1004 	struct xe_device *xe = gt_to_xe(gt);
1005 	bool clear_system_ccs = (xe_bo_needs_ccs_pages(bo) && !IS_DGFX(xe)) ? true : false;
1006 	struct dma_fence *fence = NULL;
1007 	u64 size = bo->size;
1008 	struct xe_res_cursor src_it;
1009 	struct ttm_resource *src = dst;
1010 	int err;
1011 
1012 	if (!clear_vram)
1013 		xe_res_first_sg(xe_bo_sg(bo), 0, bo->size, &src_it);
1014 	else
1015 		xe_res_first(src, 0, bo->size, &src_it);
1016 
1017 	while (size) {
1018 		u64 clear_L0_ofs;
1019 		u32 clear_L0_pt;
1020 		u32 flush_flags = 0;
1021 		u64 clear_L0;
1022 		struct xe_sched_job *job;
1023 		struct xe_bb *bb;
1024 		u32 batch_size, update_idx;
1025 
1026 		bool usm = xe->info.has_usm;
1027 		u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;
1028 
1029 		clear_L0 = xe_migrate_res_sizes(m, &src_it);
1030 
1031 		/* Calculate final sizes and batch size.. */
1032 		batch_size = 2 +
1033 			pte_update_size(m, clear_vram, src, &src_it,
1034 					&clear_L0, &clear_L0_ofs, &clear_L0_pt,
1035 					clear_system_ccs ? 0 : emit_clear_cmd_len(gt), 0,
1036 					avail_pts);
1037 
1038 		if (xe_device_has_flat_ccs(xe))
1039 			batch_size += EMIT_COPY_CCS_DW;
1040 
1041 		/* Clear commands */
1042 
1043 		if (WARN_ON_ONCE(!clear_L0))
1044 			break;
1045 
1046 		bb = xe_bb_new(gt, batch_size, usm);
1047 		if (IS_ERR(bb)) {
1048 			err = PTR_ERR(bb);
1049 			goto err_sync;
1050 		}
1051 
1052 		size -= clear_L0;
1053 		/* Preemption is enabled again by the ring ops. */
1054 		if (clear_vram && xe_migrate_allow_identity(clear_L0, &src_it))
1055 			xe_res_next(&src_it, clear_L0);
1056 		else
1057 			emit_pte(m, bb, clear_L0_pt, clear_vram, clear_system_ccs,
1058 				 &src_it, clear_L0, dst);
1059 
1060 		bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
1061 		update_idx = bb->len;
1062 
1063 		if (!clear_system_ccs)
1064 			emit_clear(gt, bb, clear_L0_ofs, clear_L0, XE_PAGE_SIZE, clear_vram);
1065 
1066 		if (xe_device_has_flat_ccs(xe)) {
1067 			emit_copy_ccs(gt, bb, clear_L0_ofs, true,
1068 				      m->cleared_mem_ofs, false, clear_L0);
1069 			flush_flags = MI_FLUSH_DW_CCS;
1070 		}
1071 
1072 		job = xe_bb_create_migration_job(m->q, bb,
1073 						 xe_migrate_batch_base(m, usm),
1074 						 update_idx);
1075 		if (IS_ERR(job)) {
1076 			err = PTR_ERR(job);
1077 			goto err;
1078 		}
1079 
1080 		xe_sched_job_add_migrate_flush(job, flush_flags);
1081 		if (!fence) {
1082 			/*
1083 			 * There can't be anything userspace related at this
1084 			 * point, so we just need to respect any potential move
1085 			 * fences, which are always tracked as
1086 			 * DMA_RESV_USAGE_KERNEL.
1087 			 */
1088 			err = xe_sched_job_add_deps(job, bo->ttm.base.resv,
1089 						    DMA_RESV_USAGE_KERNEL);
1090 			if (err)
1091 				goto err_job;
1092 		}
1093 
1094 		mutex_lock(&m->job_mutex);
1095 		xe_sched_job_arm(job);
1096 		dma_fence_put(fence);
1097 		fence = dma_fence_get(&job->drm.s_fence->finished);
1098 		xe_sched_job_push(job);
1099 
1100 		dma_fence_put(m->fence);
1101 		m->fence = dma_fence_get(fence);
1102 
1103 		mutex_unlock(&m->job_mutex);
1104 
1105 		xe_bb_free(bb, fence);
1106 		continue;
1107 
1108 err_job:
1109 		xe_sched_job_put(job);
1110 err:
1111 		xe_bb_free(bb, NULL);
1112 err_sync:
1113 		/* Sync partial copies if any. FIXME: job_mutex? */
1114 		if (fence) {
1115 			dma_fence_wait(m->fence, false);
1116 			dma_fence_put(fence);
1117 		}
1118 
1119 		return ERR_PTR(err);
1120 	}
1121 
1122 	if (clear_system_ccs)
1123 		bo->ccs_cleared = true;
1124 
1125 	return fence;
1126 }
1127 
1128 static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs,
1129 			  const struct xe_vm_pgtable_update *update,
1130 			  struct xe_migrate_pt_update *pt_update)
1131 {
1132 	const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
1133 	u32 chunk;
1134 	u32 ofs = update->ofs, size = update->qwords;
1135 
1136 	/*
1137 	 * If we have 512 entries (max), we would populate it ourselves,
1138 	 * and update the PDE above it to the new pointer.
1139 	 * The only time this can only happen if we have to update the top
1140 	 * PDE. This requires a BO that is almost vm->size big.
1141 	 *
1142 	 * This shouldn't be possible in practice.. might change when 16K
1143 	 * pages are used. Hence the assert.
1144 	 */
1145 	xe_tile_assert(tile, update->qwords < MAX_NUM_PTE);
1146 	if (!ppgtt_ofs)
1147 		ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile),
1148 						xe_bo_addr(update->pt_bo, 0,
1149 							   XE_PAGE_SIZE));
1150 
1151 	do {
1152 		u64 addr = ppgtt_ofs + ofs * 8;
1153 
1154 		chunk = min(size, MAX_PTE_PER_SDI);
1155 
1156 		/* Ensure populatefn can do memset64 by aligning bb->cs */
1157 		if (!(bb->len & 1))
1158 			bb->cs[bb->len++] = MI_NOOP;
1159 
1160 		bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk);
1161 		bb->cs[bb->len++] = lower_32_bits(addr);
1162 		bb->cs[bb->len++] = upper_32_bits(addr);
1163 		ops->populate(pt_update, tile, NULL, bb->cs + bb->len, ofs, chunk,
1164 			      update);
1165 
1166 		bb->len += chunk * 2;
1167 		ofs += chunk;
1168 		size -= chunk;
1169 	} while (size);
1170 }
1171 
1172 struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m)
1173 {
1174 	return xe_vm_get(m->q->vm);
1175 }
1176 
1177 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
1178 struct migrate_test_params {
1179 	struct xe_test_priv base;
1180 	bool force_gpu;
1181 };
1182 
1183 #define to_migrate_test_params(_priv) \
1184 	container_of(_priv, struct migrate_test_params, base)
1185 #endif
1186 
1187 static struct dma_fence *
1188 xe_migrate_update_pgtables_cpu(struct xe_migrate *m,
1189 			       struct xe_vm *vm, struct xe_bo *bo,
1190 			       const struct  xe_vm_pgtable_update *updates,
1191 			       u32 num_updates, bool wait_vm,
1192 			       struct xe_migrate_pt_update *pt_update)
1193 {
1194 	XE_TEST_DECLARE(struct migrate_test_params *test =
1195 			to_migrate_test_params
1196 			(xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));)
1197 	const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
1198 	struct dma_fence *fence;
1199 	int err;
1200 	u32 i;
1201 
1202 	if (XE_TEST_ONLY(test && test->force_gpu))
1203 		return ERR_PTR(-ETIME);
1204 
1205 	if (bo && !dma_resv_test_signaled(bo->ttm.base.resv,
1206 					  DMA_RESV_USAGE_KERNEL))
1207 		return ERR_PTR(-ETIME);
1208 
1209 	if (wait_vm && !dma_resv_test_signaled(xe_vm_resv(vm),
1210 					       DMA_RESV_USAGE_BOOKKEEP))
1211 		return ERR_PTR(-ETIME);
1212 
1213 	if (ops->pre_commit) {
1214 		pt_update->job = NULL;
1215 		err = ops->pre_commit(pt_update);
1216 		if (err)
1217 			return ERR_PTR(err);
1218 	}
1219 	for (i = 0; i < num_updates; i++) {
1220 		const struct xe_vm_pgtable_update *update = &updates[i];
1221 
1222 		ops->populate(pt_update, m->tile, &update->pt_bo->vmap, NULL,
1223 			      update->ofs, update->qwords, update);
1224 	}
1225 
1226 	if (vm) {
1227 		trace_xe_vm_cpu_bind(vm);
1228 		xe_device_wmb(vm->xe);
1229 	}
1230 
1231 	fence = dma_fence_get_stub();
1232 
1233 	return fence;
1234 }
1235 
1236 static bool no_in_syncs(struct xe_vm *vm, struct xe_exec_queue *q,
1237 			struct xe_sync_entry *syncs, u32 num_syncs)
1238 {
1239 	struct dma_fence *fence;
1240 	int i;
1241 
1242 	for (i = 0; i < num_syncs; i++) {
1243 		fence = syncs[i].fence;
1244 
1245 		if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
1246 				       &fence->flags))
1247 			return false;
1248 	}
1249 	if (q) {
1250 		fence = xe_exec_queue_last_fence_get(q, vm);
1251 		if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
1252 			dma_fence_put(fence);
1253 			return false;
1254 		}
1255 		dma_fence_put(fence);
1256 	}
1257 
1258 	return true;
1259 }
1260 
1261 /**
1262  * xe_migrate_update_pgtables() - Pipelined page-table update
1263  * @m: The migrate context.
1264  * @vm: The vm we'll be updating.
1265  * @bo: The bo whose dma-resv we will await before updating, or NULL if userptr.
1266  * @q: The exec queue to be used for the update or NULL if the default
1267  * migration engine is to be used.
1268  * @updates: An array of update descriptors.
1269  * @num_updates: Number of descriptors in @updates.
1270  * @syncs: Array of xe_sync_entry to await before updating. Note that waits
1271  * will block the engine timeline.
1272  * @num_syncs: Number of entries in @syncs.
1273  * @pt_update: Pointer to a struct xe_migrate_pt_update, which contains
1274  * pointers to callback functions and, if subclassed, private arguments to
1275  * those.
1276  *
1277  * Perform a pipelined page-table update. The update descriptors are typically
1278  * built under the same lock critical section as a call to this function. If
1279  * using the default engine for the updates, they will be performed in the
1280  * order they grab the job_mutex. If different engines are used, external
1281  * synchronization is needed for overlapping updates to maintain page-table
1282  * consistency. Note that the meaing of "overlapping" is that the updates
1283  * touch the same page-table, which might be a higher-level page-directory.
1284  * If no pipelining is needed, then updates may be performed by the cpu.
1285  *
1286  * Return: A dma_fence that, when signaled, indicates the update completion.
1287  */
1288 struct dma_fence *
1289 xe_migrate_update_pgtables(struct xe_migrate *m,
1290 			   struct xe_vm *vm,
1291 			   struct xe_bo *bo,
1292 			   struct xe_exec_queue *q,
1293 			   const struct xe_vm_pgtable_update *updates,
1294 			   u32 num_updates,
1295 			   struct xe_sync_entry *syncs, u32 num_syncs,
1296 			   struct xe_migrate_pt_update *pt_update)
1297 {
1298 	const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
1299 	struct xe_tile *tile = m->tile;
1300 	struct xe_gt *gt = tile->primary_gt;
1301 	struct xe_device *xe = tile_to_xe(tile);
1302 	struct xe_sched_job *job;
1303 	struct dma_fence *fence;
1304 	struct drm_suballoc *sa_bo = NULL;
1305 	struct xe_vma *vma = pt_update->vma;
1306 	struct xe_bb *bb;
1307 	u32 i, batch_size, ppgtt_ofs, update_idx, page_ofs = 0;
1308 	u64 addr;
1309 	int err = 0;
1310 	bool usm = !q && xe->info.has_usm;
1311 	bool first_munmap_rebind = vma &&
1312 		vma->gpuva.flags & XE_VMA_FIRST_REBIND;
1313 	struct xe_exec_queue *q_override = !q ? m->q : q;
1314 	u16 pat_index = xe->pat.idx[XE_CACHE_WB];
1315 
1316 	/* Use the CPU if no in syncs and engine is idle */
1317 	if (no_in_syncs(vm, q, syncs, num_syncs) && xe_exec_queue_is_idle(q_override)) {
1318 		fence =  xe_migrate_update_pgtables_cpu(m, vm, bo, updates,
1319 							num_updates,
1320 							first_munmap_rebind,
1321 							pt_update);
1322 		if (!IS_ERR(fence) || fence == ERR_PTR(-EAGAIN))
1323 			return fence;
1324 	}
1325 
1326 	/* fixed + PTE entries */
1327 	if (IS_DGFX(xe))
1328 		batch_size = 2;
1329 	else
1330 		batch_size = 6 + num_updates * 2;
1331 
1332 	for (i = 0; i < num_updates; i++) {
1333 		u32 num_cmds = DIV_ROUND_UP(updates[i].qwords, MAX_PTE_PER_SDI);
1334 
1335 		/* align noop + MI_STORE_DATA_IMM cmd prefix */
1336 		batch_size += 4 * num_cmds + updates[i].qwords * 2;
1337 	}
1338 
1339 	/*
1340 	 * XXX: Create temp bo to copy from, if batch_size becomes too big?
1341 	 *
1342 	 * Worst case: Sum(2 * (each lower level page size) + (top level page size))
1343 	 * Should be reasonably bound..
1344 	 */
1345 	xe_tile_assert(tile, batch_size < SZ_128K);
1346 
1347 	bb = xe_bb_new(gt, batch_size, !q && xe->info.has_usm);
1348 	if (IS_ERR(bb))
1349 		return ERR_CAST(bb);
1350 
1351 	/* For sysmem PTE's, need to map them in our hole.. */
1352 	if (!IS_DGFX(xe)) {
1353 		ppgtt_ofs = NUM_KERNEL_PDE - 1;
1354 		if (q) {
1355 			xe_tile_assert(tile, num_updates <= NUM_VMUSA_WRITES_PER_UNIT);
1356 
1357 			sa_bo = drm_suballoc_new(&m->vm_update_sa, 1,
1358 						 GFP_KERNEL, true, 0);
1359 			if (IS_ERR(sa_bo)) {
1360 				err = PTR_ERR(sa_bo);
1361 				goto err_bb;
1362 			}
1363 
1364 			ppgtt_ofs = NUM_KERNEL_PDE +
1365 				(drm_suballoc_soffset(sa_bo) /
1366 				 NUM_VMUSA_UNIT_PER_PAGE);
1367 			page_ofs = (drm_suballoc_soffset(sa_bo) %
1368 				    NUM_VMUSA_UNIT_PER_PAGE) *
1369 				VM_SA_UPDATE_UNIT_SIZE;
1370 		}
1371 
1372 		/* Map our PT's to gtt */
1373 		bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(num_updates);
1374 		bb->cs[bb->len++] = ppgtt_ofs * XE_PAGE_SIZE + page_ofs;
1375 		bb->cs[bb->len++] = 0; /* upper_32_bits */
1376 
1377 		for (i = 0; i < num_updates; i++) {
1378 			struct xe_bo *pt_bo = updates[i].pt_bo;
1379 
1380 			xe_tile_assert(tile, pt_bo->size == SZ_4K);
1381 
1382 			addr = vm->pt_ops->pte_encode_bo(pt_bo, 0, pat_index, 0);
1383 			bb->cs[bb->len++] = lower_32_bits(addr);
1384 			bb->cs[bb->len++] = upper_32_bits(addr);
1385 		}
1386 
1387 		bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
1388 		update_idx = bb->len;
1389 
1390 		addr = xe_migrate_vm_addr(ppgtt_ofs, 0) +
1391 			(page_ofs / sizeof(u64)) * XE_PAGE_SIZE;
1392 		for (i = 0; i < num_updates; i++)
1393 			write_pgtable(tile, bb, addr + i * XE_PAGE_SIZE,
1394 				      &updates[i], pt_update);
1395 	} else {
1396 		/* phys pages, no preamble required */
1397 		bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
1398 		update_idx = bb->len;
1399 
1400 		for (i = 0; i < num_updates; i++)
1401 			write_pgtable(tile, bb, 0, &updates[i], pt_update);
1402 	}
1403 
1404 	job = xe_bb_create_migration_job(q ?: m->q, bb,
1405 					 xe_migrate_batch_base(m, usm),
1406 					 update_idx);
1407 	if (IS_ERR(job)) {
1408 		err = PTR_ERR(job);
1409 		goto err_sa;
1410 	}
1411 
1412 	/* Wait on BO move */
1413 	if (bo) {
1414 		err = xe_sched_job_add_deps(job, bo->ttm.base.resv,
1415 					    DMA_RESV_USAGE_KERNEL);
1416 		if (err)
1417 			goto err_job;
1418 	}
1419 
1420 	/*
1421 	 * Munmap style VM unbind, need to wait for all jobs to be complete /
1422 	 * trigger preempts before moving forward
1423 	 */
1424 	if (first_munmap_rebind) {
1425 		err = xe_sched_job_add_deps(job, xe_vm_resv(vm),
1426 					    DMA_RESV_USAGE_BOOKKEEP);
1427 		if (err)
1428 			goto err_job;
1429 	}
1430 
1431 	err = xe_sched_job_last_fence_add_dep(job, vm);
1432 	for (i = 0; !err && i < num_syncs; i++)
1433 		err = xe_sync_entry_add_deps(&syncs[i], job);
1434 
1435 	if (err)
1436 		goto err_job;
1437 
1438 	if (ops->pre_commit) {
1439 		pt_update->job = job;
1440 		err = ops->pre_commit(pt_update);
1441 		if (err)
1442 			goto err_job;
1443 	}
1444 	if (!q)
1445 		mutex_lock(&m->job_mutex);
1446 
1447 	xe_sched_job_arm(job);
1448 	fence = dma_fence_get(&job->drm.s_fence->finished);
1449 	xe_sched_job_push(job);
1450 
1451 	if (!q)
1452 		mutex_unlock(&m->job_mutex);
1453 
1454 	xe_bb_free(bb, fence);
1455 	drm_suballoc_free(sa_bo, fence);
1456 
1457 	return fence;
1458 
1459 err_job:
1460 	xe_sched_job_put(job);
1461 err_sa:
1462 	drm_suballoc_free(sa_bo, NULL);
1463 err_bb:
1464 	xe_bb_free(bb, NULL);
1465 	return ERR_PTR(err);
1466 }
1467 
1468 /**
1469  * xe_migrate_wait() - Complete all operations using the xe_migrate context
1470  * @m: Migrate context to wait for.
1471  *
1472  * Waits until the GPU no longer uses the migrate context's default engine
1473  * or its page-table objects. FIXME: What about separate page-table update
1474  * engines?
1475  */
1476 void xe_migrate_wait(struct xe_migrate *m)
1477 {
1478 	if (m->fence)
1479 		dma_fence_wait(m->fence, false);
1480 }
1481 
1482 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
1483 #include "tests/xe_migrate.c"
1484 #endif
1485