xref: /linux/drivers/gpu/drm/xe/xe_ggtt.c (revision 7a4ffec9fd54ea27395e24dff726dbf58e2fe06b)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2021 Intel Corporation
4  */
5 
6 #include "xe_ggtt.h"
7 
8 #include <linux/io-64-nonatomic-lo-hi.h>
9 #include <linux/sizes.h>
10 
11 #include <drm/drm_drv.h>
12 #include <drm/drm_managed.h>
13 #include <drm/intel/i915_drm.h>
14 #include <generated/xe_wa_oob.h>
15 
16 #include "regs/xe_gt_regs.h"
17 #include "regs/xe_gtt_defs.h"
18 #include "regs/xe_regs.h"
19 #include "xe_assert.h"
20 #include "xe_bo.h"
21 #include "xe_device.h"
22 #include "xe_gt.h"
23 #include "xe_gt_printk.h"
24 #include "xe_gt_sriov_vf.h"
25 #include "xe_gt_tlb_invalidation.h"
26 #include "xe_map.h"
27 #include "xe_mmio.h"
28 #include "xe_pm.h"
29 #include "xe_sriov.h"
30 #include "xe_wa.h"
31 #include "xe_wopcm.h"
32 
33 /**
34  * DOC: Global Graphics Translation Table (GGTT)
35  *
36  * Xe GGTT implements the support for a Global Virtual Address space that is used
37  * for resources that are accessible to privileged (i.e. kernel-mode) processes,
38  * and not tied to a specific user-level process. For example, the Graphics
39  * micro-Controller (GuC) and Display Engine (if present) utilize this Global
40  * address space.
41  *
42  * The Global GTT (GGTT) translates from the Global virtual address to a physical
43  * address that can be accessed by HW. The GGTT is a flat, single-level table.
44  *
45  * Xe implements a simplified version of the GGTT specifically managing only a
46  * certain range of it that goes from the Write Once Protected Content Memory (WOPCM)
47  * Layout to a predefined GUC_GGTT_TOP. This approach avoids complications related to
48  * the GuC (Graphics Microcontroller) hardware limitations. The GuC address space
49  * is limited on both ends of the GGTT, because the GuC shim HW redirects
50  * accesses to those addresses to other HW areas instead of going through the
51  * GGTT. On the bottom end, the GuC can't access offsets below the WOPCM size,
52  * while on the top side the limit is fixed at GUC_GGTT_TOP. To keep things
53  * simple, instead of checking each object to see if they are accessed by GuC or
54  * not, we just exclude those areas from the allocator. Additionally, to simplify
55  * the driver load, we use the maximum WOPCM size in this logic instead of the
56  * programmed one, so we don't need to wait until the actual size to be
57  * programmed is determined (which requires FW fetch) before initializing the
58  * GGTT. These simplifications might waste space in the GGTT (about 20-25 MBs
59  * depending on the platform) but we can live with this. Another benefit of this
60  * is the GuC bootrom can't access anything below the WOPCM max size so anything
61  * the bootrom needs to access (e.g. a RSA key) needs to be placed in the GGTT
62  * above the WOPCM max size. Starting the GGTT allocations above the WOPCM max
63  * give us the correct placement for free.
64  */
65 
66 static u64 xelp_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
67 				   u16 pat_index)
68 {
69 	u64 pte;
70 
71 	pte = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE);
72 	pte |= XE_PAGE_PRESENT;
73 
74 	if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo))
75 		pte |= XE_GGTT_PTE_DM;
76 
77 	return pte;
78 }
79 
80 static u64 xelpg_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
81 				    u16 pat_index)
82 {
83 	struct xe_device *xe = xe_bo_device(bo);
84 	u64 pte;
85 
86 	pte = xelp_ggtt_pte_encode_bo(bo, bo_offset, pat_index);
87 
88 	xe_assert(xe, pat_index <= 3);
89 
90 	if (pat_index & BIT(0))
91 		pte |= XELPG_GGTT_PTE_PAT0;
92 
93 	if (pat_index & BIT(1))
94 		pte |= XELPG_GGTT_PTE_PAT1;
95 
96 	return pte;
97 }
98 
99 static unsigned int probe_gsm_size(struct pci_dev *pdev)
100 {
101 	u16 gmch_ctl, ggms;
102 
103 	pci_read_config_word(pdev, SNB_GMCH_CTRL, &gmch_ctl);
104 	ggms = (gmch_ctl >> BDW_GMCH_GGMS_SHIFT) & BDW_GMCH_GGMS_MASK;
105 	return ggms ? SZ_1M << ggms : 0;
106 }
107 
108 static void ggtt_update_access_counter(struct xe_ggtt *ggtt)
109 {
110 	struct xe_gt *gt = XE_WA(ggtt->tile->primary_gt, 22019338487) ? ggtt->tile->primary_gt :
111 			   ggtt->tile->media_gt;
112 	u32 max_gtt_writes = XE_WA(ggtt->tile->primary_gt, 22019338487) ? 1100 : 63;
113 	/*
114 	 * Wa_22019338487: GMD_ID is a RO register, a dummy write forces gunit
115 	 * to wait for completion of prior GTT writes before letting this through.
116 	 * This needs to be done for all GGTT writes originating from the CPU.
117 	 */
118 	lockdep_assert_held(&ggtt->lock);
119 
120 	if ((++ggtt->access_count % max_gtt_writes) == 0) {
121 		xe_mmio_write32(gt, GMD_ID, 0x0);
122 		ggtt->access_count = 0;
123 	}
124 }
125 
126 static void xe_ggtt_set_pte(struct xe_ggtt *ggtt, u64 addr, u64 pte)
127 {
128 	xe_tile_assert(ggtt->tile, !(addr & XE_PTE_MASK));
129 	xe_tile_assert(ggtt->tile, addr < ggtt->size);
130 
131 	writeq(pte, &ggtt->gsm[addr >> XE_PTE_SHIFT]);
132 }
133 
134 static void xe_ggtt_set_pte_and_flush(struct xe_ggtt *ggtt, u64 addr, u64 pte)
135 {
136 	xe_ggtt_set_pte(ggtt, addr, pte);
137 	ggtt_update_access_counter(ggtt);
138 }
139 
140 static void xe_ggtt_clear(struct xe_ggtt *ggtt, u64 start, u64 size)
141 {
142 	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB];
143 	u64 end = start + size - 1;
144 	u64 scratch_pte;
145 
146 	xe_tile_assert(ggtt->tile, start < end);
147 
148 	if (ggtt->scratch)
149 		scratch_pte = ggtt->pt_ops->pte_encode_bo(ggtt->scratch, 0,
150 							  pat_index);
151 	else
152 		scratch_pte = 0;
153 
154 	while (start < end) {
155 		ggtt->pt_ops->ggtt_set_pte(ggtt, start, scratch_pte);
156 		start += XE_PAGE_SIZE;
157 	}
158 }
159 
160 static void ggtt_fini_early(struct drm_device *drm, void *arg)
161 {
162 	struct xe_ggtt *ggtt = arg;
163 
164 	destroy_workqueue(ggtt->wq);
165 	mutex_destroy(&ggtt->lock);
166 	drm_mm_takedown(&ggtt->mm);
167 }
168 
169 static void ggtt_fini(void *arg)
170 {
171 	struct xe_ggtt *ggtt = arg;
172 
173 	ggtt->scratch = NULL;
174 }
175 
176 static void primelockdep(struct xe_ggtt *ggtt)
177 {
178 	if (!IS_ENABLED(CONFIG_LOCKDEP))
179 		return;
180 
181 	fs_reclaim_acquire(GFP_KERNEL);
182 	might_lock(&ggtt->lock);
183 	fs_reclaim_release(GFP_KERNEL);
184 }
185 
186 static const struct xe_ggtt_pt_ops xelp_pt_ops = {
187 	.pte_encode_bo = xelp_ggtt_pte_encode_bo,
188 	.ggtt_set_pte = xe_ggtt_set_pte,
189 };
190 
191 static const struct xe_ggtt_pt_ops xelpg_pt_ops = {
192 	.pte_encode_bo = xelpg_ggtt_pte_encode_bo,
193 	.ggtt_set_pte = xe_ggtt_set_pte,
194 };
195 
196 static const struct xe_ggtt_pt_ops xelpg_pt_wa_ops = {
197 	.pte_encode_bo = xelpg_ggtt_pte_encode_bo,
198 	.ggtt_set_pte = xe_ggtt_set_pte_and_flush,
199 };
200 
201 /**
202  * xe_ggtt_init_early - Early GGTT initialization
203  * @ggtt: the &xe_ggtt to be initialized
204  *
205  * It allows to create new mappings usable by the GuC.
206  * Mappings are not usable by the HW engines, as it doesn't have scratch nor
207  * initial clear done to it yet. That will happen in the regular, non-early
208  * GGTT initialization.
209  *
210  * Return: 0 on success or a negative error code on failure.
211  */
212 int xe_ggtt_init_early(struct xe_ggtt *ggtt)
213 {
214 	struct xe_device *xe = tile_to_xe(ggtt->tile);
215 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
216 	unsigned int gsm_size;
217 	int err;
218 
219 	if (IS_SRIOV_VF(xe))
220 		gsm_size = SZ_8M; /* GGTT is expected to be 4GiB */
221 	else
222 		gsm_size = probe_gsm_size(pdev);
223 
224 	if (gsm_size == 0) {
225 		drm_err(&xe->drm, "Hardware reported no preallocated GSM\n");
226 		return -ENOMEM;
227 	}
228 
229 	ggtt->gsm = ggtt->tile->mmio.regs + SZ_8M;
230 	ggtt->size = (gsm_size / 8) * (u64) XE_PAGE_SIZE;
231 
232 	if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)
233 		ggtt->flags |= XE_GGTT_FLAGS_64K;
234 
235 	if (ggtt->size > GUC_GGTT_TOP)
236 		ggtt->size = GUC_GGTT_TOP;
237 
238 	if (GRAPHICS_VERx100(xe) >= 1270)
239 		ggtt->pt_ops = (ggtt->tile->media_gt &&
240 			       XE_WA(ggtt->tile->media_gt, 22019338487)) ||
241 			       XE_WA(ggtt->tile->primary_gt, 22019338487) ?
242 			       &xelpg_pt_wa_ops : &xelpg_pt_ops;
243 	else
244 		ggtt->pt_ops = &xelp_pt_ops;
245 
246 	ggtt->wq = alloc_workqueue("xe-ggtt-wq", 0, 0);
247 
248 	drm_mm_init(&ggtt->mm, xe_wopcm_size(xe),
249 		    ggtt->size - xe_wopcm_size(xe));
250 	mutex_init(&ggtt->lock);
251 	primelockdep(ggtt);
252 
253 	err = drmm_add_action_or_reset(&xe->drm, ggtt_fini_early, ggtt);
254 	if (err)
255 		return err;
256 
257 	if (IS_SRIOV_VF(xe)) {
258 		err = xe_gt_sriov_vf_prepare_ggtt(xe_tile_get_gt(ggtt->tile, 0));
259 		if (err)
260 			return err;
261 	}
262 
263 	return 0;
264 }
265 
266 static void xe_ggtt_invalidate(struct xe_ggtt *ggtt);
267 
268 static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt)
269 {
270 	struct drm_mm_node *hole;
271 	u64 start, end;
272 
273 	/* Display may have allocated inside ggtt, so be careful with clearing here */
274 	mutex_lock(&ggtt->lock);
275 	drm_mm_for_each_hole(hole, &ggtt->mm, start, end)
276 		xe_ggtt_clear(ggtt, start, end - start);
277 
278 	xe_ggtt_invalidate(ggtt);
279 	mutex_unlock(&ggtt->lock);
280 }
281 
282 static void ggtt_node_remove(struct xe_ggtt_node *node)
283 {
284 	struct xe_ggtt *ggtt = node->ggtt;
285 	struct xe_device *xe = tile_to_xe(ggtt->tile);
286 	bool bound;
287 	int idx;
288 
289 	bound = drm_dev_enter(&xe->drm, &idx);
290 
291 	mutex_lock(&ggtt->lock);
292 	if (bound)
293 		xe_ggtt_clear(ggtt, node->base.start, node->base.size);
294 	drm_mm_remove_node(&node->base);
295 	node->base.size = 0;
296 	mutex_unlock(&ggtt->lock);
297 
298 	if (!bound)
299 		goto free_node;
300 
301 	if (node->invalidate_on_remove)
302 		xe_ggtt_invalidate(ggtt);
303 
304 	drm_dev_exit(idx);
305 
306 free_node:
307 	xe_ggtt_node_fini(node);
308 }
309 
310 static void ggtt_node_remove_work_func(struct work_struct *work)
311 {
312 	struct xe_ggtt_node *node = container_of(work, typeof(*node),
313 						 delayed_removal_work);
314 	struct xe_device *xe = tile_to_xe(node->ggtt->tile);
315 
316 	xe_pm_runtime_get(xe);
317 	ggtt_node_remove(node);
318 	xe_pm_runtime_put(xe);
319 }
320 
321 /**
322  * xe_ggtt_node_remove - Remove a &xe_ggtt_node from the GGTT
323  * @node: the &xe_ggtt_node to be removed
324  * @invalidate: if node needs invalidation upon removal
325  */
326 void xe_ggtt_node_remove(struct xe_ggtt_node *node, bool invalidate)
327 {
328 	struct xe_ggtt *ggtt;
329 	struct xe_device *xe;
330 
331 	if (!node || !node->ggtt)
332 		return;
333 
334 	ggtt = node->ggtt;
335 	xe = tile_to_xe(ggtt->tile);
336 
337 	node->invalidate_on_remove = invalidate;
338 
339 	if (xe_pm_runtime_get_if_active(xe)) {
340 		ggtt_node_remove(node);
341 		xe_pm_runtime_put(xe);
342 	} else {
343 		queue_work(ggtt->wq, &node->delayed_removal_work);
344 	}
345 }
346 
347 /**
348  * xe_ggtt_init - Regular non-early GGTT initialization
349  * @ggtt: the &xe_ggtt to be initialized
350  *
351  * Return: 0 on success or a negative error code on failure.
352  */
353 int xe_ggtt_init(struct xe_ggtt *ggtt)
354 {
355 	struct xe_device *xe = tile_to_xe(ggtt->tile);
356 	unsigned int flags;
357 	int err;
358 
359 	/*
360 	 * So we don't need to worry about 64K GGTT layout when dealing with
361 	 * scratch entires, rather keep the scratch page in system memory on
362 	 * platforms where 64K pages are needed for VRAM.
363 	 */
364 	flags = XE_BO_FLAG_PINNED;
365 	if (ggtt->flags & XE_GGTT_FLAGS_64K)
366 		flags |= XE_BO_FLAG_SYSTEM;
367 	else
368 		flags |= XE_BO_FLAG_VRAM_IF_DGFX(ggtt->tile);
369 
370 	ggtt->scratch = xe_managed_bo_create_pin_map(xe, ggtt->tile, XE_PAGE_SIZE, flags);
371 	if (IS_ERR(ggtt->scratch)) {
372 		err = PTR_ERR(ggtt->scratch);
373 		goto err;
374 	}
375 
376 	xe_map_memset(xe, &ggtt->scratch->vmap, 0, 0, ggtt->scratch->size);
377 
378 	xe_ggtt_initial_clear(ggtt);
379 
380 	return devm_add_action_or_reset(xe->drm.dev, ggtt_fini, ggtt);
381 err:
382 	ggtt->scratch = NULL;
383 	return err;
384 }
385 
386 static void ggtt_invalidate_gt_tlb(struct xe_gt *gt)
387 {
388 	int err;
389 
390 	if (!gt)
391 		return;
392 
393 	err = xe_gt_tlb_invalidation_ggtt(gt);
394 	if (err)
395 		drm_warn(&gt_to_xe(gt)->drm, "xe_gt_tlb_invalidation_ggtt error=%d", err);
396 }
397 
398 static void xe_ggtt_invalidate(struct xe_ggtt *ggtt)
399 {
400 	/* Each GT in a tile has its own TLB to cache GGTT lookups */
401 	ggtt_invalidate_gt_tlb(ggtt->tile->primary_gt);
402 	ggtt_invalidate_gt_tlb(ggtt->tile->media_gt);
403 }
404 
405 static void xe_ggtt_dump_node(struct xe_ggtt *ggtt,
406 			      const struct drm_mm_node *node, const char *description)
407 {
408 	char buf[10];
409 
410 	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
411 		string_get_size(node->size, 1, STRING_UNITS_2, buf, sizeof(buf));
412 		xe_gt_dbg(ggtt->tile->primary_gt, "GGTT %#llx-%#llx (%s) %s\n",
413 			  node->start, node->start + node->size, buf, description);
414 	}
415 }
416 
417 /**
418  * xe_ggtt_node_insert_balloon - prevent allocation of specified GGTT addresses
419  * @node: the &xe_ggtt_node to hold reserved GGTT node
420  * @start: the starting GGTT address of the reserved region
421  * @end: then end GGTT address of the reserved region
422  *
423  * Use xe_ggtt_node_remove_balloon() to release a reserved GGTT node.
424  *
425  * Return: 0 on success or a negative error code on failure.
426  */
427 int xe_ggtt_node_insert_balloon(struct xe_ggtt_node *node, u64 start, u64 end)
428 {
429 	struct xe_ggtt *ggtt = node->ggtt;
430 	int err;
431 
432 	xe_tile_assert(ggtt->tile, start < end);
433 	xe_tile_assert(ggtt->tile, IS_ALIGNED(start, XE_PAGE_SIZE));
434 	xe_tile_assert(ggtt->tile, IS_ALIGNED(end, XE_PAGE_SIZE));
435 	xe_tile_assert(ggtt->tile, !drm_mm_node_allocated(&node->base));
436 
437 	node->base.color = 0;
438 	node->base.start = start;
439 	node->base.size = end - start;
440 
441 	mutex_lock(&ggtt->lock);
442 	err = drm_mm_reserve_node(&ggtt->mm, &node->base);
443 	mutex_unlock(&ggtt->lock);
444 
445 	if (xe_gt_WARN(ggtt->tile->primary_gt, err,
446 		       "Failed to balloon GGTT %#llx-%#llx (%pe)\n",
447 		       node->base.start, node->base.start + node->base.size, ERR_PTR(err)))
448 		return err;
449 
450 	xe_ggtt_dump_node(ggtt, &node->base, "balloon");
451 	return 0;
452 }
453 
454 /**
455  * xe_ggtt_node_remove_balloon - release a reserved GGTT region
456  * @node: the &xe_ggtt_node with reserved GGTT region
457  *
458  * See xe_ggtt_node_insert_balloon() for details.
459  */
460 void xe_ggtt_node_remove_balloon(struct xe_ggtt_node *node)
461 {
462 	if (!node || !node->ggtt)
463 		return;
464 
465 	if (!drm_mm_node_allocated(&node->base))
466 		goto free_node;
467 
468 	xe_ggtt_dump_node(node->ggtt, &node->base, "remove-balloon");
469 
470 	mutex_lock(&node->ggtt->lock);
471 	drm_mm_remove_node(&node->base);
472 	mutex_unlock(&node->ggtt->lock);
473 
474 free_node:
475 	xe_ggtt_node_fini(node);
476 }
477 
478 /**
479  * xe_ggtt_node_insert_locked - Locked version to insert a &xe_ggtt_node into the GGTT
480  * @node: the &xe_ggtt_node to be inserted
481  * @size: size of the node
482  * @align: alignment constrain of the node
483  * @mm_flags: flags to control the node behavior
484  *
485  * It cannot be called without first having called xe_ggtt_init() once.
486  * To be used in cases where ggtt->lock is already taken.
487  *
488  * Return: 0 on success or a negative error code on failure.
489  */
490 int xe_ggtt_node_insert_locked(struct xe_ggtt_node *node,
491 			       u32 size, u32 align, u32 mm_flags)
492 {
493 	return drm_mm_insert_node_generic(&node->ggtt->mm, &node->base, size, align, 0,
494 					  mm_flags);
495 }
496 
497 /**
498  * xe_ggtt_node_insert - Insert a &xe_ggtt_node into the GGTT
499  * @node: the &xe_ggtt_node to be inserted
500  * @size: size of the node
501  * @align: alignment constrain of the node
502  *
503  * It cannot be called without first having called xe_ggtt_init() once.
504  *
505  * Return: 0 on success or a negative error code on failure.
506  */
507 int xe_ggtt_node_insert(struct xe_ggtt_node *node, u32 size, u32 align)
508 {
509 	int ret;
510 
511 	if (!node || !node->ggtt)
512 		return -ENOENT;
513 
514 	mutex_lock(&node->ggtt->lock);
515 	ret = xe_ggtt_node_insert_locked(node, size, align,
516 					 DRM_MM_INSERT_HIGH);
517 	mutex_unlock(&node->ggtt->lock);
518 
519 	return ret;
520 }
521 
522 /**
523  * xe_ggtt_node_init - Initialize %xe_ggtt_node struct
524  * @ggtt: the &xe_ggtt where the new node will later be inserted/reserved.
525  *
526  * This function will allocated the struct %xe_ggtt_node and return it's pointer.
527  * This struct will then be freed after the node removal upon xe_ggtt_node_remove()
528  * or xe_ggtt_node_remove_balloon().
529  * Having %xe_ggtt_node struct allocated doesn't mean that the node is already allocated
530  * in GGTT. Only the xe_ggtt_node_insert(), xe_ggtt_node_insert_locked(),
531  * xe_ggtt_node_insert_balloon() will ensure the node is inserted or reserved in GGTT.
532  *
533  * Return: A pointer to %xe_ggtt_node struct on success. An ERR_PTR otherwise.
534  **/
535 struct xe_ggtt_node *xe_ggtt_node_init(struct xe_ggtt *ggtt)
536 {
537 	struct xe_ggtt_node *node = kzalloc(sizeof(*node), GFP_NOFS);
538 
539 	if (!node)
540 		return ERR_PTR(-ENOMEM);
541 
542 	INIT_WORK(&node->delayed_removal_work, ggtt_node_remove_work_func);
543 	node->ggtt = ggtt;
544 
545 	return node;
546 }
547 
548 /**
549  * xe_ggtt_node_fini - Forcebly finalize %xe_ggtt_node struct
550  * @node: the &xe_ggtt_node to be freed
551  *
552  * If anything went wrong with either xe_ggtt_node_insert(), xe_ggtt_node_insert_locked(),
553  * or xe_ggtt_node_insert_balloon(); and this @node is not going to be reused, then,
554  * this function needs to be called to free the %xe_ggtt_node struct
555  **/
556 void xe_ggtt_node_fini(struct xe_ggtt_node *node)
557 {
558 	kfree(node);
559 }
560 
561 /**
562  * xe_ggtt_node_allocated - Check if node is allocated in GGTT
563  * @node: the &xe_ggtt_node to be inspected
564  *
565  * Return: True if allocated, False otherwise.
566  */
567 bool xe_ggtt_node_allocated(const struct xe_ggtt_node *node)
568 {
569 	if (!node || !node->ggtt)
570 		return false;
571 
572 	return drm_mm_node_allocated(&node->base);
573 }
574 
575 /**
576  * xe_ggtt_map_bo - Map the BO into GGTT
577  * @ggtt: the &xe_ggtt where node will be mapped
578  * @bo: the &xe_bo to be mapped
579  */
580 void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
581 {
582 	u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB;
583 	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[cache_mode];
584 	u64 start;
585 	u64 offset, pte;
586 
587 	if (XE_WARN_ON(!bo->ggtt_node))
588 		return;
589 
590 	start = bo->ggtt_node->base.start;
591 
592 	for (offset = 0; offset < bo->size; offset += XE_PAGE_SIZE) {
593 		pte = ggtt->pt_ops->pte_encode_bo(bo, offset, pat_index);
594 		ggtt->pt_ops->ggtt_set_pte(ggtt, start + offset, pte);
595 	}
596 }
597 
598 static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
599 				  u64 start, u64 end)
600 {
601 	int err;
602 	u64 alignment = XE_PAGE_SIZE;
603 
604 	if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)
605 		alignment = SZ_64K;
606 
607 	if (XE_WARN_ON(bo->ggtt_node)) {
608 		/* Someone's already inserted this BO in the GGTT */
609 		xe_tile_assert(ggtt->tile, bo->ggtt_node->base.size == bo->size);
610 		return 0;
611 	}
612 
613 	err = xe_bo_validate(bo, NULL, false);
614 	if (err)
615 		return err;
616 
617 	xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile));
618 
619 	bo->ggtt_node = xe_ggtt_node_init(ggtt);
620 	if (IS_ERR(bo->ggtt_node)) {
621 		err = PTR_ERR(bo->ggtt_node);
622 		bo->ggtt_node = NULL;
623 		goto out;
624 	}
625 
626 	mutex_lock(&ggtt->lock);
627 	err = drm_mm_insert_node_in_range(&ggtt->mm, &bo->ggtt_node->base, bo->size,
628 					  alignment, 0, start, end, 0);
629 	if (err) {
630 		xe_ggtt_node_fini(bo->ggtt_node);
631 		bo->ggtt_node = NULL;
632 	} else {
633 		xe_ggtt_map_bo(ggtt, bo);
634 	}
635 	mutex_unlock(&ggtt->lock);
636 
637 	if (!err && bo->flags & XE_BO_FLAG_GGTT_INVALIDATE)
638 		xe_ggtt_invalidate(ggtt);
639 
640 out:
641 	xe_pm_runtime_put(tile_to_xe(ggtt->tile));
642 
643 	return err;
644 }
645 
646 /**
647  * xe_ggtt_insert_bo_at - Insert BO at a specific GGTT space
648  * @ggtt: the &xe_ggtt where bo will be inserted
649  * @bo: the &xe_bo to be inserted
650  * @start: address where it will be inserted
651  * @end: end of the range where it will be inserted
652  *
653  * Return: 0 on success or a negative error code on failure.
654  */
655 int xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
656 			 u64 start, u64 end)
657 {
658 	return __xe_ggtt_insert_bo_at(ggtt, bo, start, end);
659 }
660 
661 /**
662  * xe_ggtt_insert_bo - Insert BO into GGTT
663  * @ggtt: the &xe_ggtt where bo will be inserted
664  * @bo: the &xe_bo to be inserted
665  *
666  * Return: 0 on success or a negative error code on failure.
667  */
668 int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
669 {
670 	return __xe_ggtt_insert_bo_at(ggtt, bo, 0, U64_MAX);
671 }
672 
673 /**
674  * xe_ggtt_remove_bo - Remove a BO from the GGTT
675  * @ggtt: the &xe_ggtt where node will be removed
676  * @bo: the &xe_bo to be removed
677  */
678 void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
679 {
680 	if (XE_WARN_ON(!bo->ggtt_node))
681 		return;
682 
683 	/* This BO is not currently in the GGTT */
684 	xe_tile_assert(ggtt->tile, bo->ggtt_node->base.size == bo->size);
685 
686 	xe_ggtt_node_remove(bo->ggtt_node,
687 			    bo->flags & XE_BO_FLAG_GGTT_INVALIDATE);
688 }
689 
690 /**
691  * xe_ggtt_largest_hole - Largest GGTT hole
692  * @ggtt: the &xe_ggtt that will be inspected
693  * @alignment: minimum alignment
694  * @spare: If not NULL: in: desired memory size to be spared / out: Adjusted possible spare
695  *
696  * Return: size of the largest continuous GGTT region
697  */
698 u64 xe_ggtt_largest_hole(struct xe_ggtt *ggtt, u64 alignment, u64 *spare)
699 {
700 	const struct drm_mm *mm = &ggtt->mm;
701 	const struct drm_mm_node *entry;
702 	u64 hole_min_start = xe_wopcm_size(tile_to_xe(ggtt->tile));
703 	u64 hole_start, hole_end, hole_size;
704 	u64 max_hole = 0;
705 
706 	mutex_lock(&ggtt->lock);
707 
708 	drm_mm_for_each_hole(entry, mm, hole_start, hole_end) {
709 		hole_start = max(hole_start, hole_min_start);
710 		hole_start = ALIGN(hole_start, alignment);
711 		hole_end = ALIGN_DOWN(hole_end, alignment);
712 		if (hole_start >= hole_end)
713 			continue;
714 		hole_size = hole_end - hole_start;
715 		if (spare)
716 			*spare -= min3(*spare, hole_size, max_hole);
717 		max_hole = max(max_hole, hole_size);
718 	}
719 
720 	mutex_unlock(&ggtt->lock);
721 
722 	return max_hole;
723 }
724 
725 #ifdef CONFIG_PCI_IOV
726 static u64 xe_encode_vfid_pte(u16 vfid)
727 {
728 	return FIELD_PREP(GGTT_PTE_VFID, vfid) | XE_PAGE_PRESENT;
729 }
730 
731 static void xe_ggtt_assign_locked(struct xe_ggtt *ggtt, const struct drm_mm_node *node, u16 vfid)
732 {
733 	u64 start = node->start;
734 	u64 size = node->size;
735 	u64 end = start + size - 1;
736 	u64 pte = xe_encode_vfid_pte(vfid);
737 
738 	lockdep_assert_held(&ggtt->lock);
739 
740 	if (!drm_mm_node_allocated(node))
741 		return;
742 
743 	while (start < end) {
744 		ggtt->pt_ops->ggtt_set_pte(ggtt, start, pte);
745 		start += XE_PAGE_SIZE;
746 	}
747 
748 	xe_ggtt_invalidate(ggtt);
749 }
750 
751 /**
752  * xe_ggtt_assign - assign a GGTT region to the VF
753  * @node: the &xe_ggtt_node to update
754  * @vfid: the VF identifier
755  *
756  * This function is used by the PF driver to assign a GGTT region to the VF.
757  * In addition to PTE's VFID bits 11:2 also PRESENT bit 0 is set as on some
758  * platforms VFs can't modify that either.
759  */
760 void xe_ggtt_assign(const struct xe_ggtt_node *node, u16 vfid)
761 {
762 	mutex_lock(&node->ggtt->lock);
763 	xe_ggtt_assign_locked(node->ggtt, &node->base, vfid);
764 	mutex_unlock(&node->ggtt->lock);
765 }
766 #endif
767 
768 /**
769  * xe_ggtt_dump - Dump GGTT for debug
770  * @ggtt: the &xe_ggtt to be dumped
771  * @p: the &drm_mm_printer helper handle to be used to dump the information
772  *
773  * Return: 0 on success or a negative error code on failure.
774  */
775 int xe_ggtt_dump(struct xe_ggtt *ggtt, struct drm_printer *p)
776 {
777 	int err;
778 
779 	err = mutex_lock_interruptible(&ggtt->lock);
780 	if (err)
781 		return err;
782 
783 	drm_mm_print(&ggtt->mm, p);
784 	mutex_unlock(&ggtt->lock);
785 	return err;
786 }
787 
788 /**
789  * xe_ggtt_print_holes - Print holes
790  * @ggtt: the &xe_ggtt to be inspected
791  * @alignment: min alignment
792  * @p: the &drm_printer
793  *
794  * Print GGTT ranges that are available and return total size available.
795  *
796  * Return: Total available size.
797  */
798 u64 xe_ggtt_print_holes(struct xe_ggtt *ggtt, u64 alignment, struct drm_printer *p)
799 {
800 	const struct drm_mm *mm = &ggtt->mm;
801 	const struct drm_mm_node *entry;
802 	u64 hole_min_start = xe_wopcm_size(tile_to_xe(ggtt->tile));
803 	u64 hole_start, hole_end, hole_size;
804 	u64 total = 0;
805 	char buf[10];
806 
807 	mutex_lock(&ggtt->lock);
808 
809 	drm_mm_for_each_hole(entry, mm, hole_start, hole_end) {
810 		hole_start = max(hole_start, hole_min_start);
811 		hole_start = ALIGN(hole_start, alignment);
812 		hole_end = ALIGN_DOWN(hole_end, alignment);
813 		if (hole_start >= hole_end)
814 			continue;
815 		hole_size = hole_end - hole_start;
816 		total += hole_size;
817 
818 		string_get_size(hole_size, 1, STRING_UNITS_2, buf, sizeof(buf));
819 		drm_printf(p, "range:\t%#llx-%#llx\t(%s)\n",
820 			   hole_start, hole_end - 1, buf);
821 	}
822 
823 	mutex_unlock(&ggtt->lock);
824 
825 	return total;
826 }
827