xref: /linux/drivers/gpu/drm/xe/xe_ggtt.c (revision 3fd6c59042dbba50391e30862beac979491145fe)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2021 Intel Corporation
4  */
5 
6 #include "xe_ggtt.h"
7 
8 #include <linux/fault-inject.h>
9 #include <linux/io-64-nonatomic-lo-hi.h>
10 #include <linux/sizes.h>
11 
12 #include <drm/drm_drv.h>
13 #include <drm/drm_managed.h>
14 #include <drm/intel/i915_drm.h>
15 #include <generated/xe_wa_oob.h>
16 
17 #include "regs/xe_gt_regs.h"
18 #include "regs/xe_gtt_defs.h"
19 #include "regs/xe_regs.h"
20 #include "xe_assert.h"
21 #include "xe_bo.h"
22 #include "xe_device.h"
23 #include "xe_gt.h"
24 #include "xe_gt_printk.h"
25 #include "xe_gt_sriov_vf.h"
26 #include "xe_gt_tlb_invalidation.h"
27 #include "xe_map.h"
28 #include "xe_mmio.h"
29 #include "xe_pm.h"
30 #include "xe_sriov.h"
31 #include "xe_wa.h"
32 #include "xe_wopcm.h"
33 
34 /**
35  * DOC: Global Graphics Translation Table (GGTT)
36  *
37  * Xe GGTT implements the support for a Global Virtual Address space that is used
38  * for resources that are accessible to privileged (i.e. kernel-mode) processes,
39  * and not tied to a specific user-level process. For example, the Graphics
40  * micro-Controller (GuC) and Display Engine (if present) utilize this Global
41  * address space.
42  *
43  * The Global GTT (GGTT) translates from the Global virtual address to a physical
44  * address that can be accessed by HW. The GGTT is a flat, single-level table.
45  *
46  * Xe implements a simplified version of the GGTT specifically managing only a
47  * certain range of it that goes from the Write Once Protected Content Memory (WOPCM)
48  * Layout to a predefined GUC_GGTT_TOP. This approach avoids complications related to
49  * the GuC (Graphics Microcontroller) hardware limitations. The GuC address space
50  * is limited on both ends of the GGTT, because the GuC shim HW redirects
51  * accesses to those addresses to other HW areas instead of going through the
52  * GGTT. On the bottom end, the GuC can't access offsets below the WOPCM size,
53  * while on the top side the limit is fixed at GUC_GGTT_TOP. To keep things
54  * simple, instead of checking each object to see if they are accessed by GuC or
55  * not, we just exclude those areas from the allocator. Additionally, to simplify
56  * the driver load, we use the maximum WOPCM size in this logic instead of the
57  * programmed one, so we don't need to wait until the actual size to be
58  * programmed is determined (which requires FW fetch) before initializing the
59  * GGTT. These simplifications might waste space in the GGTT (about 20-25 MBs
60  * depending on the platform) but we can live with this. Another benefit of this
61  * is the GuC bootrom can't access anything below the WOPCM max size so anything
62  * the bootrom needs to access (e.g. a RSA key) needs to be placed in the GGTT
63  * above the WOPCM max size. Starting the GGTT allocations above the WOPCM max
64  * give us the correct placement for free.
65  */
66 
xelp_ggtt_pte_encode_bo(struct xe_bo * bo,u64 bo_offset,u16 pat_index)67 static u64 xelp_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
68 				   u16 pat_index)
69 {
70 	u64 pte;
71 
72 	pte = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE);
73 	pte |= XE_PAGE_PRESENT;
74 
75 	if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo))
76 		pte |= XE_GGTT_PTE_DM;
77 
78 	return pte;
79 }
80 
xelpg_ggtt_pte_encode_bo(struct xe_bo * bo,u64 bo_offset,u16 pat_index)81 static u64 xelpg_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
82 				    u16 pat_index)
83 {
84 	struct xe_device *xe = xe_bo_device(bo);
85 	u64 pte;
86 
87 	pte = xelp_ggtt_pte_encode_bo(bo, bo_offset, pat_index);
88 
89 	xe_assert(xe, pat_index <= 3);
90 
91 	if (pat_index & BIT(0))
92 		pte |= XELPG_GGTT_PTE_PAT0;
93 
94 	if (pat_index & BIT(1))
95 		pte |= XELPG_GGTT_PTE_PAT1;
96 
97 	return pte;
98 }
99 
probe_gsm_size(struct pci_dev * pdev)100 static unsigned int probe_gsm_size(struct pci_dev *pdev)
101 {
102 	u16 gmch_ctl, ggms;
103 
104 	pci_read_config_word(pdev, SNB_GMCH_CTRL, &gmch_ctl);
105 	ggms = (gmch_ctl >> BDW_GMCH_GGMS_SHIFT) & BDW_GMCH_GGMS_MASK;
106 	return ggms ? SZ_1M << ggms : 0;
107 }
108 
ggtt_update_access_counter(struct xe_ggtt * ggtt)109 static void ggtt_update_access_counter(struct xe_ggtt *ggtt)
110 {
111 	struct xe_tile *tile = ggtt->tile;
112 	struct xe_gt *affected_gt = XE_WA(tile->primary_gt, 22019338487) ?
113 		tile->primary_gt : tile->media_gt;
114 	struct xe_mmio *mmio = &affected_gt->mmio;
115 	u32 max_gtt_writes = XE_WA(ggtt->tile->primary_gt, 22019338487) ? 1100 : 63;
116 	/*
117 	 * Wa_22019338487: GMD_ID is a RO register, a dummy write forces gunit
118 	 * to wait for completion of prior GTT writes before letting this through.
119 	 * This needs to be done for all GGTT writes originating from the CPU.
120 	 */
121 	lockdep_assert_held(&ggtt->lock);
122 
123 	if ((++ggtt->access_count % max_gtt_writes) == 0) {
124 		xe_mmio_write32(mmio, GMD_ID, 0x0);
125 		ggtt->access_count = 0;
126 	}
127 }
128 
xe_ggtt_set_pte(struct xe_ggtt * ggtt,u64 addr,u64 pte)129 static void xe_ggtt_set_pte(struct xe_ggtt *ggtt, u64 addr, u64 pte)
130 {
131 	xe_tile_assert(ggtt->tile, !(addr & XE_PTE_MASK));
132 	xe_tile_assert(ggtt->tile, addr < ggtt->size);
133 
134 	writeq(pte, &ggtt->gsm[addr >> XE_PTE_SHIFT]);
135 }
136 
xe_ggtt_set_pte_and_flush(struct xe_ggtt * ggtt,u64 addr,u64 pte)137 static void xe_ggtt_set_pte_and_flush(struct xe_ggtt *ggtt, u64 addr, u64 pte)
138 {
139 	xe_ggtt_set_pte(ggtt, addr, pte);
140 	ggtt_update_access_counter(ggtt);
141 }
142 
xe_ggtt_clear(struct xe_ggtt * ggtt,u64 start,u64 size)143 static void xe_ggtt_clear(struct xe_ggtt *ggtt, u64 start, u64 size)
144 {
145 	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB];
146 	u64 end = start + size - 1;
147 	u64 scratch_pte;
148 
149 	xe_tile_assert(ggtt->tile, start < end);
150 
151 	if (ggtt->scratch)
152 		scratch_pte = ggtt->pt_ops->pte_encode_bo(ggtt->scratch, 0,
153 							  pat_index);
154 	else
155 		scratch_pte = 0;
156 
157 	while (start < end) {
158 		ggtt->pt_ops->ggtt_set_pte(ggtt, start, scratch_pte);
159 		start += XE_PAGE_SIZE;
160 	}
161 }
162 
ggtt_fini_early(struct drm_device * drm,void * arg)163 static void ggtt_fini_early(struct drm_device *drm, void *arg)
164 {
165 	struct xe_ggtt *ggtt = arg;
166 
167 	destroy_workqueue(ggtt->wq);
168 	mutex_destroy(&ggtt->lock);
169 	drm_mm_takedown(&ggtt->mm);
170 }
171 
ggtt_fini(void * arg)172 static void ggtt_fini(void *arg)
173 {
174 	struct xe_ggtt *ggtt = arg;
175 
176 	ggtt->scratch = NULL;
177 }
178 
primelockdep(struct xe_ggtt * ggtt)179 static void primelockdep(struct xe_ggtt *ggtt)
180 {
181 	if (!IS_ENABLED(CONFIG_LOCKDEP))
182 		return;
183 
184 	fs_reclaim_acquire(GFP_KERNEL);
185 	might_lock(&ggtt->lock);
186 	fs_reclaim_release(GFP_KERNEL);
187 }
188 
189 static const struct xe_ggtt_pt_ops xelp_pt_ops = {
190 	.pte_encode_bo = xelp_ggtt_pte_encode_bo,
191 	.ggtt_set_pte = xe_ggtt_set_pte,
192 };
193 
194 static const struct xe_ggtt_pt_ops xelpg_pt_ops = {
195 	.pte_encode_bo = xelpg_ggtt_pte_encode_bo,
196 	.ggtt_set_pte = xe_ggtt_set_pte,
197 };
198 
199 static const struct xe_ggtt_pt_ops xelpg_pt_wa_ops = {
200 	.pte_encode_bo = xelpg_ggtt_pte_encode_bo,
201 	.ggtt_set_pte = xe_ggtt_set_pte_and_flush,
202 };
203 
204 /**
205  * xe_ggtt_init_early - Early GGTT initialization
206  * @ggtt: the &xe_ggtt to be initialized
207  *
208  * It allows to create new mappings usable by the GuC.
209  * Mappings are not usable by the HW engines, as it doesn't have scratch nor
210  * initial clear done to it yet. That will happen in the regular, non-early
211  * GGTT initialization.
212  *
213  * Return: 0 on success or a negative error code on failure.
214  */
xe_ggtt_init_early(struct xe_ggtt * ggtt)215 int xe_ggtt_init_early(struct xe_ggtt *ggtt)
216 {
217 	struct xe_device *xe = tile_to_xe(ggtt->tile);
218 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
219 	unsigned int gsm_size;
220 	int err;
221 
222 	if (IS_SRIOV_VF(xe))
223 		gsm_size = SZ_8M; /* GGTT is expected to be 4GiB */
224 	else
225 		gsm_size = probe_gsm_size(pdev);
226 
227 	if (gsm_size == 0) {
228 		drm_err(&xe->drm, "Hardware reported no preallocated GSM\n");
229 		return -ENOMEM;
230 	}
231 
232 	ggtt->gsm = ggtt->tile->mmio.regs + SZ_8M;
233 	ggtt->size = (gsm_size / 8) * (u64) XE_PAGE_SIZE;
234 
235 	if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)
236 		ggtt->flags |= XE_GGTT_FLAGS_64K;
237 
238 	if (ggtt->size > GUC_GGTT_TOP)
239 		ggtt->size = GUC_GGTT_TOP;
240 
241 	if (GRAPHICS_VERx100(xe) >= 1270)
242 		ggtt->pt_ops = (ggtt->tile->media_gt &&
243 			       XE_WA(ggtt->tile->media_gt, 22019338487)) ||
244 			       XE_WA(ggtt->tile->primary_gt, 22019338487) ?
245 			       &xelpg_pt_wa_ops : &xelpg_pt_ops;
246 	else
247 		ggtt->pt_ops = &xelp_pt_ops;
248 
249 	ggtt->wq = alloc_workqueue("xe-ggtt-wq", 0, WQ_MEM_RECLAIM);
250 
251 	drm_mm_init(&ggtt->mm, xe_wopcm_size(xe),
252 		    ggtt->size - xe_wopcm_size(xe));
253 	mutex_init(&ggtt->lock);
254 	primelockdep(ggtt);
255 
256 	err = drmm_add_action_or_reset(&xe->drm, ggtt_fini_early, ggtt);
257 	if (err)
258 		return err;
259 
260 	if (IS_SRIOV_VF(xe)) {
261 		err = xe_gt_sriov_vf_prepare_ggtt(xe_tile_get_gt(ggtt->tile, 0));
262 		if (err)
263 			return err;
264 	}
265 
266 	return 0;
267 }
268 ALLOW_ERROR_INJECTION(xe_ggtt_init_early, ERRNO); /* See xe_pci_probe() */
269 
270 static void xe_ggtt_invalidate(struct xe_ggtt *ggtt);
271 
xe_ggtt_initial_clear(struct xe_ggtt * ggtt)272 static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt)
273 {
274 	struct drm_mm_node *hole;
275 	u64 start, end;
276 
277 	/* Display may have allocated inside ggtt, so be careful with clearing here */
278 	mutex_lock(&ggtt->lock);
279 	drm_mm_for_each_hole(hole, &ggtt->mm, start, end)
280 		xe_ggtt_clear(ggtt, start, end - start);
281 
282 	xe_ggtt_invalidate(ggtt);
283 	mutex_unlock(&ggtt->lock);
284 }
285 
ggtt_node_remove(struct xe_ggtt_node * node)286 static void ggtt_node_remove(struct xe_ggtt_node *node)
287 {
288 	struct xe_ggtt *ggtt = node->ggtt;
289 	struct xe_device *xe = tile_to_xe(ggtt->tile);
290 	bool bound;
291 	int idx;
292 
293 	bound = drm_dev_enter(&xe->drm, &idx);
294 
295 	mutex_lock(&ggtt->lock);
296 	if (bound)
297 		xe_ggtt_clear(ggtt, node->base.start, node->base.size);
298 	drm_mm_remove_node(&node->base);
299 	node->base.size = 0;
300 	mutex_unlock(&ggtt->lock);
301 
302 	if (!bound)
303 		goto free_node;
304 
305 	if (node->invalidate_on_remove)
306 		xe_ggtt_invalidate(ggtt);
307 
308 	drm_dev_exit(idx);
309 
310 free_node:
311 	xe_ggtt_node_fini(node);
312 }
313 
ggtt_node_remove_work_func(struct work_struct * work)314 static void ggtt_node_remove_work_func(struct work_struct *work)
315 {
316 	struct xe_ggtt_node *node = container_of(work, typeof(*node),
317 						 delayed_removal_work);
318 	struct xe_device *xe = tile_to_xe(node->ggtt->tile);
319 
320 	xe_pm_runtime_get(xe);
321 	ggtt_node_remove(node);
322 	xe_pm_runtime_put(xe);
323 }
324 
325 /**
326  * xe_ggtt_node_remove - Remove a &xe_ggtt_node from the GGTT
327  * @node: the &xe_ggtt_node to be removed
328  * @invalidate: if node needs invalidation upon removal
329  */
xe_ggtt_node_remove(struct xe_ggtt_node * node,bool invalidate)330 void xe_ggtt_node_remove(struct xe_ggtt_node *node, bool invalidate)
331 {
332 	struct xe_ggtt *ggtt;
333 	struct xe_device *xe;
334 
335 	if (!node || !node->ggtt)
336 		return;
337 
338 	ggtt = node->ggtt;
339 	xe = tile_to_xe(ggtt->tile);
340 
341 	node->invalidate_on_remove = invalidate;
342 
343 	if (xe_pm_runtime_get_if_active(xe)) {
344 		ggtt_node_remove(node);
345 		xe_pm_runtime_put(xe);
346 	} else {
347 		queue_work(ggtt->wq, &node->delayed_removal_work);
348 	}
349 }
350 
351 /**
352  * xe_ggtt_init - Regular non-early GGTT initialization
353  * @ggtt: the &xe_ggtt to be initialized
354  *
355  * Return: 0 on success or a negative error code on failure.
356  */
xe_ggtt_init(struct xe_ggtt * ggtt)357 int xe_ggtt_init(struct xe_ggtt *ggtt)
358 {
359 	struct xe_device *xe = tile_to_xe(ggtt->tile);
360 	unsigned int flags;
361 	int err;
362 
363 	/*
364 	 * So we don't need to worry about 64K GGTT layout when dealing with
365 	 * scratch entires, rather keep the scratch page in system memory on
366 	 * platforms where 64K pages are needed for VRAM.
367 	 */
368 	flags = XE_BO_FLAG_PINNED;
369 	if (ggtt->flags & XE_GGTT_FLAGS_64K)
370 		flags |= XE_BO_FLAG_SYSTEM;
371 	else
372 		flags |= XE_BO_FLAG_VRAM_IF_DGFX(ggtt->tile);
373 
374 	ggtt->scratch = xe_managed_bo_create_pin_map(xe, ggtt->tile, XE_PAGE_SIZE, flags);
375 	if (IS_ERR(ggtt->scratch)) {
376 		err = PTR_ERR(ggtt->scratch);
377 		goto err;
378 	}
379 
380 	xe_map_memset(xe, &ggtt->scratch->vmap, 0, 0, ggtt->scratch->size);
381 
382 	xe_ggtt_initial_clear(ggtt);
383 
384 	return devm_add_action_or_reset(xe->drm.dev, ggtt_fini, ggtt);
385 err:
386 	ggtt->scratch = NULL;
387 	return err;
388 }
389 
ggtt_invalidate_gt_tlb(struct xe_gt * gt)390 static void ggtt_invalidate_gt_tlb(struct xe_gt *gt)
391 {
392 	int err;
393 
394 	if (!gt)
395 		return;
396 
397 	err = xe_gt_tlb_invalidation_ggtt(gt);
398 	if (err)
399 		drm_warn(&gt_to_xe(gt)->drm, "xe_gt_tlb_invalidation_ggtt error=%d", err);
400 }
401 
xe_ggtt_invalidate(struct xe_ggtt * ggtt)402 static void xe_ggtt_invalidate(struct xe_ggtt *ggtt)
403 {
404 	struct xe_device *xe = tile_to_xe(ggtt->tile);
405 
406 	/*
407 	 * XXX: Barrier for GGTT pages. Unsure exactly why this required but
408 	 * without this LNL is having issues with the GuC reading scratch page
409 	 * vs. correct GGTT page. Not particularly a hot code path so blindly
410 	 * do a mmio read here which results in GuC reading correct GGTT page.
411 	 */
412 	xe_mmio_read32(xe_root_tile_mmio(xe), VF_CAP_REG);
413 
414 	/* Each GT in a tile has its own TLB to cache GGTT lookups */
415 	ggtt_invalidate_gt_tlb(ggtt->tile->primary_gt);
416 	ggtt_invalidate_gt_tlb(ggtt->tile->media_gt);
417 }
418 
xe_ggtt_dump_node(struct xe_ggtt * ggtt,const struct drm_mm_node * node,const char * description)419 static void xe_ggtt_dump_node(struct xe_ggtt *ggtt,
420 			      const struct drm_mm_node *node, const char *description)
421 {
422 	char buf[10];
423 
424 	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
425 		string_get_size(node->size, 1, STRING_UNITS_2, buf, sizeof(buf));
426 		xe_gt_dbg(ggtt->tile->primary_gt, "GGTT %#llx-%#llx (%s) %s\n",
427 			  node->start, node->start + node->size, buf, description);
428 	}
429 }
430 
431 /**
432  * xe_ggtt_node_insert_balloon - prevent allocation of specified GGTT addresses
433  * @node: the &xe_ggtt_node to hold reserved GGTT node
434  * @start: the starting GGTT address of the reserved region
435  * @end: then end GGTT address of the reserved region
436  *
437  * Use xe_ggtt_node_remove_balloon() to release a reserved GGTT node.
438  *
439  * Return: 0 on success or a negative error code on failure.
440  */
xe_ggtt_node_insert_balloon(struct xe_ggtt_node * node,u64 start,u64 end)441 int xe_ggtt_node_insert_balloon(struct xe_ggtt_node *node, u64 start, u64 end)
442 {
443 	struct xe_ggtt *ggtt = node->ggtt;
444 	int err;
445 
446 	xe_tile_assert(ggtt->tile, start < end);
447 	xe_tile_assert(ggtt->tile, IS_ALIGNED(start, XE_PAGE_SIZE));
448 	xe_tile_assert(ggtt->tile, IS_ALIGNED(end, XE_PAGE_SIZE));
449 	xe_tile_assert(ggtt->tile, !drm_mm_node_allocated(&node->base));
450 
451 	node->base.color = 0;
452 	node->base.start = start;
453 	node->base.size = end - start;
454 
455 	mutex_lock(&ggtt->lock);
456 	err = drm_mm_reserve_node(&ggtt->mm, &node->base);
457 	mutex_unlock(&ggtt->lock);
458 
459 	if (xe_gt_WARN(ggtt->tile->primary_gt, err,
460 		       "Failed to balloon GGTT %#llx-%#llx (%pe)\n",
461 		       node->base.start, node->base.start + node->base.size, ERR_PTR(err)))
462 		return err;
463 
464 	xe_ggtt_dump_node(ggtt, &node->base, "balloon");
465 	return 0;
466 }
467 
468 /**
469  * xe_ggtt_node_remove_balloon - release a reserved GGTT region
470  * @node: the &xe_ggtt_node with reserved GGTT region
471  *
472  * See xe_ggtt_node_insert_balloon() for details.
473  */
xe_ggtt_node_remove_balloon(struct xe_ggtt_node * node)474 void xe_ggtt_node_remove_balloon(struct xe_ggtt_node *node)
475 {
476 	if (!node || !node->ggtt)
477 		return;
478 
479 	if (!drm_mm_node_allocated(&node->base))
480 		goto free_node;
481 
482 	xe_ggtt_dump_node(node->ggtt, &node->base, "remove-balloon");
483 
484 	mutex_lock(&node->ggtt->lock);
485 	drm_mm_remove_node(&node->base);
486 	mutex_unlock(&node->ggtt->lock);
487 
488 free_node:
489 	xe_ggtt_node_fini(node);
490 }
491 
492 /**
493  * xe_ggtt_node_insert_locked - Locked version to insert a &xe_ggtt_node into the GGTT
494  * @node: the &xe_ggtt_node to be inserted
495  * @size: size of the node
496  * @align: alignment constrain of the node
497  * @mm_flags: flags to control the node behavior
498  *
499  * It cannot be called without first having called xe_ggtt_init() once.
500  * To be used in cases where ggtt->lock is already taken.
501  *
502  * Return: 0 on success or a negative error code on failure.
503  */
xe_ggtt_node_insert_locked(struct xe_ggtt_node * node,u32 size,u32 align,u32 mm_flags)504 int xe_ggtt_node_insert_locked(struct xe_ggtt_node *node,
505 			       u32 size, u32 align, u32 mm_flags)
506 {
507 	return drm_mm_insert_node_generic(&node->ggtt->mm, &node->base, size, align, 0,
508 					  mm_flags);
509 }
510 
511 /**
512  * xe_ggtt_node_insert - Insert a &xe_ggtt_node into the GGTT
513  * @node: the &xe_ggtt_node to be inserted
514  * @size: size of the node
515  * @align: alignment constrain of the node
516  *
517  * It cannot be called without first having called xe_ggtt_init() once.
518  *
519  * Return: 0 on success or a negative error code on failure.
520  */
xe_ggtt_node_insert(struct xe_ggtt_node * node,u32 size,u32 align)521 int xe_ggtt_node_insert(struct xe_ggtt_node *node, u32 size, u32 align)
522 {
523 	int ret;
524 
525 	if (!node || !node->ggtt)
526 		return -ENOENT;
527 
528 	mutex_lock(&node->ggtt->lock);
529 	ret = xe_ggtt_node_insert_locked(node, size, align,
530 					 DRM_MM_INSERT_HIGH);
531 	mutex_unlock(&node->ggtt->lock);
532 
533 	return ret;
534 }
535 
536 /**
537  * xe_ggtt_node_init - Initialize %xe_ggtt_node struct
538  * @ggtt: the &xe_ggtt where the new node will later be inserted/reserved.
539  *
540  * This function will allocated the struct %xe_ggtt_node and return it's pointer.
541  * This struct will then be freed after the node removal upon xe_ggtt_node_remove()
542  * or xe_ggtt_node_remove_balloon().
543  * Having %xe_ggtt_node struct allocated doesn't mean that the node is already allocated
544  * in GGTT. Only the xe_ggtt_node_insert(), xe_ggtt_node_insert_locked(),
545  * xe_ggtt_node_insert_balloon() will ensure the node is inserted or reserved in GGTT.
546  *
547  * Return: A pointer to %xe_ggtt_node struct on success. An ERR_PTR otherwise.
548  **/
xe_ggtt_node_init(struct xe_ggtt * ggtt)549 struct xe_ggtt_node *xe_ggtt_node_init(struct xe_ggtt *ggtt)
550 {
551 	struct xe_ggtt_node *node = kzalloc(sizeof(*node), GFP_NOFS);
552 
553 	if (!node)
554 		return ERR_PTR(-ENOMEM);
555 
556 	INIT_WORK(&node->delayed_removal_work, ggtt_node_remove_work_func);
557 	node->ggtt = ggtt;
558 
559 	return node;
560 }
561 
562 /**
563  * xe_ggtt_node_fini - Forcebly finalize %xe_ggtt_node struct
564  * @node: the &xe_ggtt_node to be freed
565  *
566  * If anything went wrong with either xe_ggtt_node_insert(), xe_ggtt_node_insert_locked(),
567  * or xe_ggtt_node_insert_balloon(); and this @node is not going to be reused, then,
568  * this function needs to be called to free the %xe_ggtt_node struct
569  **/
xe_ggtt_node_fini(struct xe_ggtt_node * node)570 void xe_ggtt_node_fini(struct xe_ggtt_node *node)
571 {
572 	kfree(node);
573 }
574 
575 /**
576  * xe_ggtt_node_allocated - Check if node is allocated in GGTT
577  * @node: the &xe_ggtt_node to be inspected
578  *
579  * Return: True if allocated, False otherwise.
580  */
xe_ggtt_node_allocated(const struct xe_ggtt_node * node)581 bool xe_ggtt_node_allocated(const struct xe_ggtt_node *node)
582 {
583 	if (!node || !node->ggtt)
584 		return false;
585 
586 	return drm_mm_node_allocated(&node->base);
587 }
588 
589 /**
590  * xe_ggtt_map_bo - Map the BO into GGTT
591  * @ggtt: the &xe_ggtt where node will be mapped
592  * @bo: the &xe_bo to be mapped
593  */
xe_ggtt_map_bo(struct xe_ggtt * ggtt,struct xe_bo * bo)594 void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
595 {
596 	u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB;
597 	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[cache_mode];
598 	u64 start;
599 	u64 offset, pte;
600 
601 	if (XE_WARN_ON(!bo->ggtt_node))
602 		return;
603 
604 	start = bo->ggtt_node->base.start;
605 
606 	for (offset = 0; offset < bo->size; offset += XE_PAGE_SIZE) {
607 		pte = ggtt->pt_ops->pte_encode_bo(bo, offset, pat_index);
608 		ggtt->pt_ops->ggtt_set_pte(ggtt, start + offset, pte);
609 	}
610 }
611 
__xe_ggtt_insert_bo_at(struct xe_ggtt * ggtt,struct xe_bo * bo,u64 start,u64 end)612 static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
613 				  u64 start, u64 end)
614 {
615 	int err;
616 	u64 alignment = bo->min_align > 0 ? bo->min_align : XE_PAGE_SIZE;
617 
618 	if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)
619 		alignment = SZ_64K;
620 
621 	if (XE_WARN_ON(bo->ggtt_node)) {
622 		/* Someone's already inserted this BO in the GGTT */
623 		xe_tile_assert(ggtt->tile, bo->ggtt_node->base.size == bo->size);
624 		return 0;
625 	}
626 
627 	err = xe_bo_validate(bo, NULL, false);
628 	if (err)
629 		return err;
630 
631 	xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile));
632 
633 	bo->ggtt_node = xe_ggtt_node_init(ggtt);
634 	if (IS_ERR(bo->ggtt_node)) {
635 		err = PTR_ERR(bo->ggtt_node);
636 		bo->ggtt_node = NULL;
637 		goto out;
638 	}
639 
640 	mutex_lock(&ggtt->lock);
641 	err = drm_mm_insert_node_in_range(&ggtt->mm, &bo->ggtt_node->base, bo->size,
642 					  alignment, 0, start, end, 0);
643 	if (err) {
644 		xe_ggtt_node_fini(bo->ggtt_node);
645 		bo->ggtt_node = NULL;
646 	} else {
647 		xe_ggtt_map_bo(ggtt, bo);
648 	}
649 	mutex_unlock(&ggtt->lock);
650 
651 	if (!err && bo->flags & XE_BO_FLAG_GGTT_INVALIDATE)
652 		xe_ggtt_invalidate(ggtt);
653 
654 out:
655 	xe_pm_runtime_put(tile_to_xe(ggtt->tile));
656 
657 	return err;
658 }
659 
660 /**
661  * xe_ggtt_insert_bo_at - Insert BO at a specific GGTT space
662  * @ggtt: the &xe_ggtt where bo will be inserted
663  * @bo: the &xe_bo to be inserted
664  * @start: address where it will be inserted
665  * @end: end of the range where it will be inserted
666  *
667  * Return: 0 on success or a negative error code on failure.
668  */
xe_ggtt_insert_bo_at(struct xe_ggtt * ggtt,struct xe_bo * bo,u64 start,u64 end)669 int xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
670 			 u64 start, u64 end)
671 {
672 	return __xe_ggtt_insert_bo_at(ggtt, bo, start, end);
673 }
674 
675 /**
676  * xe_ggtt_insert_bo - Insert BO into GGTT
677  * @ggtt: the &xe_ggtt where bo will be inserted
678  * @bo: the &xe_bo to be inserted
679  *
680  * Return: 0 on success or a negative error code on failure.
681  */
xe_ggtt_insert_bo(struct xe_ggtt * ggtt,struct xe_bo * bo)682 int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
683 {
684 	return __xe_ggtt_insert_bo_at(ggtt, bo, 0, U64_MAX);
685 }
686 
687 /**
688  * xe_ggtt_remove_bo - Remove a BO from the GGTT
689  * @ggtt: the &xe_ggtt where node will be removed
690  * @bo: the &xe_bo to be removed
691  */
xe_ggtt_remove_bo(struct xe_ggtt * ggtt,struct xe_bo * bo)692 void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
693 {
694 	if (XE_WARN_ON(!bo->ggtt_node))
695 		return;
696 
697 	/* This BO is not currently in the GGTT */
698 	xe_tile_assert(ggtt->tile, bo->ggtt_node->base.size == bo->size);
699 
700 	xe_ggtt_node_remove(bo->ggtt_node,
701 			    bo->flags & XE_BO_FLAG_GGTT_INVALIDATE);
702 }
703 
704 /**
705  * xe_ggtt_largest_hole - Largest GGTT hole
706  * @ggtt: the &xe_ggtt that will be inspected
707  * @alignment: minimum alignment
708  * @spare: If not NULL: in: desired memory size to be spared / out: Adjusted possible spare
709  *
710  * Return: size of the largest continuous GGTT region
711  */
xe_ggtt_largest_hole(struct xe_ggtt * ggtt,u64 alignment,u64 * spare)712 u64 xe_ggtt_largest_hole(struct xe_ggtt *ggtt, u64 alignment, u64 *spare)
713 {
714 	const struct drm_mm *mm = &ggtt->mm;
715 	const struct drm_mm_node *entry;
716 	u64 hole_min_start = xe_wopcm_size(tile_to_xe(ggtt->tile));
717 	u64 hole_start, hole_end, hole_size;
718 	u64 max_hole = 0;
719 
720 	mutex_lock(&ggtt->lock);
721 
722 	drm_mm_for_each_hole(entry, mm, hole_start, hole_end) {
723 		hole_start = max(hole_start, hole_min_start);
724 		hole_start = ALIGN(hole_start, alignment);
725 		hole_end = ALIGN_DOWN(hole_end, alignment);
726 		if (hole_start >= hole_end)
727 			continue;
728 		hole_size = hole_end - hole_start;
729 		if (spare)
730 			*spare -= min3(*spare, hole_size, max_hole);
731 		max_hole = max(max_hole, hole_size);
732 	}
733 
734 	mutex_unlock(&ggtt->lock);
735 
736 	return max_hole;
737 }
738 
739 #ifdef CONFIG_PCI_IOV
xe_encode_vfid_pte(u16 vfid)740 static u64 xe_encode_vfid_pte(u16 vfid)
741 {
742 	return FIELD_PREP(GGTT_PTE_VFID, vfid) | XE_PAGE_PRESENT;
743 }
744 
xe_ggtt_assign_locked(struct xe_ggtt * ggtt,const struct drm_mm_node * node,u16 vfid)745 static void xe_ggtt_assign_locked(struct xe_ggtt *ggtt, const struct drm_mm_node *node, u16 vfid)
746 {
747 	u64 start = node->start;
748 	u64 size = node->size;
749 	u64 end = start + size - 1;
750 	u64 pte = xe_encode_vfid_pte(vfid);
751 
752 	lockdep_assert_held(&ggtt->lock);
753 
754 	if (!drm_mm_node_allocated(node))
755 		return;
756 
757 	while (start < end) {
758 		ggtt->pt_ops->ggtt_set_pte(ggtt, start, pte);
759 		start += XE_PAGE_SIZE;
760 	}
761 
762 	xe_ggtt_invalidate(ggtt);
763 }
764 
765 /**
766  * xe_ggtt_assign - assign a GGTT region to the VF
767  * @node: the &xe_ggtt_node to update
768  * @vfid: the VF identifier
769  *
770  * This function is used by the PF driver to assign a GGTT region to the VF.
771  * In addition to PTE's VFID bits 11:2 also PRESENT bit 0 is set as on some
772  * platforms VFs can't modify that either.
773  */
xe_ggtt_assign(const struct xe_ggtt_node * node,u16 vfid)774 void xe_ggtt_assign(const struct xe_ggtt_node *node, u16 vfid)
775 {
776 	mutex_lock(&node->ggtt->lock);
777 	xe_ggtt_assign_locked(node->ggtt, &node->base, vfid);
778 	mutex_unlock(&node->ggtt->lock);
779 }
780 #endif
781 
782 /**
783  * xe_ggtt_dump - Dump GGTT for debug
784  * @ggtt: the &xe_ggtt to be dumped
785  * @p: the &drm_mm_printer helper handle to be used to dump the information
786  *
787  * Return: 0 on success or a negative error code on failure.
788  */
xe_ggtt_dump(struct xe_ggtt * ggtt,struct drm_printer * p)789 int xe_ggtt_dump(struct xe_ggtt *ggtt, struct drm_printer *p)
790 {
791 	int err;
792 
793 	err = mutex_lock_interruptible(&ggtt->lock);
794 	if (err)
795 		return err;
796 
797 	drm_mm_print(&ggtt->mm, p);
798 	mutex_unlock(&ggtt->lock);
799 	return err;
800 }
801 
802 /**
803  * xe_ggtt_print_holes - Print holes
804  * @ggtt: the &xe_ggtt to be inspected
805  * @alignment: min alignment
806  * @p: the &drm_printer
807  *
808  * Print GGTT ranges that are available and return total size available.
809  *
810  * Return: Total available size.
811  */
xe_ggtt_print_holes(struct xe_ggtt * ggtt,u64 alignment,struct drm_printer * p)812 u64 xe_ggtt_print_holes(struct xe_ggtt *ggtt, u64 alignment, struct drm_printer *p)
813 {
814 	const struct drm_mm *mm = &ggtt->mm;
815 	const struct drm_mm_node *entry;
816 	u64 hole_min_start = xe_wopcm_size(tile_to_xe(ggtt->tile));
817 	u64 hole_start, hole_end, hole_size;
818 	u64 total = 0;
819 	char buf[10];
820 
821 	mutex_lock(&ggtt->lock);
822 
823 	drm_mm_for_each_hole(entry, mm, hole_start, hole_end) {
824 		hole_start = max(hole_start, hole_min_start);
825 		hole_start = ALIGN(hole_start, alignment);
826 		hole_end = ALIGN_DOWN(hole_end, alignment);
827 		if (hole_start >= hole_end)
828 			continue;
829 		hole_size = hole_end - hole_start;
830 		total += hole_size;
831 
832 		string_get_size(hole_size, 1, STRING_UNITS_2, buf, sizeof(buf));
833 		drm_printf(p, "range:\t%#llx-%#llx\t(%s)\n",
834 			   hole_start, hole_end - 1, buf);
835 	}
836 
837 	mutex_unlock(&ggtt->lock);
838 
839 	return total;
840 }
841