xref: /linux/drivers/gpu/drm/xe/xe_ggtt.c (revision 78f608d7aff05c245bf0aab00ce7273a7d9f04b9) 
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2021 Intel Corporation
4  */
5 
6 #include "xe_ggtt.h"
7 
8 #include <linux/io-64-nonatomic-lo-hi.h>
9 #include <linux/sizes.h>
10 
11 #include <drm/drm_drv.h>
12 #include <drm/drm_managed.h>
13 #include <drm/intel/i915_drm.h>
14 
15 #include "regs/xe_gt_regs.h"
16 #include "regs/xe_gtt_defs.h"
17 #include "regs/xe_regs.h"
18 #include "xe_assert.h"
19 #include "xe_bo.h"
20 #include "xe_device.h"
21 #include "xe_gt.h"
22 #include "xe_gt_printk.h"
23 #include "xe_gt_sriov_vf.h"
24 #include "xe_gt_tlb_invalidation.h"
25 #include "xe_map.h"
26 #include "xe_pm.h"
27 #include "xe_sriov.h"
28 #include "xe_wopcm.h"
29 
30 static u64 xelp_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
31 				   u16 pat_index)
32 {
33 	u64 pte;
34 
35 	pte = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE);
36 	pte |= XE_PAGE_PRESENT;
37 
38 	if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo))
39 		pte |= XE_GGTT_PTE_DM;
40 
41 	return pte;
42 }
43 
44 static u64 xelpg_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
45 				    u16 pat_index)
46 {
47 	struct xe_device *xe = xe_bo_device(bo);
48 	u64 pte;
49 
50 	pte = xelp_ggtt_pte_encode_bo(bo, bo_offset, pat_index);
51 
52 	xe_assert(xe, pat_index <= 3);
53 
54 	if (pat_index & BIT(0))
55 		pte |= XELPG_GGTT_PTE_PAT0;
56 
57 	if (pat_index & BIT(1))
58 		pte |= XELPG_GGTT_PTE_PAT1;
59 
60 	return pte;
61 }
62 
63 static unsigned int probe_gsm_size(struct pci_dev *pdev)
64 {
65 	u16 gmch_ctl, ggms;
66 
67 	pci_read_config_word(pdev, SNB_GMCH_CTRL, &gmch_ctl);
68 	ggms = (gmch_ctl >> BDW_GMCH_GGMS_SHIFT) & BDW_GMCH_GGMS_MASK;
69 	return ggms ? SZ_1M << ggms : 0;
70 }
71 
72 void xe_ggtt_set_pte(struct xe_ggtt *ggtt, u64 addr, u64 pte)
73 {
74 	xe_tile_assert(ggtt->tile, !(addr & XE_PTE_MASK));
75 	xe_tile_assert(ggtt->tile, addr < ggtt->size);
76 
77 	writeq(pte, &ggtt->gsm[addr >> XE_PTE_SHIFT]);
78 }
79 
80 static void xe_ggtt_clear(struct xe_ggtt *ggtt, u64 start, u64 size)
81 {
82 	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB];
83 	u64 end = start + size - 1;
84 	u64 scratch_pte;
85 
86 	xe_tile_assert(ggtt->tile, start < end);
87 
88 	if (ggtt->scratch)
89 		scratch_pte = ggtt->pt_ops->pte_encode_bo(ggtt->scratch, 0,
90 							  pat_index);
91 	else
92 		scratch_pte = 0;
93 
94 	while (start < end) {
95 		xe_ggtt_set_pte(ggtt, start, scratch_pte);
96 		start += XE_PAGE_SIZE;
97 	}
98 }
99 
100 static void ggtt_fini_early(struct drm_device *drm, void *arg)
101 {
102 	struct xe_ggtt *ggtt = arg;
103 
104 	mutex_destroy(&ggtt->lock);
105 	drm_mm_takedown(&ggtt->mm);
106 }
107 
108 static void ggtt_fini(struct drm_device *drm, void *arg)
109 {
110 	struct xe_ggtt *ggtt = arg;
111 
112 	ggtt->scratch = NULL;
113 }
114 
115 static void primelockdep(struct xe_ggtt *ggtt)
116 {
117 	if (!IS_ENABLED(CONFIG_LOCKDEP))
118 		return;
119 
120 	fs_reclaim_acquire(GFP_KERNEL);
121 	might_lock(&ggtt->lock);
122 	fs_reclaim_release(GFP_KERNEL);
123 }
124 
125 static const struct xe_ggtt_pt_ops xelp_pt_ops = {
126 	.pte_encode_bo = xelp_ggtt_pte_encode_bo,
127 };
128 
129 static const struct xe_ggtt_pt_ops xelpg_pt_ops = {
130 	.pte_encode_bo = xelpg_ggtt_pte_encode_bo,
131 };
132 
133 /*
134  * Early GGTT initialization, which allows to create new mappings usable by the
135  * GuC.
136  * Mappings are not usable by the HW engines, as it doesn't have scratch /
137  * initial clear done to it yet. That will happen in the regular, non-early
138  * GGTT init.
139  */
140 int xe_ggtt_init_early(struct xe_ggtt *ggtt)
141 {
142 	struct xe_device *xe = tile_to_xe(ggtt->tile);
143 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
144 	unsigned int gsm_size;
145 	int err;
146 
147 	if (IS_SRIOV_VF(xe))
148 		gsm_size = SZ_8M; /* GGTT is expected to be 4GiB */
149 	else
150 		gsm_size = probe_gsm_size(pdev);
151 
152 	if (gsm_size == 0) {
153 		drm_err(&xe->drm, "Hardware reported no preallocated GSM\n");
154 		return -ENOMEM;
155 	}
156 
157 	ggtt->gsm = ggtt->tile->mmio.regs + SZ_8M;
158 	ggtt->size = (gsm_size / 8) * (u64) XE_PAGE_SIZE;
159 
160 	if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)
161 		ggtt->flags |= XE_GGTT_FLAGS_64K;
162 
163 	/*
164 	 * 8B per entry, each points to a 4KB page.
165 	 *
166 	 * The GuC address space is limited on both ends of the GGTT, because
167 	 * the GuC shim HW redirects accesses to those addresses to other HW
168 	 * areas instead of going through the GGTT. On the bottom end, the GuC
169 	 * can't access offsets below the WOPCM size, while on the top side the
170 	 * limit is fixed at GUC_GGTT_TOP. To keep things simple, instead of
171 	 * checking each object to see if they are accessed by GuC or not, we
172 	 * just exclude those areas from the allocator. Additionally, to
173 	 * simplify the driver load, we use the maximum WOPCM size in this logic
174 	 * instead of the programmed one, so we don't need to wait until the
175 	 * actual size to be programmed is determined (which requires FW fetch)
176 	 * before initializing the GGTT. These simplifications might waste space
177 	 * in the GGTT (about 20-25 MBs depending on the platform) but we can
178 	 * live with this.
179 	 *
180 	 * Another benifit of this is the GuC bootrom can't access anything
181 	 * below the WOPCM max size so anything the bootom needs to access (e.g.
182 	 * a RSA key) needs to be placed in the GGTT above the WOPCM max size.
183 	 * Starting the GGTT allocations above the WOPCM max give us the correct
184 	 * placement for free.
185 	 */
186 	if (ggtt->size > GUC_GGTT_TOP)
187 		ggtt->size = GUC_GGTT_TOP;
188 
189 	if (GRAPHICS_VERx100(xe) >= 1270)
190 		ggtt->pt_ops = &xelpg_pt_ops;
191 	else
192 		ggtt->pt_ops = &xelp_pt_ops;
193 
194 	drm_mm_init(&ggtt->mm, xe_wopcm_size(xe),
195 		    ggtt->size - xe_wopcm_size(xe));
196 	mutex_init(&ggtt->lock);
197 	primelockdep(ggtt);
198 
199 	err = drmm_add_action_or_reset(&xe->drm, ggtt_fini_early, ggtt);
200 	if (err)
201 		return err;
202 
203 	if (IS_SRIOV_VF(xe)) {
204 		err = xe_gt_sriov_vf_prepare_ggtt(xe_tile_get_gt(ggtt->tile, 0));
205 		if (err)
206 			return err;
207 	}
208 
209 	return 0;
210 }
211 
212 static void xe_ggtt_invalidate(struct xe_ggtt *ggtt);
213 
214 static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt)
215 {
216 	struct drm_mm_node *hole;
217 	u64 start, end;
218 
219 	/* Display may have allocated inside ggtt, so be careful with clearing here */
220 	mutex_lock(&ggtt->lock);
221 	drm_mm_for_each_hole(hole, &ggtt->mm, start, end)
222 		xe_ggtt_clear(ggtt, start, end - start);
223 
224 	xe_ggtt_invalidate(ggtt);
225 	mutex_unlock(&ggtt->lock);
226 }
227 
228 int xe_ggtt_init(struct xe_ggtt *ggtt)
229 {
230 	struct xe_device *xe = tile_to_xe(ggtt->tile);
231 	unsigned int flags;
232 	int err;
233 
234 	/*
235 	 * So we don't need to worry about 64K GGTT layout when dealing with
236 	 * scratch entires, rather keep the scratch page in system memory on
237 	 * platforms where 64K pages are needed for VRAM.
238 	 */
239 	flags = XE_BO_FLAG_PINNED;
240 	if (ggtt->flags & XE_GGTT_FLAGS_64K)
241 		flags |= XE_BO_FLAG_SYSTEM;
242 	else
243 		flags |= XE_BO_FLAG_VRAM_IF_DGFX(ggtt->tile);
244 
245 	ggtt->scratch = xe_managed_bo_create_pin_map(xe, ggtt->tile, XE_PAGE_SIZE, flags);
246 	if (IS_ERR(ggtt->scratch)) {
247 		err = PTR_ERR(ggtt->scratch);
248 		goto err;
249 	}
250 
251 	xe_map_memset(xe, &ggtt->scratch->vmap, 0, 0, ggtt->scratch->size);
252 
253 	xe_ggtt_initial_clear(ggtt);
254 
255 	return drmm_add_action_or_reset(&xe->drm, ggtt_fini, ggtt);
256 err:
257 	ggtt->scratch = NULL;
258 	return err;
259 }
260 
261 static void ggtt_invalidate_gt_tlb(struct xe_gt *gt)
262 {
263 	int err;
264 
265 	if (!gt)
266 		return;
267 
268 	err = xe_gt_tlb_invalidation_ggtt(gt);
269 	if (err)
270 		drm_warn(&gt_to_xe(gt)->drm, "xe_gt_tlb_invalidation_ggtt error=%d", err);
271 }
272 
273 static void xe_ggtt_invalidate(struct xe_ggtt *ggtt)
274 {
275 	/* Each GT in a tile has its own TLB to cache GGTT lookups */
276 	ggtt_invalidate_gt_tlb(ggtt->tile->primary_gt);
277 	ggtt_invalidate_gt_tlb(ggtt->tile->media_gt);
278 }
279 
280 void xe_ggtt_printk(struct xe_ggtt *ggtt, const char *prefix)
281 {
282 	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB];
283 	u64 addr, scratch_pte;
284 
285 	scratch_pte = ggtt->pt_ops->pte_encode_bo(ggtt->scratch, 0, pat_index);
286 
287 	printk("%sGlobal GTT:", prefix);
288 	for (addr = 0; addr < ggtt->size; addr += XE_PAGE_SIZE) {
289 		unsigned int i = addr / XE_PAGE_SIZE;
290 
291 		xe_tile_assert(ggtt->tile, addr <= U32_MAX);
292 		if (ggtt->gsm[i] == scratch_pte)
293 			continue;
294 
295 		printk("%s    ggtt[0x%08x] = 0x%016llx",
296 		       prefix, (u32)addr, ggtt->gsm[i]);
297 	}
298 }
299 
300 static void xe_ggtt_dump_node(struct xe_ggtt *ggtt,
301 			      const struct drm_mm_node *node, const char *description)
302 {
303 	char buf[10];
304 
305 	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
306 		string_get_size(node->size, 1, STRING_UNITS_2, buf, sizeof(buf));
307 		xe_gt_dbg(ggtt->tile->primary_gt, "GGTT %#llx-%#llx (%s) %s\n",
308 			  node->start, node->start + node->size, buf, description);
309 	}
310 }
311 
312 /**
313  * xe_ggtt_balloon - prevent allocation of specified GGTT addresses
314  * @ggtt: the &xe_ggtt where we want to make reservation
315  * @start: the starting GGTT address of the reserved region
316  * @end: then end GGTT address of the reserved region
317  * @node: the &drm_mm_node to hold reserved GGTT node
318  *
319  * Use xe_ggtt_deballoon() to release a reserved GGTT node.
320  *
321  * Return: 0 on success or a negative error code on failure.
322  */
323 int xe_ggtt_balloon(struct xe_ggtt *ggtt, u64 start, u64 end, struct drm_mm_node *node)
324 {
325 	int err;
326 
327 	xe_tile_assert(ggtt->tile, start < end);
328 	xe_tile_assert(ggtt->tile, IS_ALIGNED(start, XE_PAGE_SIZE));
329 	xe_tile_assert(ggtt->tile, IS_ALIGNED(end, XE_PAGE_SIZE));
330 	xe_tile_assert(ggtt->tile, !drm_mm_node_allocated(node));
331 
332 	node->color = 0;
333 	node->start = start;
334 	node->size = end - start;
335 
336 	mutex_lock(&ggtt->lock);
337 	err = drm_mm_reserve_node(&ggtt->mm, node);
338 	mutex_unlock(&ggtt->lock);
339 
340 	if (xe_gt_WARN(ggtt->tile->primary_gt, err,
341 		       "Failed to balloon GGTT %#llx-%#llx (%pe)\n",
342 		       node->start, node->start + node->size, ERR_PTR(err)))
343 		return err;
344 
345 	xe_ggtt_dump_node(ggtt, node, "balloon");
346 	return 0;
347 }
348 
349 /**
350  * xe_ggtt_deballoon - release a reserved GGTT region
351  * @ggtt: the &xe_ggtt where reserved node belongs
352  * @node: the &drm_mm_node with reserved GGTT region
353  *
354  * See xe_ggtt_balloon() for details.
355  */
356 void xe_ggtt_deballoon(struct xe_ggtt *ggtt, struct drm_mm_node *node)
357 {
358 	if (!drm_mm_node_allocated(node))
359 		return;
360 
361 	xe_ggtt_dump_node(ggtt, node, "deballoon");
362 
363 	mutex_lock(&ggtt->lock);
364 	drm_mm_remove_node(node);
365 	mutex_unlock(&ggtt->lock);
366 }
367 
368 int xe_ggtt_insert_special_node_locked(struct xe_ggtt *ggtt, struct drm_mm_node *node,
369 				       u32 size, u32 align, u32 mm_flags)
370 {
371 	return drm_mm_insert_node_generic(&ggtt->mm, node, size, align, 0,
372 					  mm_flags);
373 }
374 
375 int xe_ggtt_insert_special_node(struct xe_ggtt *ggtt, struct drm_mm_node *node,
376 				u32 size, u32 align)
377 {
378 	int ret;
379 
380 	mutex_lock(&ggtt->lock);
381 	ret = xe_ggtt_insert_special_node_locked(ggtt, node, size,
382 						 align, DRM_MM_INSERT_HIGH);
383 	mutex_unlock(&ggtt->lock);
384 
385 	return ret;
386 }
387 
388 void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
389 {
390 	u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB;
391 	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[cache_mode];
392 	u64 start = bo->ggtt_node.start;
393 	u64 offset, pte;
394 
395 	for (offset = 0; offset < bo->size; offset += XE_PAGE_SIZE) {
396 		pte = ggtt->pt_ops->pte_encode_bo(bo, offset, pat_index);
397 		xe_ggtt_set_pte(ggtt, start + offset, pte);
398 	}
399 }
400 
401 static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
402 				  u64 start, u64 end)
403 {
404 	int err;
405 	u64 alignment = XE_PAGE_SIZE;
406 
407 	if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)
408 		alignment = SZ_64K;
409 
410 	if (XE_WARN_ON(bo->ggtt_node.size)) {
411 		/* Someone's already inserted this BO in the GGTT */
412 		xe_tile_assert(ggtt->tile, bo->ggtt_node.size == bo->size);
413 		return 0;
414 	}
415 
416 	err = xe_bo_validate(bo, NULL, false);
417 	if (err)
418 		return err;
419 
420 	xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile));
421 	mutex_lock(&ggtt->lock);
422 	err = drm_mm_insert_node_in_range(&ggtt->mm, &bo->ggtt_node, bo->size,
423 					  alignment, 0, start, end, 0);
424 	if (!err)
425 		xe_ggtt_map_bo(ggtt, bo);
426 	mutex_unlock(&ggtt->lock);
427 
428 	if (!err && bo->flags & XE_BO_FLAG_GGTT_INVALIDATE)
429 		xe_ggtt_invalidate(ggtt);
430 	xe_pm_runtime_put(tile_to_xe(ggtt->tile));
431 
432 	return err;
433 }
434 
435 int xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
436 			 u64 start, u64 end)
437 {
438 	return __xe_ggtt_insert_bo_at(ggtt, bo, start, end);
439 }
440 
441 int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
442 {
443 	return __xe_ggtt_insert_bo_at(ggtt, bo, 0, U64_MAX);
444 }
445 
446 void xe_ggtt_remove_node(struct xe_ggtt *ggtt, struct drm_mm_node *node,
447 			 bool invalidate)
448 {
449 	struct xe_device *xe = tile_to_xe(ggtt->tile);
450 	bool bound;
451 	int idx;
452 
453 	bound = drm_dev_enter(&xe->drm, &idx);
454 	if (bound)
455 		xe_pm_runtime_get_noresume(xe);
456 
457 	mutex_lock(&ggtt->lock);
458 	if (bound)
459 		xe_ggtt_clear(ggtt, node->start, node->size);
460 	drm_mm_remove_node(node);
461 	node->size = 0;
462 	mutex_unlock(&ggtt->lock);
463 
464 	if (!bound)
465 		return;
466 
467 	if (invalidate)
468 		xe_ggtt_invalidate(ggtt);
469 
470 	xe_pm_runtime_put(xe);
471 	drm_dev_exit(idx);
472 }
473 
474 void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
475 {
476 	if (XE_WARN_ON(!bo->ggtt_node.size))
477 		return;
478 
479 	/* This BO is not currently in the GGTT */
480 	xe_tile_assert(ggtt->tile, bo->ggtt_node.size == bo->size);
481 
482 	xe_ggtt_remove_node(ggtt, &bo->ggtt_node,
483 			    bo->flags & XE_BO_FLAG_GGTT_INVALIDATE);
484 }
485 
486 #ifdef CONFIG_PCI_IOV
487 static u64 xe_encode_vfid_pte(u16 vfid)
488 {
489 	return FIELD_PREP(GGTT_PTE_VFID, vfid) | XE_PAGE_PRESENT;
490 }
491 
492 static void xe_ggtt_assign_locked(struct xe_ggtt *ggtt, const struct drm_mm_node *node, u16 vfid)
493 {
494 	u64 start = node->start;
495 	u64 size = node->size;
496 	u64 end = start + size - 1;
497 	u64 pte = xe_encode_vfid_pte(vfid);
498 
499 	lockdep_assert_held(&ggtt->lock);
500 
501 	if (!drm_mm_node_allocated(node))
502 		return;
503 
504 	while (start < end) {
505 		xe_ggtt_set_pte(ggtt, start, pte);
506 		start += XE_PAGE_SIZE;
507 	}
508 
509 	xe_ggtt_invalidate(ggtt);
510 }
511 
512 /**
513  * xe_ggtt_assign - assign a GGTT region to the VF
514  * @ggtt: the &xe_ggtt where the node belongs
515  * @node: the &drm_mm_node to update
516  * @vfid: the VF identifier
517  *
518  * This function is used by the PF driver to assign a GGTT region to the VF.
519  * In addition to PTE's VFID bits 11:2 also PRESENT bit 0 is set as on some
520  * platforms VFs can't modify that either.
521  */
522 void xe_ggtt_assign(struct xe_ggtt *ggtt, const struct drm_mm_node *node, u16 vfid)
523 {
524 	mutex_lock(&ggtt->lock);
525 	xe_ggtt_assign_locked(ggtt, node, vfid);
526 	mutex_unlock(&ggtt->lock);
527 }
528 #endif
529 
530 int xe_ggtt_dump(struct xe_ggtt *ggtt, struct drm_printer *p)
531 {
532 	int err;
533 
534 	err = mutex_lock_interruptible(&ggtt->lock);
535 	if (err)
536 		return err;
537 
538 	drm_mm_print(&ggtt->mm, p);
539 	mutex_unlock(&ggtt->lock);
540 	return err;
541 }
542