xref: /linux/drivers/gpu/drm/xe/xe_ggtt.c (revision 001821b0e79716c4e17c71d8e053a23599a7a508) 
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2021 Intel Corporation
4  */
5 
6 #include "xe_ggtt.h"
7 
8 #include <linux/io-64-nonatomic-lo-hi.h>
9 #include <linux/sizes.h>
10 
11 #include <drm/drm_managed.h>
12 #include <drm/i915_drm.h>
13 
14 #include "regs/xe_gt_regs.h"
15 #include "regs/xe_gtt_defs.h"
16 #include "regs/xe_regs.h"
17 #include "xe_assert.h"
18 #include "xe_bo.h"
19 #include "xe_device.h"
20 #include "xe_gt.h"
21 #include "xe_gt_printk.h"
22 #include "xe_gt_tlb_invalidation.h"
23 #include "xe_map.h"
24 #include "xe_pm.h"
25 #include "xe_sriov.h"
26 #include "xe_wopcm.h"
27 
28 static u64 xelp_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
29 				   u16 pat_index)
30 {
31 	u64 pte;
32 
33 	pte = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE);
34 	pte |= XE_PAGE_PRESENT;
35 
36 	if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo))
37 		pte |= XE_GGTT_PTE_DM;
38 
39 	return pte;
40 }
41 
42 static u64 xelpg_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
43 				    u16 pat_index)
44 {
45 	struct xe_device *xe = xe_bo_device(bo);
46 	u64 pte;
47 
48 	pte = xelp_ggtt_pte_encode_bo(bo, bo_offset, pat_index);
49 
50 	xe_assert(xe, pat_index <= 3);
51 
52 	if (pat_index & BIT(0))
53 		pte |= XELPG_GGTT_PTE_PAT0;
54 
55 	if (pat_index & BIT(1))
56 		pte |= XELPG_GGTT_PTE_PAT1;
57 
58 	return pte;
59 }
60 
61 static unsigned int probe_gsm_size(struct pci_dev *pdev)
62 {
63 	u16 gmch_ctl, ggms;
64 
65 	pci_read_config_word(pdev, SNB_GMCH_CTRL, &gmch_ctl);
66 	ggms = (gmch_ctl >> BDW_GMCH_GGMS_SHIFT) & BDW_GMCH_GGMS_MASK;
67 	return ggms ? SZ_1M << ggms : 0;
68 }
69 
70 void xe_ggtt_set_pte(struct xe_ggtt *ggtt, u64 addr, u64 pte)
71 {
72 	xe_tile_assert(ggtt->tile, !(addr & XE_PTE_MASK));
73 	xe_tile_assert(ggtt->tile, addr < ggtt->size);
74 
75 	writeq(pte, &ggtt->gsm[addr >> XE_PTE_SHIFT]);
76 }
77 
78 static void xe_ggtt_clear(struct xe_ggtt *ggtt, u64 start, u64 size)
79 {
80 	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB];
81 	u64 end = start + size - 1;
82 	u64 scratch_pte;
83 
84 	xe_tile_assert(ggtt->tile, start < end);
85 
86 	if (ggtt->scratch)
87 		scratch_pte = ggtt->pt_ops->pte_encode_bo(ggtt->scratch, 0,
88 							  pat_index);
89 	else
90 		scratch_pte = 0;
91 
92 	while (start < end) {
93 		xe_ggtt_set_pte(ggtt, start, scratch_pte);
94 		start += XE_PAGE_SIZE;
95 	}
96 }
97 
98 static void ggtt_fini_early(struct drm_device *drm, void *arg)
99 {
100 	struct xe_ggtt *ggtt = arg;
101 
102 	mutex_destroy(&ggtt->lock);
103 	drm_mm_takedown(&ggtt->mm);
104 }
105 
106 static void ggtt_fini(struct drm_device *drm, void *arg)
107 {
108 	struct xe_ggtt *ggtt = arg;
109 
110 	ggtt->scratch = NULL;
111 }
112 
113 static void primelockdep(struct xe_ggtt *ggtt)
114 {
115 	if (!IS_ENABLED(CONFIG_LOCKDEP))
116 		return;
117 
118 	fs_reclaim_acquire(GFP_KERNEL);
119 	might_lock(&ggtt->lock);
120 	fs_reclaim_release(GFP_KERNEL);
121 }
122 
123 static const struct xe_ggtt_pt_ops xelp_pt_ops = {
124 	.pte_encode_bo = xelp_ggtt_pte_encode_bo,
125 };
126 
127 static const struct xe_ggtt_pt_ops xelpg_pt_ops = {
128 	.pte_encode_bo = xelpg_ggtt_pte_encode_bo,
129 };
130 
131 /*
132  * Early GGTT initialization, which allows to create new mappings usable by the
133  * GuC.
134  * Mappings are not usable by the HW engines, as it doesn't have scratch /
135  * initial clear done to it yet. That will happen in the regular, non-early
136  * GGTT init.
137  */
138 int xe_ggtt_init_early(struct xe_ggtt *ggtt)
139 {
140 	struct xe_device *xe = tile_to_xe(ggtt->tile);
141 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
142 	unsigned int gsm_size;
143 
144 	if (IS_SRIOV_VF(xe))
145 		gsm_size = SZ_8M; /* GGTT is expected to be 4GiB */
146 	else
147 		gsm_size = probe_gsm_size(pdev);
148 
149 	if (gsm_size == 0) {
150 		drm_err(&xe->drm, "Hardware reported no preallocated GSM\n");
151 		return -ENOMEM;
152 	}
153 
154 	ggtt->gsm = ggtt->tile->mmio.regs + SZ_8M;
155 	ggtt->size = (gsm_size / 8) * (u64) XE_PAGE_SIZE;
156 
157 	if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)
158 		ggtt->flags |= XE_GGTT_FLAGS_64K;
159 
160 	/*
161 	 * 8B per entry, each points to a 4KB page.
162 	 *
163 	 * The GuC address space is limited on both ends of the GGTT, because
164 	 * the GuC shim HW redirects accesses to those addresses to other HW
165 	 * areas instead of going through the GGTT. On the bottom end, the GuC
166 	 * can't access offsets below the WOPCM size, while on the top side the
167 	 * limit is fixed at GUC_GGTT_TOP. To keep things simple, instead of
168 	 * checking each object to see if they are accessed by GuC or not, we
169 	 * just exclude those areas from the allocator. Additionally, to
170 	 * simplify the driver load, we use the maximum WOPCM size in this logic
171 	 * instead of the programmed one, so we don't need to wait until the
172 	 * actual size to be programmed is determined (which requires FW fetch)
173 	 * before initializing the GGTT. These simplifications might waste space
174 	 * in the GGTT (about 20-25 MBs depending on the platform) but we can
175 	 * live with this.
176 	 *
177 	 * Another benifit of this is the GuC bootrom can't access anything
178 	 * below the WOPCM max size so anything the bootom needs to access (e.g.
179 	 * a RSA key) needs to be placed in the GGTT above the WOPCM max size.
180 	 * Starting the GGTT allocations above the WOPCM max give us the correct
181 	 * placement for free.
182 	 */
183 	if (ggtt->size > GUC_GGTT_TOP)
184 		ggtt->size = GUC_GGTT_TOP;
185 
186 	if (GRAPHICS_VERx100(xe) >= 1270)
187 		ggtt->pt_ops = &xelpg_pt_ops;
188 	else
189 		ggtt->pt_ops = &xelp_pt_ops;
190 
191 	drm_mm_init(&ggtt->mm, xe_wopcm_size(xe),
192 		    ggtt->size - xe_wopcm_size(xe));
193 	mutex_init(&ggtt->lock);
194 	primelockdep(ggtt);
195 
196 	return drmm_add_action_or_reset(&xe->drm, ggtt_fini_early, ggtt);
197 }
198 
199 static void xe_ggtt_invalidate(struct xe_ggtt *ggtt);
200 
201 static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt)
202 {
203 	struct drm_mm_node *hole;
204 	u64 start, end;
205 
206 	/* Display may have allocated inside ggtt, so be careful with clearing here */
207 	mutex_lock(&ggtt->lock);
208 	drm_mm_for_each_hole(hole, &ggtt->mm, start, end)
209 		xe_ggtt_clear(ggtt, start, end - start);
210 
211 	xe_ggtt_invalidate(ggtt);
212 	mutex_unlock(&ggtt->lock);
213 }
214 
215 int xe_ggtt_init(struct xe_ggtt *ggtt)
216 {
217 	struct xe_device *xe = tile_to_xe(ggtt->tile);
218 	unsigned int flags;
219 	int err;
220 
221 	/*
222 	 * So we don't need to worry about 64K GGTT layout when dealing with
223 	 * scratch entires, rather keep the scratch page in system memory on
224 	 * platforms where 64K pages are needed for VRAM.
225 	 */
226 	flags = XE_BO_FLAG_PINNED;
227 	if (ggtt->flags & XE_GGTT_FLAGS_64K)
228 		flags |= XE_BO_FLAG_SYSTEM;
229 	else
230 		flags |= XE_BO_FLAG_VRAM_IF_DGFX(ggtt->tile);
231 
232 	ggtt->scratch = xe_managed_bo_create_pin_map(xe, ggtt->tile, XE_PAGE_SIZE, flags);
233 	if (IS_ERR(ggtt->scratch)) {
234 		err = PTR_ERR(ggtt->scratch);
235 		goto err;
236 	}
237 
238 	xe_map_memset(xe, &ggtt->scratch->vmap, 0, 0, ggtt->scratch->size);
239 
240 	xe_ggtt_initial_clear(ggtt);
241 
242 	return drmm_add_action_or_reset(&xe->drm, ggtt_fini, ggtt);
243 err:
244 	ggtt->scratch = NULL;
245 	return err;
246 }
247 
248 static void ggtt_invalidate_gt_tlb(struct xe_gt *gt)
249 {
250 	int err;
251 
252 	if (!gt)
253 		return;
254 
255 	err = xe_gt_tlb_invalidation_ggtt(gt);
256 	if (err)
257 		drm_warn(&gt_to_xe(gt)->drm, "xe_gt_tlb_invalidation_ggtt error=%d", err);
258 }
259 
260 static void xe_ggtt_invalidate(struct xe_ggtt *ggtt)
261 {
262 	/* Each GT in a tile has its own TLB to cache GGTT lookups */
263 	ggtt_invalidate_gt_tlb(ggtt->tile->primary_gt);
264 	ggtt_invalidate_gt_tlb(ggtt->tile->media_gt);
265 }
266 
267 void xe_ggtt_printk(struct xe_ggtt *ggtt, const char *prefix)
268 {
269 	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB];
270 	u64 addr, scratch_pte;
271 
272 	scratch_pte = ggtt->pt_ops->pte_encode_bo(ggtt->scratch, 0, pat_index);
273 
274 	printk("%sGlobal GTT:", prefix);
275 	for (addr = 0; addr < ggtt->size; addr += XE_PAGE_SIZE) {
276 		unsigned int i = addr / XE_PAGE_SIZE;
277 
278 		xe_tile_assert(ggtt->tile, addr <= U32_MAX);
279 		if (ggtt->gsm[i] == scratch_pte)
280 			continue;
281 
282 		printk("%s    ggtt[0x%08x] = 0x%016llx",
283 		       prefix, (u32)addr, ggtt->gsm[i]);
284 	}
285 }
286 
287 static void xe_ggtt_dump_node(struct xe_ggtt *ggtt,
288 			      const struct drm_mm_node *node, const char *description)
289 {
290 	char buf[10];
291 
292 	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
293 		string_get_size(node->size, 1, STRING_UNITS_2, buf, sizeof(buf));
294 		xe_gt_dbg(ggtt->tile->primary_gt, "GGTT %#llx-%#llx (%s) %s\n",
295 			  node->start, node->start + node->size, buf, description);
296 	}
297 }
298 
299 /**
300  * xe_ggtt_balloon - prevent allocation of specified GGTT addresses
301  * @ggtt: the &xe_ggtt where we want to make reservation
302  * @start: the starting GGTT address of the reserved region
303  * @end: then end GGTT address of the reserved region
304  * @node: the &drm_mm_node to hold reserved GGTT node
305  *
306  * Use xe_ggtt_deballoon() to release a reserved GGTT node.
307  *
308  * Return: 0 on success or a negative error code on failure.
309  */
310 int xe_ggtt_balloon(struct xe_ggtt *ggtt, u64 start, u64 end, struct drm_mm_node *node)
311 {
312 	int err;
313 
314 	xe_tile_assert(ggtt->tile, start < end);
315 	xe_tile_assert(ggtt->tile, IS_ALIGNED(start, XE_PAGE_SIZE));
316 	xe_tile_assert(ggtt->tile, IS_ALIGNED(end, XE_PAGE_SIZE));
317 	xe_tile_assert(ggtt->tile, !drm_mm_node_allocated(node));
318 
319 	node->color = 0;
320 	node->start = start;
321 	node->size = end - start;
322 
323 	mutex_lock(&ggtt->lock);
324 	err = drm_mm_reserve_node(&ggtt->mm, node);
325 	mutex_unlock(&ggtt->lock);
326 
327 	if (xe_gt_WARN(ggtt->tile->primary_gt, err,
328 		       "Failed to balloon GGTT %#llx-%#llx (%pe)\n",
329 		       node->start, node->start + node->size, ERR_PTR(err)))
330 		return err;
331 
332 	xe_ggtt_dump_node(ggtt, node, "balloon");
333 	return 0;
334 }
335 
336 /**
337  * xe_ggtt_deballoon - release a reserved GGTT region
338  * @ggtt: the &xe_ggtt where reserved node belongs
339  * @node: the &drm_mm_node with reserved GGTT region
340  *
341  * See xe_ggtt_balloon() for details.
342  */
343 void xe_ggtt_deballoon(struct xe_ggtt *ggtt, struct drm_mm_node *node)
344 {
345 	if (!drm_mm_node_allocated(node))
346 		return;
347 
348 	xe_ggtt_dump_node(ggtt, node, "deballoon");
349 
350 	mutex_lock(&ggtt->lock);
351 	drm_mm_remove_node(node);
352 	mutex_unlock(&ggtt->lock);
353 }
354 
355 int xe_ggtt_insert_special_node_locked(struct xe_ggtt *ggtt, struct drm_mm_node *node,
356 				       u32 size, u32 align, u32 mm_flags)
357 {
358 	return drm_mm_insert_node_generic(&ggtt->mm, node, size, align, 0,
359 					  mm_flags);
360 }
361 
362 int xe_ggtt_insert_special_node(struct xe_ggtt *ggtt, struct drm_mm_node *node,
363 				u32 size, u32 align)
364 {
365 	int ret;
366 
367 	mutex_lock(&ggtt->lock);
368 	ret = xe_ggtt_insert_special_node_locked(ggtt, node, size,
369 						 align, DRM_MM_INSERT_HIGH);
370 	mutex_unlock(&ggtt->lock);
371 
372 	return ret;
373 }
374 
375 void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
376 {
377 	u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB;
378 	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[cache_mode];
379 	u64 start = bo->ggtt_node.start;
380 	u64 offset, pte;
381 
382 	for (offset = 0; offset < bo->size; offset += XE_PAGE_SIZE) {
383 		pte = ggtt->pt_ops->pte_encode_bo(bo, offset, pat_index);
384 		xe_ggtt_set_pte(ggtt, start + offset, pte);
385 	}
386 }
387 
388 static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
389 				  u64 start, u64 end)
390 {
391 	int err;
392 	u64 alignment = XE_PAGE_SIZE;
393 
394 	if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)
395 		alignment = SZ_64K;
396 
397 	if (XE_WARN_ON(bo->ggtt_node.size)) {
398 		/* Someone's already inserted this BO in the GGTT */
399 		xe_tile_assert(ggtt->tile, bo->ggtt_node.size == bo->size);
400 		return 0;
401 	}
402 
403 	err = xe_bo_validate(bo, NULL, false);
404 	if (err)
405 		return err;
406 
407 	xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile));
408 	mutex_lock(&ggtt->lock);
409 	err = drm_mm_insert_node_in_range(&ggtt->mm, &bo->ggtt_node, bo->size,
410 					  alignment, 0, start, end, 0);
411 	if (!err)
412 		xe_ggtt_map_bo(ggtt, bo);
413 	mutex_unlock(&ggtt->lock);
414 
415 	if (!err && bo->flags & XE_BO_FLAG_GGTT_INVALIDATE)
416 		xe_ggtt_invalidate(ggtt);
417 	xe_pm_runtime_put(tile_to_xe(ggtt->tile));
418 
419 	return err;
420 }
421 
422 int xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
423 			 u64 start, u64 end)
424 {
425 	return __xe_ggtt_insert_bo_at(ggtt, bo, start, end);
426 }
427 
428 int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
429 {
430 	return __xe_ggtt_insert_bo_at(ggtt, bo, 0, U64_MAX);
431 }
432 
433 void xe_ggtt_remove_node(struct xe_ggtt *ggtt, struct drm_mm_node *node,
434 			 bool invalidate)
435 {
436 	xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile));
437 
438 	mutex_lock(&ggtt->lock);
439 	xe_ggtt_clear(ggtt, node->start, node->size);
440 	drm_mm_remove_node(node);
441 	node->size = 0;
442 	mutex_unlock(&ggtt->lock);
443 
444 	if (invalidate)
445 		xe_ggtt_invalidate(ggtt);
446 
447 	xe_pm_runtime_put(tile_to_xe(ggtt->tile));
448 }
449 
450 void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
451 {
452 	if (XE_WARN_ON(!bo->ggtt_node.size))
453 		return;
454 
455 	/* This BO is not currently in the GGTT */
456 	xe_tile_assert(ggtt->tile, bo->ggtt_node.size == bo->size);
457 
458 	xe_ggtt_remove_node(ggtt, &bo->ggtt_node,
459 			    bo->flags & XE_BO_FLAG_GGTT_INVALIDATE);
460 }
461 
462 #ifdef CONFIG_PCI_IOV
463 static u64 xe_encode_vfid_pte(u16 vfid)
464 {
465 	return FIELD_PREP(GGTT_PTE_VFID, vfid) | XE_PAGE_PRESENT;
466 }
467 
468 static void xe_ggtt_assign_locked(struct xe_ggtt *ggtt, const struct drm_mm_node *node, u16 vfid)
469 {
470 	u64 start = node->start;
471 	u64 size = node->size;
472 	u64 end = start + size - 1;
473 	u64 pte = xe_encode_vfid_pte(vfid);
474 
475 	lockdep_assert_held(&ggtt->lock);
476 
477 	if (!drm_mm_node_allocated(node))
478 		return;
479 
480 	while (start < end) {
481 		xe_ggtt_set_pte(ggtt, start, pte);
482 		start += XE_PAGE_SIZE;
483 	}
484 
485 	xe_ggtt_invalidate(ggtt);
486 }
487 
488 /**
489  * xe_ggtt_assign - assign a GGTT region to the VF
490  * @ggtt: the &xe_ggtt where the node belongs
491  * @node: the &drm_mm_node to update
492  * @vfid: the VF identifier
493  *
494  * This function is used by the PF driver to assign a GGTT region to the VF.
495  * In addition to PTE's VFID bits 11:2 also PRESENT bit 0 is set as on some
496  * platforms VFs can't modify that either.
497  */
498 void xe_ggtt_assign(struct xe_ggtt *ggtt, const struct drm_mm_node *node, u16 vfid)
499 {
500 	mutex_lock(&ggtt->lock);
501 	xe_ggtt_assign_locked(ggtt, node, vfid);
502 	mutex_unlock(&ggtt->lock);
503 }
504 #endif
505 
506 int xe_ggtt_dump(struct xe_ggtt *ggtt, struct drm_printer *p)
507 {
508 	int err;
509 
510 	err = mutex_lock_interruptible(&ggtt->lock);
511 	if (err)
512 		return err;
513 
514 	drm_mm_print(&ggtt->mm, p);
515 	mutex_unlock(&ggtt->lock);
516 	return err;
517 }
518