xref: /linux/drivers/gpu/drm/xe/xe_ggtt.c (revision 955abe0a1b41de5ba61fe4cd614ebc123084d499) 
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2021 Intel Corporation
4  */
5 
6 #include "xe_ggtt.h"
7 
8 #include <linux/io-64-nonatomic-lo-hi.h>
9 #include <linux/sizes.h>
10 
11 #include <drm/drm_drv.h>
12 #include <drm/drm_managed.h>
13 #include <drm/intel/i915_drm.h>
14 #include <generated/xe_wa_oob.h>
15 
16 #include "regs/xe_gt_regs.h"
17 #include "regs/xe_gtt_defs.h"
18 #include "regs/xe_regs.h"
19 #include "xe_assert.h"
20 #include "xe_bo.h"
21 #include "xe_device.h"
22 #include "xe_gt.h"
23 #include "xe_gt_printk.h"
24 #include "xe_gt_sriov_vf.h"
25 #include "xe_gt_tlb_invalidation.h"
26 #include "xe_map.h"
27 #include "xe_mmio.h"
28 #include "xe_pm.h"
29 #include "xe_sriov.h"
30 #include "xe_wa.h"
31 #include "xe_wopcm.h"
32 
33 static u64 xelp_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
34 				   u16 pat_index)
35 {
36 	u64 pte;
37 
38 	pte = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE);
39 	pte |= XE_PAGE_PRESENT;
40 
41 	if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo))
42 		pte |= XE_GGTT_PTE_DM;
43 
44 	return pte;
45 }
46 
47 static u64 xelpg_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
48 				    u16 pat_index)
49 {
50 	struct xe_device *xe = xe_bo_device(bo);
51 	u64 pte;
52 
53 	pte = xelp_ggtt_pte_encode_bo(bo, bo_offset, pat_index);
54 
55 	xe_assert(xe, pat_index <= 3);
56 
57 	if (pat_index & BIT(0))
58 		pte |= XELPG_GGTT_PTE_PAT0;
59 
60 	if (pat_index & BIT(1))
61 		pte |= XELPG_GGTT_PTE_PAT1;
62 
63 	return pte;
64 }
65 
66 static unsigned int probe_gsm_size(struct pci_dev *pdev)
67 {
68 	u16 gmch_ctl, ggms;
69 
70 	pci_read_config_word(pdev, SNB_GMCH_CTRL, &gmch_ctl);
71 	ggms = (gmch_ctl >> BDW_GMCH_GGMS_SHIFT) & BDW_GMCH_GGMS_MASK;
72 	return ggms ? SZ_1M << ggms : 0;
73 }
74 
75 static void ggtt_update_access_counter(struct xe_ggtt *ggtt)
76 {
77 	struct xe_gt *gt = XE_WA(ggtt->tile->primary_gt, 22019338487) ? ggtt->tile->primary_gt :
78 			   ggtt->tile->media_gt;
79 	u32 max_gtt_writes = XE_WA(ggtt->tile->primary_gt, 22019338487) ? 1100 : 63;
80 	/*
81 	 * Wa_22019338487: GMD_ID is a RO register, a dummy write forces gunit
82 	 * to wait for completion of prior GTT writes before letting this through.
83 	 * This needs to be done for all GGTT writes originating from the CPU.
84 	 */
85 	lockdep_assert_held(&ggtt->lock);
86 
87 	if ((++ggtt->access_count % max_gtt_writes) == 0) {
88 		xe_mmio_write32(gt, GMD_ID, 0x0);
89 		ggtt->access_count = 0;
90 	}
91 }
92 
93 static void xe_ggtt_set_pte(struct xe_ggtt *ggtt, u64 addr, u64 pte)
94 {
95 	xe_tile_assert(ggtt->tile, !(addr & XE_PTE_MASK));
96 	xe_tile_assert(ggtt->tile, addr < ggtt->size);
97 
98 	writeq(pte, &ggtt->gsm[addr >> XE_PTE_SHIFT]);
99 }
100 
101 static void xe_ggtt_set_pte_and_flush(struct xe_ggtt *ggtt, u64 addr, u64 pte)
102 {
103 	xe_ggtt_set_pte(ggtt, addr, pte);
104 	ggtt_update_access_counter(ggtt);
105 }
106 
107 static void xe_ggtt_clear(struct xe_ggtt *ggtt, u64 start, u64 size)
108 {
109 	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB];
110 	u64 end = start + size - 1;
111 	u64 scratch_pte;
112 
113 	xe_tile_assert(ggtt->tile, start < end);
114 
115 	if (ggtt->scratch)
116 		scratch_pte = ggtt->pt_ops->pte_encode_bo(ggtt->scratch, 0,
117 							  pat_index);
118 	else
119 		scratch_pte = 0;
120 
121 	while (start < end) {
122 		ggtt->pt_ops->ggtt_set_pte(ggtt, start, scratch_pte);
123 		start += XE_PAGE_SIZE;
124 	}
125 }
126 
127 static void ggtt_fini_early(struct drm_device *drm, void *arg)
128 {
129 	struct xe_ggtt *ggtt = arg;
130 
131 	mutex_destroy(&ggtt->lock);
132 	drm_mm_takedown(&ggtt->mm);
133 }
134 
135 static void ggtt_fini(struct drm_device *drm, void *arg)
136 {
137 	struct xe_ggtt *ggtt = arg;
138 
139 	ggtt->scratch = NULL;
140 }
141 
142 static void primelockdep(struct xe_ggtt *ggtt)
143 {
144 	if (!IS_ENABLED(CONFIG_LOCKDEP))
145 		return;
146 
147 	fs_reclaim_acquire(GFP_KERNEL);
148 	might_lock(&ggtt->lock);
149 	fs_reclaim_release(GFP_KERNEL);
150 }
151 
152 static const struct xe_ggtt_pt_ops xelp_pt_ops = {
153 	.pte_encode_bo = xelp_ggtt_pte_encode_bo,
154 	.ggtt_set_pte = xe_ggtt_set_pte,
155 };
156 
157 static const struct xe_ggtt_pt_ops xelpg_pt_ops = {
158 	.pte_encode_bo = xelpg_ggtt_pte_encode_bo,
159 	.ggtt_set_pte = xe_ggtt_set_pte,
160 };
161 
162 static const struct xe_ggtt_pt_ops xelpg_pt_wa_ops = {
163 	.pte_encode_bo = xelpg_ggtt_pte_encode_bo,
164 	.ggtt_set_pte = xe_ggtt_set_pte_and_flush,
165 };
166 
167 /*
168  * Early GGTT initialization, which allows to create new mappings usable by the
169  * GuC.
170  * Mappings are not usable by the HW engines, as it doesn't have scratch /
171  * initial clear done to it yet. That will happen in the regular, non-early
172  * GGTT init.
173  */
174 int xe_ggtt_init_early(struct xe_ggtt *ggtt)
175 {
176 	struct xe_device *xe = tile_to_xe(ggtt->tile);
177 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
178 	unsigned int gsm_size;
179 	int err;
180 
181 	if (IS_SRIOV_VF(xe))
182 		gsm_size = SZ_8M; /* GGTT is expected to be 4GiB */
183 	else
184 		gsm_size = probe_gsm_size(pdev);
185 
186 	if (gsm_size == 0) {
187 		drm_err(&xe->drm, "Hardware reported no preallocated GSM\n");
188 		return -ENOMEM;
189 	}
190 
191 	ggtt->gsm = ggtt->tile->mmio.regs + SZ_8M;
192 	ggtt->size = (gsm_size / 8) * (u64) XE_PAGE_SIZE;
193 
194 	if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)
195 		ggtt->flags |= XE_GGTT_FLAGS_64K;
196 
197 	/*
198 	 * 8B per entry, each points to a 4KB page.
199 	 *
200 	 * The GuC address space is limited on both ends of the GGTT, because
201 	 * the GuC shim HW redirects accesses to those addresses to other HW
202 	 * areas instead of going through the GGTT. On the bottom end, the GuC
203 	 * can't access offsets below the WOPCM size, while on the top side the
204 	 * limit is fixed at GUC_GGTT_TOP. To keep things simple, instead of
205 	 * checking each object to see if they are accessed by GuC or not, we
206 	 * just exclude those areas from the allocator. Additionally, to
207 	 * simplify the driver load, we use the maximum WOPCM size in this logic
208 	 * instead of the programmed one, so we don't need to wait until the
209 	 * actual size to be programmed is determined (which requires FW fetch)
210 	 * before initializing the GGTT. These simplifications might waste space
211 	 * in the GGTT (about 20-25 MBs depending on the platform) but we can
212 	 * live with this.
213 	 *
214 	 * Another benifit of this is the GuC bootrom can't access anything
215 	 * below the WOPCM max size so anything the bootom needs to access (e.g.
216 	 * a RSA key) needs to be placed in the GGTT above the WOPCM max size.
217 	 * Starting the GGTT allocations above the WOPCM max give us the correct
218 	 * placement for free.
219 	 */
220 	if (ggtt->size > GUC_GGTT_TOP)
221 		ggtt->size = GUC_GGTT_TOP;
222 
223 	if (GRAPHICS_VERx100(xe) >= 1270)
224 		ggtt->pt_ops = (ggtt->tile->media_gt &&
225 			       XE_WA(ggtt->tile->media_gt, 22019338487)) ||
226 			       XE_WA(ggtt->tile->primary_gt, 22019338487) ?
227 			       &xelpg_pt_wa_ops : &xelpg_pt_ops;
228 	else
229 		ggtt->pt_ops = &xelp_pt_ops;
230 
231 	drm_mm_init(&ggtt->mm, xe_wopcm_size(xe),
232 		    ggtt->size - xe_wopcm_size(xe));
233 	mutex_init(&ggtt->lock);
234 	primelockdep(ggtt);
235 
236 	err = drmm_add_action_or_reset(&xe->drm, ggtt_fini_early, ggtt);
237 	if (err)
238 		return err;
239 
240 	if (IS_SRIOV_VF(xe)) {
241 		err = xe_gt_sriov_vf_prepare_ggtt(xe_tile_get_gt(ggtt->tile, 0));
242 		if (err)
243 			return err;
244 	}
245 
246 	return 0;
247 }
248 
249 static void xe_ggtt_invalidate(struct xe_ggtt *ggtt);
250 
251 static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt)
252 {
253 	struct drm_mm_node *hole;
254 	u64 start, end;
255 
256 	/* Display may have allocated inside ggtt, so be careful with clearing here */
257 	mutex_lock(&ggtt->lock);
258 	drm_mm_for_each_hole(hole, &ggtt->mm, start, end)
259 		xe_ggtt_clear(ggtt, start, end - start);
260 
261 	xe_ggtt_invalidate(ggtt);
262 	mutex_unlock(&ggtt->lock);
263 }
264 
265 int xe_ggtt_init(struct xe_ggtt *ggtt)
266 {
267 	struct xe_device *xe = tile_to_xe(ggtt->tile);
268 	unsigned int flags;
269 	int err;
270 
271 	/*
272 	 * So we don't need to worry about 64K GGTT layout when dealing with
273 	 * scratch entires, rather keep the scratch page in system memory on
274 	 * platforms where 64K pages are needed for VRAM.
275 	 */
276 	flags = XE_BO_FLAG_PINNED;
277 	if (ggtt->flags & XE_GGTT_FLAGS_64K)
278 		flags |= XE_BO_FLAG_SYSTEM;
279 	else
280 		flags |= XE_BO_FLAG_VRAM_IF_DGFX(ggtt->tile);
281 
282 	ggtt->scratch = xe_managed_bo_create_pin_map(xe, ggtt->tile, XE_PAGE_SIZE, flags);
283 	if (IS_ERR(ggtt->scratch)) {
284 		err = PTR_ERR(ggtt->scratch);
285 		goto err;
286 	}
287 
288 	xe_map_memset(xe, &ggtt->scratch->vmap, 0, 0, ggtt->scratch->size);
289 
290 	xe_ggtt_initial_clear(ggtt);
291 
292 	return drmm_add_action_or_reset(&xe->drm, ggtt_fini, ggtt);
293 err:
294 	ggtt->scratch = NULL;
295 	return err;
296 }
297 
298 static void ggtt_invalidate_gt_tlb(struct xe_gt *gt)
299 {
300 	int err;
301 
302 	if (!gt)
303 		return;
304 
305 	err = xe_gt_tlb_invalidation_ggtt(gt);
306 	if (err)
307 		drm_warn(&gt_to_xe(gt)->drm, "xe_gt_tlb_invalidation_ggtt error=%d", err);
308 }
309 
310 static void xe_ggtt_invalidate(struct xe_ggtt *ggtt)
311 {
312 	/* Each GT in a tile has its own TLB to cache GGTT lookups */
313 	ggtt_invalidate_gt_tlb(ggtt->tile->primary_gt);
314 	ggtt_invalidate_gt_tlb(ggtt->tile->media_gt);
315 }
316 
317 void xe_ggtt_printk(struct xe_ggtt *ggtt, const char *prefix)
318 {
319 	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB];
320 	u64 addr, scratch_pte;
321 
322 	scratch_pte = ggtt->pt_ops->pte_encode_bo(ggtt->scratch, 0, pat_index);
323 
324 	printk("%sGlobal GTT:", prefix);
325 	for (addr = 0; addr < ggtt->size; addr += XE_PAGE_SIZE) {
326 		unsigned int i = addr / XE_PAGE_SIZE;
327 
328 		xe_tile_assert(ggtt->tile, addr <= U32_MAX);
329 		if (ggtt->gsm[i] == scratch_pte)
330 			continue;
331 
332 		printk("%s    ggtt[0x%08x] = 0x%016llx",
333 		       prefix, (u32)addr, ggtt->gsm[i]);
334 	}
335 }
336 
337 static void xe_ggtt_dump_node(struct xe_ggtt *ggtt,
338 			      const struct drm_mm_node *node, const char *description)
339 {
340 	char buf[10];
341 
342 	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
343 		string_get_size(node->size, 1, STRING_UNITS_2, buf, sizeof(buf));
344 		xe_gt_dbg(ggtt->tile->primary_gt, "GGTT %#llx-%#llx (%s) %s\n",
345 			  node->start, node->start + node->size, buf, description);
346 	}
347 }
348 
349 /**
350  * xe_ggtt_balloon - prevent allocation of specified GGTT addresses
351  * @ggtt: the &xe_ggtt where we want to make reservation
352  * @start: the starting GGTT address of the reserved region
353  * @end: then end GGTT address of the reserved region
354  * @node: the &drm_mm_node to hold reserved GGTT node
355  *
356  * Use xe_ggtt_deballoon() to release a reserved GGTT node.
357  *
358  * Return: 0 on success or a negative error code on failure.
359  */
360 int xe_ggtt_balloon(struct xe_ggtt *ggtt, u64 start, u64 end, struct drm_mm_node *node)
361 {
362 	int err;
363 
364 	xe_tile_assert(ggtt->tile, start < end);
365 	xe_tile_assert(ggtt->tile, IS_ALIGNED(start, XE_PAGE_SIZE));
366 	xe_tile_assert(ggtt->tile, IS_ALIGNED(end, XE_PAGE_SIZE));
367 	xe_tile_assert(ggtt->tile, !drm_mm_node_allocated(node));
368 
369 	node->color = 0;
370 	node->start = start;
371 	node->size = end - start;
372 
373 	mutex_lock(&ggtt->lock);
374 	err = drm_mm_reserve_node(&ggtt->mm, node);
375 	mutex_unlock(&ggtt->lock);
376 
377 	if (xe_gt_WARN(ggtt->tile->primary_gt, err,
378 		       "Failed to balloon GGTT %#llx-%#llx (%pe)\n",
379 		       node->start, node->start + node->size, ERR_PTR(err)))
380 		return err;
381 
382 	xe_ggtt_dump_node(ggtt, node, "balloon");
383 	return 0;
384 }
385 
386 /**
387  * xe_ggtt_deballoon - release a reserved GGTT region
388  * @ggtt: the &xe_ggtt where reserved node belongs
389  * @node: the &drm_mm_node with reserved GGTT region
390  *
391  * See xe_ggtt_balloon() for details.
392  */
393 void xe_ggtt_deballoon(struct xe_ggtt *ggtt, struct drm_mm_node *node)
394 {
395 	if (!drm_mm_node_allocated(node))
396 		return;
397 
398 	xe_ggtt_dump_node(ggtt, node, "deballoon");
399 
400 	mutex_lock(&ggtt->lock);
401 	drm_mm_remove_node(node);
402 	mutex_unlock(&ggtt->lock);
403 }
404 
405 int xe_ggtt_insert_special_node_locked(struct xe_ggtt *ggtt, struct drm_mm_node *node,
406 				       u32 size, u32 align, u32 mm_flags)
407 {
408 	return drm_mm_insert_node_generic(&ggtt->mm, node, size, align, 0,
409 					  mm_flags);
410 }
411 
412 int xe_ggtt_insert_special_node(struct xe_ggtt *ggtt, struct drm_mm_node *node,
413 				u32 size, u32 align)
414 {
415 	int ret;
416 
417 	mutex_lock(&ggtt->lock);
418 	ret = xe_ggtt_insert_special_node_locked(ggtt, node, size,
419 						 align, DRM_MM_INSERT_HIGH);
420 	mutex_unlock(&ggtt->lock);
421 
422 	return ret;
423 }
424 
425 void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
426 {
427 	u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB;
428 	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[cache_mode];
429 	u64 start = bo->ggtt_node.start;
430 	u64 offset, pte;
431 
432 	for (offset = 0; offset < bo->size; offset += XE_PAGE_SIZE) {
433 		pte = ggtt->pt_ops->pte_encode_bo(bo, offset, pat_index);
434 		ggtt->pt_ops->ggtt_set_pte(ggtt, start + offset, pte);
435 	}
436 }
437 
438 static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
439 				  u64 start, u64 end)
440 {
441 	int err;
442 	u64 alignment = XE_PAGE_SIZE;
443 
444 	if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)
445 		alignment = SZ_64K;
446 
447 	if (XE_WARN_ON(bo->ggtt_node.size)) {
448 		/* Someone's already inserted this BO in the GGTT */
449 		xe_tile_assert(ggtt->tile, bo->ggtt_node.size == bo->size);
450 		return 0;
451 	}
452 
453 	err = xe_bo_validate(bo, NULL, false);
454 	if (err)
455 		return err;
456 
457 	xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile));
458 	mutex_lock(&ggtt->lock);
459 	err = drm_mm_insert_node_in_range(&ggtt->mm, &bo->ggtt_node, bo->size,
460 					  alignment, 0, start, end, 0);
461 	if (!err)
462 		xe_ggtt_map_bo(ggtt, bo);
463 	mutex_unlock(&ggtt->lock);
464 
465 	if (!err && bo->flags & XE_BO_FLAG_GGTT_INVALIDATE)
466 		xe_ggtt_invalidate(ggtt);
467 	xe_pm_runtime_put(tile_to_xe(ggtt->tile));
468 
469 	return err;
470 }
471 
472 int xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
473 			 u64 start, u64 end)
474 {
475 	return __xe_ggtt_insert_bo_at(ggtt, bo, start, end);
476 }
477 
478 int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
479 {
480 	return __xe_ggtt_insert_bo_at(ggtt, bo, 0, U64_MAX);
481 }
482 
483 void xe_ggtt_remove_node(struct xe_ggtt *ggtt, struct drm_mm_node *node,
484 			 bool invalidate)
485 {
486 	struct xe_device *xe = tile_to_xe(ggtt->tile);
487 	bool bound;
488 	int idx;
489 
490 	bound = drm_dev_enter(&xe->drm, &idx);
491 	if (bound)
492 		xe_pm_runtime_get_noresume(xe);
493 
494 	mutex_lock(&ggtt->lock);
495 	if (bound)
496 		xe_ggtt_clear(ggtt, node->start, node->size);
497 	drm_mm_remove_node(node);
498 	node->size = 0;
499 	mutex_unlock(&ggtt->lock);
500 
501 	if (!bound)
502 		return;
503 
504 	if (invalidate)
505 		xe_ggtt_invalidate(ggtt);
506 
507 	xe_pm_runtime_put(xe);
508 	drm_dev_exit(idx);
509 }
510 
511 void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
512 {
513 	if (XE_WARN_ON(!bo->ggtt_node.size))
514 		return;
515 
516 	/* This BO is not currently in the GGTT */
517 	xe_tile_assert(ggtt->tile, bo->ggtt_node.size == bo->size);
518 
519 	xe_ggtt_remove_node(ggtt, &bo->ggtt_node,
520 			    bo->flags & XE_BO_FLAG_GGTT_INVALIDATE);
521 }
522 
523 #ifdef CONFIG_PCI_IOV
524 static u64 xe_encode_vfid_pte(u16 vfid)
525 {
526 	return FIELD_PREP(GGTT_PTE_VFID, vfid) | XE_PAGE_PRESENT;
527 }
528 
529 static void xe_ggtt_assign_locked(struct xe_ggtt *ggtt, const struct drm_mm_node *node, u16 vfid)
530 {
531 	u64 start = node->start;
532 	u64 size = node->size;
533 	u64 end = start + size - 1;
534 	u64 pte = xe_encode_vfid_pte(vfid);
535 
536 	lockdep_assert_held(&ggtt->lock);
537 
538 	if (!drm_mm_node_allocated(node))
539 		return;
540 
541 	while (start < end) {
542 		ggtt->pt_ops->ggtt_set_pte(ggtt, start, pte);
543 		start += XE_PAGE_SIZE;
544 	}
545 
546 	xe_ggtt_invalidate(ggtt);
547 }
548 
549 /**
550  * xe_ggtt_assign - assign a GGTT region to the VF
551  * @ggtt: the &xe_ggtt where the node belongs
552  * @node: the &drm_mm_node to update
553  * @vfid: the VF identifier
554  *
555  * This function is used by the PF driver to assign a GGTT region to the VF.
556  * In addition to PTE's VFID bits 11:2 also PRESENT bit 0 is set as on some
557  * platforms VFs can't modify that either.
558  */
559 void xe_ggtt_assign(struct xe_ggtt *ggtt, const struct drm_mm_node *node, u16 vfid)
560 {
561 	mutex_lock(&ggtt->lock);
562 	xe_ggtt_assign_locked(ggtt, node, vfid);
563 	mutex_unlock(&ggtt->lock);
564 }
565 #endif
566 
567 int xe_ggtt_dump(struct xe_ggtt *ggtt, struct drm_printer *p)
568 {
569 	int err;
570 
571 	err = mutex_lock_interruptible(&ggtt->lock);
572 	if (err)
573 		return err;
574 
575 	drm_mm_print(&ggtt->mm, p);
576 	mutex_unlock(&ggtt->lock);
577 	return err;
578 }
579