xref: /linux/drivers/gpu/drm/xe/xe_device.c (revision 7255fcc80d4b525cc10cfaaf7f485830d4ed2000)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2021 Intel Corporation
4  */
5 
6 #include "xe_device.h"
7 
8 #include <linux/units.h>
9 
10 #include <drm/drm_aperture.h>
11 #include <drm/drm_atomic_helper.h>
12 #include <drm/drm_gem_ttm_helper.h>
13 #include <drm/drm_ioctl.h>
14 #include <drm/drm_managed.h>
15 #include <drm/drm_print.h>
16 #include <drm/xe_drm.h>
17 
18 #include "display/xe_display.h"
19 #include "regs/xe_gt_regs.h"
20 #include "regs/xe_regs.h"
21 #include "xe_bo.h"
22 #include "xe_debugfs.h"
23 #include "xe_dma_buf.h"
24 #include "xe_drm_client.h"
25 #include "xe_drv.h"
26 #include "xe_exec.h"
27 #include "xe_exec_queue.h"
28 #include "xe_ggtt.h"
29 #include "xe_gsc_proxy.h"
30 #include "xe_gt.h"
31 #include "xe_gt_mcr.h"
32 #include "xe_hwmon.h"
33 #include "xe_irq.h"
34 #include "xe_memirq.h"
35 #include "xe_mmio.h"
36 #include "xe_module.h"
37 #include "xe_pat.h"
38 #include "xe_pcode.h"
39 #include "xe_pm.h"
40 #include "xe_query.h"
41 #include "xe_sriov.h"
42 #include "xe_tile.h"
43 #include "xe_ttm_stolen_mgr.h"
44 #include "xe_ttm_sys_mgr.h"
45 #include "xe_vm.h"
46 #include "xe_wait_user_fence.h"
47 
48 #ifdef CONFIG_LOCKDEP
49 struct lockdep_map xe_device_mem_access_lockdep_map = {
50 	.name = "xe_device_mem_access_lockdep_map"
51 };
52 #endif
53 
54 static int xe_file_open(struct drm_device *dev, struct drm_file *file)
55 {
56 	struct xe_device *xe = to_xe_device(dev);
57 	struct xe_drm_client *client;
58 	struct xe_file *xef;
59 	int ret = -ENOMEM;
60 
61 	xef = kzalloc(sizeof(*xef), GFP_KERNEL);
62 	if (!xef)
63 		return ret;
64 
65 	client = xe_drm_client_alloc();
66 	if (!client) {
67 		kfree(xef);
68 		return ret;
69 	}
70 
71 	xef->drm = file;
72 	xef->client = client;
73 	xef->xe = xe;
74 
75 	mutex_init(&xef->vm.lock);
76 	xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1);
77 
78 	mutex_init(&xef->exec_queue.lock);
79 	xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1);
80 
81 	spin_lock(&xe->clients.lock);
82 	xe->clients.count++;
83 	spin_unlock(&xe->clients.lock);
84 
85 	file->driver_priv = xef;
86 	return 0;
87 }
88 
89 static void xe_file_close(struct drm_device *dev, struct drm_file *file)
90 {
91 	struct xe_device *xe = to_xe_device(dev);
92 	struct xe_file *xef = file->driver_priv;
93 	struct xe_vm *vm;
94 	struct xe_exec_queue *q;
95 	unsigned long idx;
96 
97 	mutex_lock(&xef->exec_queue.lock);
98 	xa_for_each(&xef->exec_queue.xa, idx, q) {
99 		xe_exec_queue_kill(q);
100 		xe_exec_queue_put(q);
101 	}
102 	mutex_unlock(&xef->exec_queue.lock);
103 	xa_destroy(&xef->exec_queue.xa);
104 	mutex_destroy(&xef->exec_queue.lock);
105 	mutex_lock(&xef->vm.lock);
106 	xa_for_each(&xef->vm.xa, idx, vm)
107 		xe_vm_close_and_put(vm);
108 	mutex_unlock(&xef->vm.lock);
109 	xa_destroy(&xef->vm.xa);
110 	mutex_destroy(&xef->vm.lock);
111 
112 	spin_lock(&xe->clients.lock);
113 	xe->clients.count--;
114 	spin_unlock(&xe->clients.lock);
115 
116 	xe_drm_client_put(xef->client);
117 	kfree(xef);
118 }
119 
120 static const struct drm_ioctl_desc xe_ioctls[] = {
121 	DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW),
122 	DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW),
123 	DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl,
124 			  DRM_RENDER_ALLOW),
125 	DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW),
126 	DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW),
127 	DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW),
128 	DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW),
129 	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl,
130 			  DRM_RENDER_ALLOW),
131 	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl,
132 			  DRM_RENDER_ALLOW),
133 	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl,
134 			  DRM_RENDER_ALLOW),
135 	DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl,
136 			  DRM_RENDER_ALLOW),
137 };
138 
139 static const struct file_operations xe_driver_fops = {
140 	.owner = THIS_MODULE,
141 	.open = drm_open,
142 	.release = drm_release_noglobal,
143 	.unlocked_ioctl = drm_ioctl,
144 	.mmap = drm_gem_mmap,
145 	.poll = drm_poll,
146 	.read = drm_read,
147 	.compat_ioctl = drm_compat_ioctl,
148 	.llseek = noop_llseek,
149 #ifdef CONFIG_PROC_FS
150 	.show_fdinfo = drm_show_fdinfo,
151 #endif
152 };
153 
154 static void xe_driver_release(struct drm_device *dev)
155 {
156 	struct xe_device *xe = to_xe_device(dev);
157 
158 	pci_set_drvdata(to_pci_dev(xe->drm.dev), NULL);
159 }
160 
161 static struct drm_driver driver = {
162 	/* Don't use MTRRs here; the Xserver or userspace app should
163 	 * deal with them for Intel hardware.
164 	 */
165 	.driver_features =
166 	    DRIVER_GEM |
167 	    DRIVER_RENDER | DRIVER_SYNCOBJ |
168 	    DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA,
169 	.open = xe_file_open,
170 	.postclose = xe_file_close,
171 
172 	.gem_prime_import = xe_gem_prime_import,
173 
174 	.dumb_create = xe_bo_dumb_create,
175 	.dumb_map_offset = drm_gem_ttm_dumb_map_offset,
176 #ifdef CONFIG_PROC_FS
177 	.show_fdinfo = xe_drm_client_fdinfo,
178 #endif
179 	.release = &xe_driver_release,
180 
181 	.ioctls = xe_ioctls,
182 	.num_ioctls = ARRAY_SIZE(xe_ioctls),
183 	.fops = &xe_driver_fops,
184 	.name = DRIVER_NAME,
185 	.desc = DRIVER_DESC,
186 	.date = DRIVER_DATE,
187 	.major = DRIVER_MAJOR,
188 	.minor = DRIVER_MINOR,
189 	.patchlevel = DRIVER_PATCHLEVEL,
190 };
191 
192 static void xe_device_destroy(struct drm_device *dev, void *dummy)
193 {
194 	struct xe_device *xe = to_xe_device(dev);
195 
196 	if (xe->preempt_fence_wq)
197 		destroy_workqueue(xe->preempt_fence_wq);
198 
199 	if (xe->ordered_wq)
200 		destroy_workqueue(xe->ordered_wq);
201 
202 	if (xe->unordered_wq)
203 		destroy_workqueue(xe->unordered_wq);
204 
205 	ttm_device_fini(&xe->ttm);
206 }
207 
208 struct xe_device *xe_device_create(struct pci_dev *pdev,
209 				   const struct pci_device_id *ent)
210 {
211 	struct xe_device *xe;
212 	int err;
213 
214 	xe_display_driver_set_hooks(&driver);
215 
216 	err = drm_aperture_remove_conflicting_pci_framebuffers(pdev, &driver);
217 	if (err)
218 		return ERR_PTR(err);
219 
220 	xe = devm_drm_dev_alloc(&pdev->dev, &driver, struct xe_device, drm);
221 	if (IS_ERR(xe))
222 		return xe;
223 
224 	err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev,
225 			      xe->drm.anon_inode->i_mapping,
226 			      xe->drm.vma_offset_manager, false, false);
227 	if (WARN_ON(err))
228 		goto err;
229 
230 	err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL);
231 	if (err)
232 		goto err;
233 
234 	xe->info.devid = pdev->device;
235 	xe->info.revid = pdev->revision;
236 	xe->info.force_execlist = xe_modparam.force_execlist;
237 
238 	spin_lock_init(&xe->irq.lock);
239 	spin_lock_init(&xe->clients.lock);
240 
241 	init_waitqueue_head(&xe->ufence_wq);
242 
243 	drmm_mutex_init(&xe->drm, &xe->usm.lock);
244 	xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC);
245 
246 	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
247 		/* Trigger a large asid and an early asid wrap. */
248 		u32 asid;
249 
250 		BUILD_BUG_ON(XE_MAX_ASID < 2);
251 		err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, NULL,
252 				      XA_LIMIT(XE_MAX_ASID - 2, XE_MAX_ASID - 1),
253 				      &xe->usm.next_asid, GFP_KERNEL);
254 		drm_WARN_ON(&xe->drm, err);
255 		if (err >= 0)
256 			xa_erase(&xe->usm.asid_to_vm, asid);
257 	}
258 
259 	spin_lock_init(&xe->pinned.lock);
260 	INIT_LIST_HEAD(&xe->pinned.kernel_bo_present);
261 	INIT_LIST_HEAD(&xe->pinned.external_vram);
262 	INIT_LIST_HEAD(&xe->pinned.evicted);
263 
264 	xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq", 0);
265 	xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0);
266 	xe->unordered_wq = alloc_workqueue("xe-unordered-wq", 0, 0);
267 	if (!xe->ordered_wq || !xe->unordered_wq ||
268 	    !xe->preempt_fence_wq) {
269 		/*
270 		 * Cleanup done in xe_device_destroy via
271 		 * drmm_add_action_or_reset register above
272 		 */
273 		drm_err(&xe->drm, "Failed to allocate xe workqueues\n");
274 		err = -ENOMEM;
275 		goto err;
276 	}
277 
278 	err = xe_display_create(xe);
279 	if (WARN_ON(err))
280 		goto err;
281 
282 	return xe;
283 
284 err:
285 	return ERR_PTR(err);
286 }
287 
288 /*
289  * The driver-initiated FLR is the highest level of reset that we can trigger
290  * from within the driver. It is different from the PCI FLR in that it doesn't
291  * fully reset the SGUnit and doesn't modify the PCI config space and therefore
292  * it doesn't require a re-enumeration of the PCI BARs. However, the
293  * driver-initiated FLR does still cause a reset of both GT and display and a
294  * memory wipe of local and stolen memory, so recovery would require a full HW
295  * re-init and saving/restoring (or re-populating) the wiped memory. Since we
296  * perform the FLR as the very last action before releasing access to the HW
297  * during the driver release flow, we don't attempt recovery at all, because
298  * if/when a new instance of i915 is bound to the device it will do a full
299  * re-init anyway.
300  */
301 static void xe_driver_flr(struct xe_device *xe)
302 {
303 	const unsigned int flr_timeout = 3 * MICRO; /* specs recommend a 3s wait */
304 	struct xe_gt *gt = xe_root_mmio_gt(xe);
305 	int ret;
306 
307 	if (xe_mmio_read32(gt, GU_CNTL_PROTECTED) & DRIVERINT_FLR_DIS) {
308 		drm_info_once(&xe->drm, "BIOS Disabled Driver-FLR\n");
309 		return;
310 	}
311 
312 	drm_dbg(&xe->drm, "Triggering Driver-FLR\n");
313 
314 	/*
315 	 * Make sure any pending FLR requests have cleared by waiting for the
316 	 * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS
317 	 * to make sure it's not still set from a prior attempt (it's a write to
318 	 * clear bit).
319 	 * Note that we should never be in a situation where a previous attempt
320 	 * is still pending (unless the HW is totally dead), but better to be
321 	 * safe in case something unexpected happens
322 	 */
323 	ret = xe_mmio_wait32(gt, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false);
324 	if (ret) {
325 		drm_err(&xe->drm, "Driver-FLR-prepare wait for ready failed! %d\n", ret);
326 		return;
327 	}
328 	xe_mmio_write32(gt, GU_DEBUG, DRIVERFLR_STATUS);
329 
330 	/* Trigger the actual Driver-FLR */
331 	xe_mmio_rmw32(gt, GU_CNTL, 0, DRIVERFLR);
332 
333 	/* Wait for hardware teardown to complete */
334 	ret = xe_mmio_wait32(gt, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false);
335 	if (ret) {
336 		drm_err(&xe->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret);
337 		return;
338 	}
339 
340 	/* Wait for hardware/firmware re-init to complete */
341 	ret = xe_mmio_wait32(gt, GU_DEBUG, DRIVERFLR_STATUS, DRIVERFLR_STATUS,
342 			     flr_timeout, NULL, false);
343 	if (ret) {
344 		drm_err(&xe->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret);
345 		return;
346 	}
347 
348 	/* Clear sticky completion status */
349 	xe_mmio_write32(gt, GU_DEBUG, DRIVERFLR_STATUS);
350 }
351 
352 static void xe_driver_flr_fini(struct drm_device *drm, void *arg)
353 {
354 	struct xe_device *xe = arg;
355 
356 	if (xe->needs_flr_on_fini)
357 		xe_driver_flr(xe);
358 }
359 
360 static void xe_device_sanitize(struct drm_device *drm, void *arg)
361 {
362 	struct xe_device *xe = arg;
363 	struct xe_gt *gt;
364 	u8 id;
365 
366 	for_each_gt(gt, xe, id)
367 		xe_gt_sanitize(gt);
368 }
369 
370 static int xe_set_dma_info(struct xe_device *xe)
371 {
372 	unsigned int mask_size = xe->info.dma_mask_size;
373 	int err;
374 
375 	dma_set_max_seg_size(xe->drm.dev, xe_sg_segment_size(xe->drm.dev));
376 
377 	err = dma_set_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));
378 	if (err)
379 		goto mask_err;
380 
381 	err = dma_set_coherent_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));
382 	if (err)
383 		goto mask_err;
384 
385 	return 0;
386 
387 mask_err:
388 	drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err);
389 	return err;
390 }
391 
392 /*
393  * Initialize MMIO resources that don't require any knowledge about tile count.
394  */
395 int xe_device_probe_early(struct xe_device *xe)
396 {
397 	int err;
398 
399 	err = xe_mmio_init(xe);
400 	if (err)
401 		return err;
402 
403 	err = xe_mmio_root_tile_init(xe);
404 	if (err)
405 		return err;
406 
407 	return 0;
408 }
409 
410 static int xe_device_set_has_flat_ccs(struct  xe_device *xe)
411 {
412 	u32 reg;
413 	int err;
414 
415 	if (GRAPHICS_VER(xe) < 20 || !xe->info.has_flat_ccs)
416 		return 0;
417 
418 	struct xe_gt *gt = xe_root_mmio_gt(xe);
419 
420 	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
421 	if (err)
422 		return err;
423 
424 	reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER);
425 	xe->info.has_flat_ccs = (reg & XE2_FLAT_CCS_ENABLE);
426 
427 	if (!xe->info.has_flat_ccs)
428 		drm_dbg(&xe->drm,
429 			"Flat CCS has been disabled in bios, May lead to performance impact");
430 
431 	return xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
432 }
433 
434 int xe_device_probe(struct xe_device *xe)
435 {
436 	struct xe_tile *tile;
437 	struct xe_gt *gt;
438 	int err;
439 	u8 last_gt;
440 	u8 id;
441 
442 	xe_pat_init_early(xe);
443 
444 	err = xe_sriov_init(xe);
445 	if (err)
446 		return err;
447 
448 	xe->info.mem_region_mask = 1;
449 	err = xe_display_init_nommio(xe);
450 	if (err)
451 		return err;
452 
453 	err = xe_set_dma_info(xe);
454 	if (err)
455 		return err;
456 
457 	xe_mmio_probe_tiles(xe);
458 
459 	xe_ttm_sys_mgr_init(xe);
460 
461 	for_each_gt(gt, xe, id)
462 		xe_force_wake_init_gt(gt, gt_to_fw(gt));
463 
464 	for_each_tile(tile, xe, id) {
465 		err = xe_ggtt_init_early(tile->mem.ggtt);
466 		if (err)
467 			return err;
468 		if (IS_SRIOV_VF(xe)) {
469 			err = xe_memirq_init(&tile->sriov.vf.memirq);
470 			if (err)
471 				return err;
472 		}
473 	}
474 
475 	for_each_gt(gt, xe, id) {
476 		err = xe_gt_init_hwconfig(gt);
477 		if (err)
478 			return err;
479 	}
480 
481 	err = drmm_add_action_or_reset(&xe->drm, xe_driver_flr_fini, xe);
482 	if (err)
483 		return err;
484 
485 	for_each_gt(gt, xe, id) {
486 		err = xe_pcode_probe(gt);
487 		if (err)
488 			return err;
489 	}
490 
491 	err = xe_display_init_noirq(xe);
492 	if (err)
493 		return err;
494 
495 	err = xe_irq_install(xe);
496 	if (err)
497 		goto err;
498 
499 	for_each_gt(gt, xe, id) {
500 		err = xe_gt_init_early(gt);
501 		if (err)
502 			goto err_irq_shutdown;
503 	}
504 
505 	err = xe_device_set_has_flat_ccs(xe);
506 	if (err)
507 		goto err_irq_shutdown;
508 
509 	err = xe_mmio_probe_vram(xe);
510 	if (err)
511 		goto err_irq_shutdown;
512 
513 	for_each_tile(tile, xe, id) {
514 		err = xe_tile_init_noalloc(tile);
515 		if (err)
516 			goto err_irq_shutdown;
517 	}
518 
519 	/* Allocate and map stolen after potential VRAM resize */
520 	xe_ttm_stolen_mgr_init(xe);
521 
522 	/*
523 	 * Now that GT is initialized (TTM in particular),
524 	 * we can try to init display, and inherit the initial fb.
525 	 * This is the reason the first allocation needs to be done
526 	 * inside display.
527 	 */
528 	err = xe_display_init_noaccel(xe);
529 	if (err)
530 		goto err_irq_shutdown;
531 
532 	for_each_gt(gt, xe, id) {
533 		last_gt = id;
534 
535 		err = xe_gt_init(gt);
536 		if (err)
537 			goto err_fini_gt;
538 	}
539 
540 	xe_heci_gsc_init(xe);
541 
542 	err = xe_display_init(xe);
543 	if (err)
544 		goto err_fini_gt;
545 
546 	err = drm_dev_register(&xe->drm, 0);
547 	if (err)
548 		goto err_fini_display;
549 
550 	xe_display_register(xe);
551 
552 	xe_debugfs_register(xe);
553 
554 	xe_hwmon_register(xe);
555 
556 	err = drmm_add_action_or_reset(&xe->drm, xe_device_sanitize, xe);
557 	if (err)
558 		return err;
559 
560 	return 0;
561 
562 err_fini_display:
563 	xe_display_driver_remove(xe);
564 
565 err_fini_gt:
566 	for_each_gt(gt, xe, id) {
567 		if (id < last_gt)
568 			xe_gt_remove(gt);
569 		else
570 			break;
571 	}
572 
573 err_irq_shutdown:
574 	xe_irq_shutdown(xe);
575 err:
576 	xe_display_fini(xe);
577 	return err;
578 }
579 
580 static void xe_device_remove_display(struct xe_device *xe)
581 {
582 	xe_display_unregister(xe);
583 
584 	drm_dev_unplug(&xe->drm);
585 	xe_display_driver_remove(xe);
586 }
587 
588 void xe_device_remove(struct xe_device *xe)
589 {
590 	struct xe_gt *gt;
591 	u8 id;
592 
593 	xe_device_remove_display(xe);
594 
595 	xe_display_fini(xe);
596 
597 	xe_heci_gsc_fini(xe);
598 
599 	for_each_gt(gt, xe, id)
600 		xe_gt_remove(gt);
601 
602 	xe_irq_shutdown(xe);
603 }
604 
605 void xe_device_shutdown(struct xe_device *xe)
606 {
607 }
608 
609 void xe_device_wmb(struct xe_device *xe)
610 {
611 	struct xe_gt *gt = xe_root_mmio_gt(xe);
612 
613 	wmb();
614 	if (IS_DGFX(xe))
615 		xe_mmio_write32(gt, SOFTWARE_FLAGS_SPR33, 0);
616 }
617 
618 u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size)
619 {
620 	return xe_device_has_flat_ccs(xe) ?
621 		DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0;
622 }
623 
624 bool xe_device_mem_access_ongoing(struct xe_device *xe)
625 {
626 	if (xe_pm_read_callback_task(xe) != NULL)
627 		return true;
628 
629 	return atomic_read(&xe->mem_access.ref);
630 }
631 
632 void xe_device_assert_mem_access(struct xe_device *xe)
633 {
634 	XE_WARN_ON(!xe_device_mem_access_ongoing(xe));
635 }
636 
637 bool xe_device_mem_access_get_if_ongoing(struct xe_device *xe)
638 {
639 	bool active;
640 
641 	if (xe_pm_read_callback_task(xe) == current)
642 		return true;
643 
644 	active = xe_pm_runtime_get_if_active(xe);
645 	if (active) {
646 		int ref = atomic_inc_return(&xe->mem_access.ref);
647 
648 		xe_assert(xe, ref != S32_MAX);
649 	}
650 
651 	return active;
652 }
653 
654 void xe_device_mem_access_get(struct xe_device *xe)
655 {
656 	int ref;
657 
658 	/*
659 	 * This looks racy, but should be fine since the pm_callback_task only
660 	 * transitions from NULL -> current (and back to NULL again), during the
661 	 * runtime_resume() or runtime_suspend() callbacks, for which there can
662 	 * only be a single one running for our device. We only need to prevent
663 	 * recursively calling the runtime_get or runtime_put from those
664 	 * callbacks, as well as preventing triggering any access_ongoing
665 	 * asserts.
666 	 */
667 	if (xe_pm_read_callback_task(xe) == current)
668 		return;
669 
670 	/*
671 	 * Since the resume here is synchronous it can be quite easy to deadlock
672 	 * if we are not careful. Also in practice it might be quite timing
673 	 * sensitive to ever see the 0 -> 1 transition with the callers locks
674 	 * held, so deadlocks might exist but are hard for lockdep to ever see.
675 	 * With this in mind, help lockdep learn about the potentially scary
676 	 * stuff that can happen inside the runtime_resume callback by acquiring
677 	 * a dummy lock (it doesn't protect anything and gets compiled out on
678 	 * non-debug builds).  Lockdep then only needs to see the
679 	 * mem_access_lockdep_map -> runtime_resume callback once, and then can
680 	 * hopefully validate all the (callers_locks) -> mem_access_lockdep_map.
681 	 * For example if the (callers_locks) are ever grabbed in the
682 	 * runtime_resume callback, lockdep should give us a nice splat.
683 	 */
684 	lock_map_acquire(&xe_device_mem_access_lockdep_map);
685 	lock_map_release(&xe_device_mem_access_lockdep_map);
686 
687 	xe_pm_runtime_get(xe);
688 	ref = atomic_inc_return(&xe->mem_access.ref);
689 
690 	xe_assert(xe, ref != S32_MAX);
691 
692 }
693 
694 void xe_device_mem_access_put(struct xe_device *xe)
695 {
696 	int ref;
697 
698 	if (xe_pm_read_callback_task(xe) == current)
699 		return;
700 
701 	ref = atomic_dec_return(&xe->mem_access.ref);
702 	xe_pm_runtime_put(xe);
703 
704 	xe_assert(xe, ref >= 0);
705 }
706 
707 void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p)
708 {
709 	struct xe_gt *gt;
710 	u8 id;
711 
712 	drm_printf(p, "PCI ID: 0x%04x\n", xe->info.devid);
713 	drm_printf(p, "PCI revision: 0x%02x\n", xe->info.revid);
714 
715 	for_each_gt(gt, xe, id) {
716 		drm_printf(p, "GT id: %u\n", id);
717 		drm_printf(p, "\tType: %s\n",
718 			   gt->info.type == XE_GT_TYPE_MAIN ? "main" : "media");
719 		drm_printf(p, "\tIP ver: %u.%u.%u\n",
720 			   REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid),
721 			   REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid),
722 			   REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid));
723 		drm_printf(p, "\tCS reference clock: %u\n", gt->info.reference_clock);
724 	}
725 }
726 
727 u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address)
728 {
729 	return sign_extend64(address, xe->info.va_bits - 1);
730 }
731 
732 u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address)
733 {
734 	return address & GENMASK_ULL(xe->info.va_bits - 1, 0);
735 }
736