xref: /linux/drivers/gpu/drm/xe/xe_device.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2021 Intel Corporation
4  */
5 
6 #include "xe_device.h"
7 
8 #include <linux/aperture.h>
9 #include <linux/delay.h>
10 #include <linux/error-injection.h>
11 #include <linux/units.h>
12 
13 #include <drm/drm_client.h>
14 #include <drm/drm_gem_ttm_helper.h>
15 #include <drm/drm_ioctl.h>
16 #include <drm/drm_managed.h>
17 #include <drm/drm_pagemap_util.h>
18 #include <drm/drm_print.h>
19 #include <kunit/static_stub.h>
20 #include <uapi/drm/xe_drm.h>
21 
22 #include "display/xe_display.h"
23 #include "instructions/xe_gpu_commands.h"
24 #include "regs/xe_gt_regs.h"
25 #include "regs/xe_regs.h"
26 #include "xe_bo.h"
27 #include "xe_bo_evict.h"
28 #include "xe_configfs.h"
29 #include "xe_debugfs.h"
30 #include "xe_cpu_bind.h"
31 #include "xe_defaults.h"
32 #include "xe_devcoredump.h"
33 #include "xe_device_sysfs.h"
34 #include "xe_dma_buf.h"
35 #include "xe_drm_client.h"
36 #include "xe_drv.h"
37 #include "xe_exec.h"
38 #include "xe_exec_queue.h"
39 #include "xe_force_wake.h"
40 #include "xe_ggtt.h"
41 #include "xe_gt.h"
42 #include "xe_gt_mcr.h"
43 #include "xe_gt_printk.h"
44 #include "xe_gt_sriov_vf.h"
45 #include "xe_guc.h"
46 #include "xe_guc_pc.h"
47 #include "xe_hw_engine_group.h"
48 #include "xe_hwmon.h"
49 #include "xe_i2c.h"
50 #include "xe_irq.h"
51 #include "xe_late_bind_fw.h"
52 #include "xe_log.h"
53 #include "xe_mmio.h"
54 #include "xe_mmio_gem.h"
55 #include "xe_module.h"
56 #include "xe_nvm.h"
57 #include "xe_oa.h"
58 #include "xe_observation.h"
59 #include "xe_pagefault.h"
60 #include "xe_pat.h"
61 #include "xe_pcode.h"
62 #include "xe_pm.h"
63 #include "xe_pmu.h"
64 #include "xe_psmi.h"
65 #include "xe_pxp.h"
66 #include "xe_query.h"
67 #include "xe_ras.h"
68 #include "xe_shrinker.h"
69 #include "xe_soc_remapper.h"
70 #include "xe_survivability_mode.h"
71 #include "xe_sriov.h"
72 #include "xe_svm.h"
73 #include "xe_sysctrl.h"
74 #include "xe_tile.h"
75 #include "xe_ttm_stolen_mgr.h"
76 #include "xe_ttm_sys_mgr.h"
77 #include "xe_vm.h"
78 #include "xe_vm_madvise.h"
79 #include "xe_vram.h"
80 #include "xe_vram_types.h"
81 #include "xe_vsec.h"
82 #include "xe_wait_user_fence.h"
83 #include "xe_wa.h"
84 
85 #include <generated/xe_device_wa_oob.h>
86 #include <generated/xe_wa_oob.h>
87 
88 static int xe_file_open(struct drm_device *dev, struct drm_file *file)
89 {
90 	struct xe_device *xe = to_xe_device(dev);
91 	struct xe_drm_client *client;
92 	struct xe_file *xef;
93 	int ret = -ENOMEM;
94 	struct task_struct *task = NULL;
95 
96 	xef = kzalloc_obj(*xef);
97 	if (!xef)
98 		return ret;
99 
100 	client = xe_drm_client_alloc();
101 	if (!client) {
102 		kfree(xef);
103 		return ret;
104 	}
105 
106 	xef->drm = file;
107 	xef->client = client;
108 	xef->xe = xe;
109 
110 	mutex_init(&xef->vm.lock);
111 	xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1);
112 
113 	mutex_init(&xef->exec_queue.lock);
114 	xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1);
115 
116 	mutex_init(&xef->mmio_gem.lock);
117 
118 	file->driver_priv = xef;
119 	kref_init(&xef->refcount);
120 
121 	task = get_pid_task(rcu_access_pointer(file->pid), PIDTYPE_PID);
122 	if (task) {
123 		xef->process_name = kstrdup(task->comm, GFP_KERNEL);
124 		xef->pid = task->pid;
125 		put_task_struct(task);
126 	}
127 
128 	return 0;
129 }
130 
131 static void xe_file_destroy(struct kref *ref)
132 {
133 	struct xe_file *xef = container_of(ref, struct xe_file, refcount);
134 
135 	xa_destroy(&xef->exec_queue.xa);
136 	mutex_destroy(&xef->exec_queue.lock);
137 	xa_destroy(&xef->vm.xa);
138 	mutex_destroy(&xef->vm.lock);
139 
140 	mutex_destroy(&xef->mmio_gem.lock);
141 
142 	xe_drm_client_put(xef->client);
143 	kfree(xef->process_name);
144 	kfree(xef);
145 }
146 
147 /**
148  * xe_file_get() - Take a reference to the xe file object
149  * @xef: Pointer to the xe file
150  *
151  * Anyone with a pointer to xef must take a reference to the xe file
152  * object using this call.
153  *
154  * Return: xe file pointer
155  */
156 struct xe_file *xe_file_get(struct xe_file *xef)
157 {
158 	kref_get(&xef->refcount);
159 	return xef;
160 }
161 
162 /**
163  * xe_file_put() - Drop a reference to the xe file object
164  * @xef: Pointer to the xe file
165  *
166  * Used to drop reference to the xef object
167  */
168 void xe_file_put(struct xe_file *xef)
169 {
170 	kref_put(&xef->refcount, xe_file_destroy);
171 }
172 
173 static void xe_file_close(struct drm_device *dev, struct drm_file *file)
174 {
175 	struct xe_device *xe = to_xe_device(dev);
176 	struct xe_file *xef = file->driver_priv;
177 	struct xe_vm *vm;
178 	struct xe_exec_queue *q;
179 	unsigned long idx;
180 
181 	guard(xe_pm_runtime)(xe);
182 
183 	/*
184 	 * No need for exec_queue.lock here as there is no contention for it
185 	 * when FD is closing as IOCTLs presumably can't be modifying the
186 	 * xarray. Taking exec_queue.lock here causes undue dependency on
187 	 * vm->lock taken during xe_exec_queue_kill().
188 	 */
189 	xa_for_each(&xef->exec_queue.xa, idx, q) {
190 		if (q->vm && q->hwe->hw_engine_group)
191 			xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
192 		xe_exec_queue_kill(q);
193 		xe_exec_queue_put(q);
194 	}
195 	xa_for_each(&xef->vm.xa, idx, vm)
196 		xe_vm_close_and_put(vm);
197 
198 	scoped_guard(mutex, &xef->mmio_gem.lock) {
199 		if (xef->mmio_gem.pci_barrier) {
200 			xe_mmio_gem_destroy(xef->mmio_gem.pci_barrier, file);
201 			xef->mmio_gem.pci_barrier = NULL;
202 		}
203 	}
204 
205 	xe_file_put(xef);
206 }
207 
208 static const struct drm_ioctl_desc xe_ioctls[] = {
209 	DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW),
210 	DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW),
211 	DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl,
212 			  DRM_RENDER_ALLOW),
213 	DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW),
214 	DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW),
215 	DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW),
216 	DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW),
217 	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl,
218 			  DRM_RENDER_ALLOW),
219 	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl,
220 			  DRM_RENDER_ALLOW),
221 	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl,
222 			  DRM_RENDER_ALLOW),
223 	DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl,
224 			  DRM_RENDER_ALLOW),
225 	DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW),
226 	DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW),
227 	DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl,
228 			  DRM_RENDER_ALLOW),
229 	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl,
230 			  DRM_RENDER_ALLOW),
231 	DRM_IOCTL_DEF_DRV(XE_VM_GET_PROPERTY, xe_vm_get_property_ioctl,
232 			  DRM_RENDER_ALLOW),
233 };
234 
235 static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
236 {
237 	struct drm_file *file_priv = file->private_data;
238 	struct xe_device *xe = to_xe_device(file_priv->minor->dev);
239 	long ret;
240 
241 	if (xe_device_wedged(xe))
242 		return -ECANCELED;
243 
244 	ACQUIRE(xe_pm_runtime_ioctl, pm)(xe);
245 	ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm);
246 	if (ret >= 0)
247 		ret = drm_ioctl(file, cmd, arg);
248 
249 	return ret;
250 }
251 
252 #ifdef CONFIG_COMPAT
253 static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
254 {
255 	struct drm_file *file_priv = file->private_data;
256 	struct xe_device *xe = to_xe_device(file_priv->minor->dev);
257 	long ret;
258 
259 	if (xe_device_wedged(xe))
260 		return -ECANCELED;
261 
262 	ACQUIRE(xe_pm_runtime_ioctl, pm)(xe);
263 	ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm);
264 	if (ret >= 0)
265 		ret = drm_compat_ioctl(file, cmd, arg);
266 
267 	return ret;
268 }
269 #else
270 /* similarly to drm_compat_ioctl, let's it be assigned to .compat_ioct unconditionally */
271 #define xe_drm_compat_ioctl NULL
272 #endif
273 
274 static int xe_mmap(struct file *filp, struct vm_area_struct *vma)
275 {
276 	struct drm_file *priv = filp->private_data;
277 	struct drm_device *dev = priv->minor->dev;
278 
279 	if (drm_dev_is_unplugged(dev))
280 		return -ENODEV;
281 
282 	return drm_gem_mmap(filp, vma);
283 }
284 
285 static const struct file_operations xe_driver_fops = {
286 	.owner = THIS_MODULE,
287 	.open = drm_open,
288 	.release = drm_release_noglobal,
289 	.unlocked_ioctl = xe_drm_ioctl,
290 	.mmap = xe_mmap,
291 	.poll = drm_poll,
292 	.read = drm_read,
293 	.compat_ioctl = xe_drm_compat_ioctl,
294 	.llseek = noop_llseek,
295 #ifdef CONFIG_PROC_FS
296 	.show_fdinfo = drm_show_fdinfo,
297 #endif
298 	.fop_flags = FOP_UNSIGNED_OFFSET,
299 };
300 
301 /**
302  * xe_is_xe_file() - Is the file an xe device file?
303  * @file: The file.
304  *
305  * Checks whether the file is opened against
306  * an xe device.
307  *
308  * Return: %true if an xe file, %false if not.
309  */
310 bool xe_is_xe_file(const struct file *file)
311 {
312 	return file->f_op == &xe_driver_fops;
313 }
314 
315 static const struct drm_driver regular_driver = {
316 	.driver_features =
317 	    XE_DISPLAY_DRIVER_FEATURES |
318 	    DRIVER_GEM |
319 	    DRIVER_RENDER | DRIVER_SYNCOBJ |
320 	    DRIVER_SYNCOBJ_TIMELINE,
321 	.open = xe_file_open,
322 	.postclose = xe_file_close,
323 
324 	.gem_prime_import = xe_gem_prime_import,
325 
326 	.dumb_create = xe_bo_dumb_create,
327 	.dumb_map_offset = drm_gem_ttm_dumb_map_offset,
328 #ifdef CONFIG_PROC_FS
329 	.show_fdinfo = xe_drm_client_fdinfo,
330 #endif
331 	.ioctls = xe_ioctls,
332 	.num_ioctls = ARRAY_SIZE(xe_ioctls),
333 	.fops = &xe_driver_fops,
334 	.name = DRIVER_NAME,
335 	.desc = DRIVER_DESC,
336 	.major = DRIVER_MAJOR,
337 	.minor = DRIVER_MINOR,
338 	.patchlevel = DRIVER_PATCHLEVEL,
339 	XE_DISPLAY_DRIVER_OPS,
340 };
341 
342 #ifdef CONFIG_PCI_IOV
343 static const struct drm_ioctl_desc xe_ioctls_admin_only[] = {
344 	DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW),
345 	DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW),
346 };
347 
348 static const struct drm_driver admin_only_driver = {
349 	.driver_features =
350 	    DRIVER_GEM | DRIVER_RENDER,
351 	.open = xe_file_open,
352 	.postclose = xe_file_close,
353 	.ioctls = xe_ioctls_admin_only,
354 	.num_ioctls = ARRAY_SIZE(xe_ioctls_admin_only),
355 	.fops = &xe_driver_fops,
356 	.name = DRIVER_NAME,
357 	.desc = DRIVER_DESC,
358 	.major = DRIVER_MAJOR,
359 	.minor = DRIVER_MINOR,
360 	.patchlevel = DRIVER_PATCHLEVEL,
361 };
362 
363 /**
364  * xe_device_is_admin_only() - Check whether device is admin only or not.
365  * @xe: the &xe_device to check
366  *
367  * Return: true if the device is admin only, false otherwise.
368  */
369 bool xe_device_is_admin_only(const struct xe_device *xe)
370 {
371 	KUNIT_STATIC_STUB_REDIRECT(xe_device_is_admin_only, xe);
372 	return xe->drm.driver == &admin_only_driver;
373 }
374 #endif
375 
376 static void xe_device_destroy(struct drm_device *dev, void *dummy)
377 {
378 	struct xe_device *xe = to_xe_device(dev);
379 
380 	xe_bo_dev_fini(&xe->bo_device);
381 
382 	if (xe->preempt_fence_wq)
383 		destroy_workqueue(xe->preempt_fence_wq);
384 
385 	if (xe->ordered_wq)
386 		destroy_workqueue(xe->ordered_wq);
387 
388 	if (xe->unordered_wq)
389 		destroy_workqueue(xe->unordered_wq);
390 
391 	if (xe->destroy_wq)
392 		destroy_workqueue(xe->destroy_wq);
393 
394 	ttm_device_fini(&xe->ttm);
395 }
396 
397 /**
398  * xe_device_create() - Create a new &xe_device instance
399  * @pdev: the parent &pci_dev
400  *
401  * Allocate and initialize a device managed Xe device structure.
402  *
403  * Return: pointer to new &xe_device on success, or ERR_PTR on failure.
404  */
405 struct xe_device *xe_device_create(struct pci_dev *pdev)
406 {
407 	const struct drm_driver *driver = &regular_driver;
408 	struct xe_device *xe;
409 	int err;
410 
411 #ifdef CONFIG_PCI_IOV
412 	/*
413 	 * Since XE device is not initialized yet, read from configfs
414 	 * directly to decide whether we are in admin-only PF mode or not.
415 	 */
416 	if (xe_configfs_admin_only_pf(pdev))
417 		driver = &admin_only_driver;
418 #endif
419 
420 	err = aperture_remove_conflicting_pci_devices(pdev, driver->name);
421 	if (err)
422 		return ERR_PTR(err);
423 
424 	xe = devm_drm_dev_alloc(&pdev->dev, driver, struct xe_device, drm);
425 	if (IS_ERR(xe))
426 		return xe;
427 
428 	err = xe_device_init_early(xe);
429 	if (err)
430 		return ERR_PTR(err);
431 
432 	return xe;
433 }
434 ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */
435 
436 static void xe_device_parse_modparam(struct xe_device *xe)
437 {
438 	xe->atomic_svm_timeslice_ms = 5;
439 	xe->min_run_period_lr_ms = 5;
440 	xe->info.num_pf_work = xe_modparam.num_pf_work;
441 	if (xe->info.num_pf_work < 1)
442 		xe->info.num_pf_work = 1;
443 	else if (xe->info.num_pf_work > XE_PAGEFAULT_WORK_MAX)
444 		xe->info.num_pf_work = XE_PAGEFAULT_WORK_MAX;
445 }
446 
447 /**
448  * xe_device_init_early() - Initialize a new &xe_device instance
449  * @xe: the &xe_device to initialize
450  *
451  * Return: 0 on success or a negative error code on failure.
452  */
453 int xe_device_init_early(struct xe_device *xe)
454 {
455 	int err;
456 
457 	err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev,
458 			      xe->drm.anon_inode->i_mapping,
459 			      xe->drm.vma_offset_manager,
460 			      TTM_ALLOCATION_POOL_BENEFICIAL_ORDER(get_order(SZ_2M)));
461 	if (err)
462 		return err;
463 
464 	xe_bo_dev_init(&xe->bo_device);
465 	err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL);
466 	if (err)
467 		return err;
468 
469 	err = xe_shrinker_create(xe);
470 	if (err)
471 		return err;
472 
473 	xe_device_parse_modparam(xe);
474 
475 	err = xe_irq_init(xe);
476 	if (err)
477 		return err;
478 
479 	xe_validation_device_init(&xe->val);
480 
481 	init_waitqueue_head(&xe->ufence_wq);
482 
483 	init_rwsem(&xe->usm.lock);
484 
485 	err = xe_pagemap_shrinker_create(xe);
486 	if (err)
487 		return err;
488 
489 	xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC);
490 
491 	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
492 		/* Trigger a large asid and an early asid wrap. */
493 		u32 asid;
494 
495 		BUILD_BUG_ON(XE_MAX_ASID < 2);
496 		err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, NULL,
497 				      XA_LIMIT(XE_MAX_ASID - 2, XE_MAX_ASID - 1),
498 				      &xe->usm.next_asid, GFP_KERNEL);
499 		drm_WARN_ON(&xe->drm, err);
500 		if (err >= 0)
501 			xa_erase(&xe->usm.asid_to_vm, asid);
502 	}
503 
504 	err = xe_bo_pinned_init(xe);
505 	if (err)
506 		return err;
507 
508 	xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq",
509 						       WQ_MEM_RECLAIM);
510 	xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0);
511 	xe->unordered_wq = alloc_workqueue("xe-unordered-wq", WQ_PERCPU, 0);
512 	xe->destroy_wq = alloc_workqueue("xe-destroy-wq", WQ_PERCPU | WQ_MEM_RECLAIM, 0);
513 	if (!xe->ordered_wq || !xe->unordered_wq ||
514 	    !xe->preempt_fence_wq || !xe->destroy_wq) {
515 		/*
516 		 * Cleanup done in xe_device_destroy via
517 		 * drmm_add_action_or_reset register above
518 		 */
519 		drm_err(&xe->drm, "Failed to allocate xe workqueues\n");
520 		return -ENOMEM;
521 	}
522 
523 	err = drmm_mutex_init(&xe->drm, &xe->pmt.lock);
524 	if (err)
525 		return err;
526 
527 	err = xe_pm_init_early(xe);
528 	if (err)
529 		return err;
530 
531 	return 0;
532 }
533 
534 static bool xe_driver_flr_disabled(struct xe_device *xe)
535 {
536 	if (IS_SRIOV_VF(xe))
537 		return true;
538 
539 	if (xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL_PROTECTED) & DRIVERINT_FLR_DIS) {
540 		drm_info(&xe->drm, "Driver-FLR disabled by BIOS\n");
541 		return true;
542 	}
543 
544 	return false;
545 }
546 
547 /*
548  * The driver-initiated FLR is the highest level of reset that we can trigger
549  * from within the driver. It is different from the PCI FLR in that it doesn't
550  * fully reset the SGUnit and doesn't modify the PCI config space and therefore
551  * it doesn't require a re-enumeration of the PCI BARs. However, the
552  * driver-initiated FLR does still cause a reset of both GT and display and a
553  * memory wipe of local and stolen memory, so recovery would require a full HW
554  * re-init and saving/restoring (or re-populating) the wiped memory. Since we
555  * perform the FLR as the very last action before releasing access to the HW
556  * during the driver release flow, we don't attempt recovery at all, because
557  * if/when a new instance of Xe is bound to the device it will do a full
558  * re-init anyway.
559  */
560 static void __xe_driver_flr(struct xe_device *xe)
561 {
562 	const unsigned int flr_timeout = 3 * USEC_PER_SEC; /* specs recommend a 3s wait */
563 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);
564 	int ret;
565 
566 	drm_dbg(&xe->drm, "Triggering Driver-FLR\n");
567 
568 	/*
569 	 * Make sure any pending FLR requests have cleared by waiting for the
570 	 * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS
571 	 * to make sure it's not still set from a prior attempt (it's a write to
572 	 * clear bit).
573 	 * Note that we should never be in a situation where a previous attempt
574 	 * is still pending (unless the HW is totally dead), but better to be
575 	 * safe in case something unexpected happens
576 	 */
577 	ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false);
578 	if (ret) {
579 		drm_err(&xe->drm, "Driver-FLR-prepare wait for ready failed! %d\n", ret);
580 		return;
581 	}
582 	xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS);
583 
584 	/* Trigger the actual Driver-FLR */
585 	xe_mmio_rmw32(mmio, GU_CNTL, 0, DRIVERFLR);
586 
587 	/* Wait for hardware teardown to complete */
588 	ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false);
589 	if (ret) {
590 		drm_err(&xe->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret);
591 		return;
592 	}
593 
594 	/* Wait for hardware/firmware re-init to complete */
595 	ret = xe_mmio_wait32(mmio, GU_DEBUG, DRIVERFLR_STATUS, DRIVERFLR_STATUS,
596 			     flr_timeout, NULL, false);
597 	if (ret) {
598 		drm_err(&xe->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret);
599 		return;
600 	}
601 
602 	/* Clear sticky completion status */
603 	xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS);
604 }
605 
606 static void xe_driver_flr(struct xe_device *xe)
607 {
608 	if (xe_driver_flr_disabled(xe))
609 		return;
610 
611 	__xe_driver_flr(xe);
612 }
613 
614 static void xe_driver_flr_fini(void *arg)
615 {
616 	struct xe_device *xe = arg;
617 
618 	if (xe->needs_flr_on_fini)
619 		xe_driver_flr(xe);
620 }
621 
622 static void xe_device_sanitize(void *arg)
623 {
624 	struct xe_device *xe = arg;
625 	struct xe_gt *gt;
626 	u8 id;
627 
628 	for_each_gt(gt, xe, id)
629 		xe_gt_sanitize(gt);
630 }
631 
632 static int xe_set_dma_info(struct xe_device *xe)
633 {
634 	unsigned int mask_size = xe->info.dma_mask_size;
635 	int err;
636 
637 	dma_set_max_seg_size(xe->drm.dev, xe_sg_segment_size(xe->drm.dev));
638 
639 	err = dma_set_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));
640 	if (err)
641 		goto mask_err;
642 
643 	err = dma_set_coherent_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));
644 	if (err)
645 		goto mask_err;
646 
647 	return 0;
648 
649 mask_err:
650 	drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err);
651 	return err;
652 }
653 
654 static void assert_lmem_ready(struct xe_device *xe)
655 {
656 	if (!IS_DGFX(xe) || IS_SRIOV_VF(xe))
657 		return;
658 
659 	xe_assert(xe, xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL) &
660 		  LMEM_INIT);
661 }
662 
663 static void vf_update_device_info(struct xe_device *xe)
664 {
665 	xe_assert(xe, IS_SRIOV_VF(xe));
666 	/* disable features that are not available/applicable to VFs */
667 	xe->info.probe_display = 0;
668 	xe->info.has_heci_cscfi = 0;
669 	xe->info.has_heci_gscfi = 0;
670 	xe->info.has_i2c = 0;
671 	xe->info.has_late_bind = 0;
672 	xe->info.skip_guc_pc = 1;
673 	xe->info.skip_pcode = 1;
674 	xe->info.has_drm_ras = false;
675 	xe->info.has_device_uid = false;
676 }
677 
678 static int xe_device_vram_alloc(struct xe_device *xe)
679 {
680 	struct xe_vram_region *vram;
681 
682 	if (!IS_DGFX(xe))
683 		return 0;
684 
685 	vram = drmm_kzalloc(&xe->drm, sizeof(*vram), GFP_KERNEL);
686 	if (!vram)
687 		return -ENOMEM;
688 
689 	xe->mem.vram = vram;
690 	return 0;
691 }
692 
693 /**
694  * xe_device_probe_early: Device early probe
695  * @xe: xe device instance
696  *
697  * Initialize MMIO resources that don't require any
698  * knowledge about tile count. Also initialize pcode and
699  * check vram initialization on root tile.
700  *
701  * Return: 0 on success, error code on failure
702  */
703 int xe_device_probe_early(struct xe_device *xe)
704 {
705 	int err;
706 
707 	xe_wa_device_init(xe);
708 	xe_wa_process_device_oob(xe);
709 
710 	err = xe_mmio_probe_early(xe);
711 	if (err)
712 		return err;
713 
714 	xe_sriov_probe_early(xe);
715 
716 	if (xe_device_is_admin_only(xe) && !IS_SRIOV_PF(xe)) {
717 		xe_err(xe, "Can't run Admin-only mode without SR-IOV PF mode!\n");
718 		return -ENODEV;
719 	}
720 
721 	if (IS_SRIOV_VF(xe))
722 		vf_update_device_info(xe);
723 
724 	/*
725 	 * Check for pcode uncore_init status to confirm if the SoC
726 	 * initialization is complete. Until done, any MMIO or lmem access from
727 	 * the driver will be blocked
728 	 */
729 	err = xe_pcode_probe_early(xe);
730 	if (err || xe_survivability_mode_is_requested(xe)) {
731 		int save_err = err;
732 
733 		/*
734 		 * Try to leave device in survivability mode if device is
735 		 * possible, but still return the previous error for error
736 		 * propagation
737 		 */
738 		err = xe_survivability_mode_boot_enable(xe);
739 		if (err)
740 			return err;
741 
742 		return save_err;
743 	}
744 
745 	/*
746 	 * Make sure the lmem is initialized and ready to use. xe_pcode_ready()
747 	 * is flagged after full initialization is complete. Assert if lmem is
748 	 * not initialized.
749 	 */
750 	assert_lmem_ready(xe);
751 
752 	xe->wedged.mode = xe_device_validate_wedged_mode(xe, xe_modparam.wedged_mode) ?
753 			  XE_DEFAULT_WEDGED_MODE : xe_modparam.wedged_mode;
754 	drm_dbg(&xe->drm, "wedged_mode: setting mode (%u) %s\n",
755 		xe->wedged.mode, xe_wedged_mode_to_string(xe->wedged.mode));
756 
757 	err = xe_device_vram_alloc(xe);
758 	if (err)
759 		return err;
760 
761 	return 0;
762 }
763 ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */
764 
765 static int probe_has_flat_ccs(struct xe_device *xe)
766 {
767 	struct xe_gt *gt;
768 	u32 reg;
769 
770 	/* Always enabled/disabled, no runtime check to do */
771 	if (GRAPHICS_VER(xe) < 20 || !xe->info.has_flat_ccs || IS_SRIOV_VF(xe))
772 		return 0;
773 
774 	gt = xe_root_mmio_gt(xe);
775 	if (!gt)
776 		return 0;
777 
778 	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
779 	if (!fw_ref.domains)
780 		return -ETIMEDOUT;
781 
782 	reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER);
783 	xe->info.has_flat_ccs = (reg & XE2_FLAT_CCS_ENABLE);
784 
785 	if (!xe->info.has_flat_ccs)
786 		drm_dbg(&xe->drm,
787 			"Flat CCS has been disabled in bios, May lead to performance impact");
788 
789 	return 0;
790 }
791 
792 /*
793  * Detect if the driver is being run on pre-production hardware.  We don't
794  * keep workarounds for pre-production hardware long term, so print an
795  * error and add taint if we're being loaded on a pre-production platform
796  * for which the pre-prod workarounds have already been removed.
797  *
798  * The general policy is that we'll remove any workarounds that only apply to
799  * pre-production hardware around the time force_probe restrictions are lifted
800  * for a platform of the next major IP generation (for example, Xe2 pre-prod
801  * workarounds should be removed around the time the first Xe3 platforms have
802  * force_probe lifted).
803  */
804 static void detect_preproduction_hw(struct xe_device *xe)
805 {
806 	struct xe_gt *gt;
807 	int id;
808 
809 	/*
810 	 * SR-IOV VFs don't have access to the FUSE2 register, so we can't
811 	 * check pre-production status there.  But the host OS will notice
812 	 * and report the pre-production status, which should be enough to
813 	 * help us catch mistaken use of pre-production hardware.
814 	 */
815 	if (IS_SRIOV_VF(xe))
816 		return;
817 
818 	/*
819 	 * The "SW_CAP" fuse contains a bit indicating whether the device is a
820 	 * production or pre-production device.  This fuse is reflected through
821 	 * the GT "FUSE2" register, even though the contents of the fuse are
822 	 * not GT-specific.  Every GT's reflection of this fuse should show the
823 	 * same value, so we'll just use the first available GT for lookup.
824 	 */
825 	for_each_gt(gt, xe, id)
826 		break;
827 
828 	if (!gt)
829 		return;
830 
831 	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
832 	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) {
833 		xe_gt_err(gt, "Forcewake failure; cannot determine production/pre-production hw status.\n");
834 		return;
835 	}
836 
837 	if (xe_mmio_read32(&gt->mmio, FUSE2) & PRODUCTION_HW)
838 		return;
839 
840 	xe_info(xe, "Pre-production hardware detected.\n");
841 	if (!xe->info.has_pre_prod_wa) {
842 		xe_err(xe, "Pre-production workarounds for this platform have already been removed.\n");
843 		add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK);
844 	}
845 }
846 
847 static void xe_device_wedged_fini(struct drm_device *drm, void *arg)
848 {
849 	struct xe_device *xe = arg;
850 
851 	if (atomic_read(&xe->wedged.flag))
852 		xe_pm_runtime_put(xe);
853 }
854 
855 #ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE
856 static int xe_debug_page_size_alloc_ctrl_init(struct xe_device *xe)
857 {
858 	int err;
859 
860 	err = drmm_mutex_init(&xe->drm, &xe->page_size_alloc_ctrl.lock);
861 	if (err)
862 		return err;
863 
864 	xe->page_size_alloc_ctrl.mode = XE_PAGE_SIZE_ALLOC_CTRL_MODE_NONE;
865 	xe->page_size_alloc_ctrl.cur_index = 0;
866 
867 	return 0;
868 }
869 #else
870 static int xe_debug_page_size_alloc_ctrl_init(struct xe_device *xe)
871 {
872 	return 0;
873 }
874 #endif
875 
876 static void xe_uid_probe(struct xe_device *xe)
877 {
878 	if (xe->info.has_device_uid)
879 		xe->device_uid = xe_mmio_read64_2x32(xe_root_tile_mmio(xe), CRI_DEVICE_UID);
880 }
881 
882 int xe_device_probe(struct xe_device *xe)
883 {
884 	struct xe_tile *tile;
885 	struct xe_gt *gt;
886 	int err;
887 	u8 id;
888 
889 	xe_uid_probe(xe);
890 
891 	xe_pat_init_early(xe);
892 
893 	err = xe_sriov_init(xe);
894 	if (err)
895 		return err;
896 
897 	xe->info.mem_region_mask = 1;
898 
899 	err = xe_set_dma_info(xe);
900 	if (err)
901 		return err;
902 
903 	err = xe_mmio_probe_tiles(xe);
904 	if (err)
905 		return err;
906 
907 	for_each_gt(gt, xe, id) {
908 		err = xe_gt_init_early(gt);
909 		if (err)
910 			return err;
911 	}
912 
913 	/*
914 	 * Wa_16029380221: The affected GT will always use non-coherent
915 	 * access to page tables, so we must do uncached writes from the
916 	 * CPU.
917 	 */
918 	for_each_gt(gt, xe, id)
919 		if (XE_GT_WA(gt, 16029380221))
920 			xe->info.has_cached_pt = false;
921 
922 	for_each_tile(tile, xe, id) {
923 		err = xe_ggtt_init_early(tile->mem.ggtt);
924 		if (err)
925 			return err;
926 	}
927 
928 	/*
929 	 * From here on, if a step fails, make sure a Driver-FLR is triggereed
930 	 */
931 	err = devm_add_action_or_reset(xe->drm.dev, xe_driver_flr_fini, xe);
932 	if (err)
933 		return err;
934 
935 	err = probe_has_flat_ccs(xe);
936 	if (err)
937 		return err;
938 
939 	err = xe_vram_probe(xe);
940 	if (err)
941 		return err;
942 
943 	for_each_tile(tile, xe, id) {
944 		err = xe_tile_init_noalloc(tile);
945 		if (err)
946 			return err;
947 	}
948 
949 	/*
950 	 * Allow allocations only now to ensure xe_display_init_early()
951 	 * is the first to allocate, always.
952 	 */
953 	err = xe_ttm_sys_mgr_init(xe);
954 	if (err)
955 		return err;
956 
957 	/* Allocate and map stolen after potential VRAM resize */
958 	err = xe_ttm_stolen_mgr_init(xe);
959 	if (err)
960 		return err;
961 
962 	err = xe_soc_remapper_init(xe);
963 	if (err)
964 		return err;
965 
966 	err = xe_sysctrl_init(xe);
967 	if (err)
968 		return err;
969 
970 	xe_ras_init(xe);
971 
972 	/*
973 	 * Now that GT is initialized (TTM in particular),
974 	 * we can try to init display, and inherit the initial fb.
975 	 * This is the reason the first allocation needs to be done
976 	 * inside display.
977 	 */
978 	err = xe_display_init_early(xe);
979 	if (err)
980 		return err;
981 
982 	err = xe_vram_reserve_memtest_bo(xe);
983 	if (err)
984 		return err;
985 
986 	for_each_tile(tile, xe, id) {
987 		err = xe_tile_init(tile);
988 		if (err)
989 			return err;
990 	}
991 
992 	err = xe_irq_install(xe);
993 	if (err)
994 		return err;
995 
996 	for_each_gt(gt, xe, id) {
997 		err = xe_gt_init(gt);
998 		if (err)
999 			return err;
1000 	}
1001 
1002 	err = xe_vram_memtest(xe);
1003 	if (err)
1004 		return err;
1005 
1006 	err = xe_cpu_bind_init(xe);
1007 	if (err)
1008 		return err;
1009 
1010 	err = xe_pagefault_init(xe);
1011 	if (err)
1012 		return err;
1013 
1014 	if (xe->tiles->media_gt &&
1015 	    XE_GT_WA(xe->tiles->media_gt, 15015404425_disable))
1016 		XE_DEVICE_WA_DISABLE(xe, 15015404425);
1017 
1018 	err = xe_devcoredump_init(xe);
1019 	if (err)
1020 		return err;
1021 
1022 	xe_nvm_init(xe);
1023 
1024 	err = xe_heci_gsc_init(xe);
1025 	if (err)
1026 		return err;
1027 
1028 	err = xe_late_bind_init(&xe->late_bind);
1029 	if (err)
1030 		return err;
1031 
1032 	err = xe_oa_init(xe);
1033 	if (err)
1034 		return err;
1035 
1036 	err = xe_display_init(xe);
1037 	if (err)
1038 		return err;
1039 
1040 	err = xe_pxp_init(xe);
1041 	if (err)
1042 		return err;
1043 
1044 	err = xe_psmi_init(xe);
1045 	if (err)
1046 		return err;
1047 
1048 	err = xe_debug_page_size_alloc_ctrl_init(xe);
1049 	if (err)
1050 		return err;
1051 
1052 	err = drm_dev_register(&xe->drm, 0);
1053 	if (err)
1054 		return err;
1055 
1056 	xe_display_register(xe);
1057 
1058 	err = xe_oa_register(xe);
1059 	if (err)
1060 		goto err_unregister_display;
1061 
1062 	err = xe_pmu_register(&xe->pmu);
1063 	if (err)
1064 		goto err_unregister_display;
1065 
1066 	err = xe_device_sysfs_init(xe);
1067 	if (err)
1068 		goto err_unregister_display;
1069 
1070 	xe_debugfs_register(xe);
1071 
1072 	err = xe_hwmon_register(xe);
1073 	if (err)
1074 		goto err_unregister_display;
1075 
1076 	err = xe_i2c_probe(xe);
1077 	if (err)
1078 		goto err_unregister_display;
1079 
1080 	for_each_gt(gt, xe, id)
1081 		xe_gt_sanitize_freq(gt);
1082 
1083 	xe_vsec_init(xe);
1084 
1085 	err = xe_sriov_init_late(xe);
1086 	if (err)
1087 		goto err_unregister_display;
1088 
1089 	detect_preproduction_hw(xe);
1090 
1091 	err = drmm_add_action_or_reset(&xe->drm, xe_device_wedged_fini, xe);
1092 	if (err)
1093 		goto err_unregister_display;
1094 
1095 	/*
1096 	 * Process and log any errors detected by hardware. Possible results can
1097 	 * include declaring the device as wedged, which must be done only after
1098 	 * xe_device_wedged_fini() is registered.
1099 	 */
1100 	xe_ras_process_errors(xe);
1101 
1102 	err = devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe);
1103 	if (err)
1104 		goto err_unregister_display;
1105 
1106 	return 0;
1107 
1108 err_unregister_display:
1109 	xe_display_unregister(xe);
1110 	drm_dev_unregister(&xe->drm);
1111 
1112 	return err;
1113 }
1114 
1115 void xe_device_remove(struct xe_device *xe)
1116 {
1117 	xe_display_unregister(xe);
1118 
1119 	drm_dev_unplug(&xe->drm);
1120 
1121 	xe_bo_pci_dev_remove_all(xe);
1122 }
1123 
1124 void xe_device_shutdown(struct xe_device *xe)
1125 {
1126 	struct xe_gt *gt;
1127 	u8 id;
1128 
1129 	drm_dbg(&xe->drm, "Shutting down device\n");
1130 
1131 	xe_display_shutdown(xe);
1132 
1133 	xe_irq_suspend(xe);
1134 
1135 	for_each_gt(gt, xe, id)
1136 		xe_gt_shutdown(gt);
1137 
1138 	xe_display_shutdown_late(xe);
1139 
1140 	if (!xe_driver_flr_disabled(xe)) {
1141 		/* BOOM! */
1142 		__xe_driver_flr(xe);
1143 	}
1144 }
1145 
1146 /**
1147  * xe_device_wmb() - Device specific write memory barrier
1148  * @xe: the &xe_device
1149  *
1150  * While wmb() is sufficient for a barrier if we use system memory, on discrete
1151  * platforms with device memory we additionally need to issue a register write.
1152  * Since it doesn't matter which register we write to, use the read-only VF_CAP
1153  * register that is also marked as accessible by the VFs.
1154  */
1155 void xe_device_wmb(struct xe_device *xe)
1156 {
1157 	wmb();
1158 	if (IS_DGFX(xe))
1159 		xe_mmio_write32(xe_root_tile_mmio(xe), VF_CAP_REG, 0);
1160 }
1161 
1162 /*
1163  * Issue a TRANSIENT_FLUSH_REQUEST and wait for completion on each gt.
1164  */
1165 static void tdf_request_sync(struct xe_device *xe)
1166 {
1167 	struct xe_gt *gt;
1168 	u8 id;
1169 
1170 	for_each_gt_with_type(gt, xe, id, BIT(XE_GT_TYPE_MAIN)) {
1171 		CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
1172 		if (!fw_ref.domains)
1173 			return;
1174 
1175 		xe_mmio_write32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST);
1176 
1177 		/*
1178 		 * FIXME: We can likely do better here with our choice of
1179 		 * timeout. Currently we just assume the worst case, i.e. 150us,
1180 		 * which is believed to be sufficient to cover the worst case
1181 		 * scenario on current platforms if all cache entries are
1182 		 * transient and need to be flushed..
1183 		 */
1184 		if (xe_mmio_wait32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0,
1185 				   300, NULL, false))
1186 			xe_gt_err_once(gt, "TD flush timeout\n");
1187 	}
1188 }
1189 
1190 /**
1191  * xe_device_is_l2_flush_optimized - if L2 flush is optimized by HW
1192  * @xe: The device to check.
1193  *
1194  * Return: true if the HW device optimizing L2 flush, false otherwise.
1195  */
1196 bool xe_device_is_l2_flush_optimized(struct xe_device *xe)
1197 {
1198 	/* XA is *always* flushed, like at the end-of-submssion (and maybe other
1199 	 * places), just that internally as an optimisation hw doesn't need to make
1200 	 * that a full flush (which will also include XA) when Media is
1201 	 * off/powergated, since it doesn't need to worry about GT caches vs Media
1202 	 * coherency, and only CPU vs GPU coherency, so can make that flush a
1203 	 * targeted XA flush, since stuff tagged with XA now means it's shared with
1204 	 * the CPU. The main implication is that we now need to somehow flush non-XA before
1205 	 * freeing system memory pages, otherwise dirty cachelines could be flushed after the free
1206 	 * (like if Media suddenly turns on and does a full flush)
1207 	 */
1208 	if (GRAPHICS_VER(xe) >= 35 && !IS_DGFX(xe))
1209 		return true;
1210 	return false;
1211 }
1212 
1213 void xe_device_l2_flush(struct xe_device *xe, bool force)
1214 {
1215 	struct xe_gt *gt;
1216 
1217 	gt = xe_root_mmio_gt(xe);
1218 	if (!gt)
1219 		return;
1220 
1221 	if (!force && !XE_GT_WA(gt, 16023588340))
1222 		return;
1223 
1224 	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
1225 	if (!fw_ref.domains)
1226 		return;
1227 
1228 	spin_lock(&gt->global_invl_lock);
1229 
1230 	xe_mmio_write32(&gt->mmio, XE2_GLOBAL_INVAL, 0x1);
1231 	if (xe_mmio_wait32(&gt->mmio, XE2_GLOBAL_INVAL, 0x1, 0x0, 1000, NULL, true))
1232 		xe_gt_err_once(gt, "Global invalidation timeout\n");
1233 
1234 	spin_unlock(&gt->global_invl_lock);
1235 }
1236 
1237 /**
1238  * xe_device_td_flush() - Flush transient L3 cache entries
1239  * @xe: The device
1240  *
1241  * Display engine has direct access to memory and is never coherent with L3/L4
1242  * caches (or CPU caches), however KMD is responsible for specifically flushing
1243  * transient L3 GPU cache entries prior to the flip sequence to ensure scanout
1244  * can happen from such a surface without seeing corruption.
1245  *
1246  * Display surfaces can be tagged as transient by mapping it using one of the
1247  * various L3:XD PAT index modes on Xe2.
1248  *
1249  * Note: On non-discrete xe2 platforms, like LNL, the entire L3 cache is flushed
1250  * at the end of each submission via PIPE_CONTROL for compute/render, since SA
1251  * Media is not coherent with L3 and we want to support render-vs-media
1252  * usescases. For other engines like copy/blt the HW internally forces uncached
1253  * behaviour, hence why we can skip the TDF on such platforms.
1254  */
1255 void xe_device_td_flush(struct xe_device *xe)
1256 {
1257 	struct xe_gt *root_gt;
1258 
1259 	/*
1260 	 * From Xe3p onward the HW takes care of flush of TD entries also along
1261 	 * with flushing XA entries, which will be at the usual sync points,
1262 	 * like at the end of submission, so no manual flush is needed here.
1263 	 */
1264 	if (GRAPHICS_VER(xe) >= 35)
1265 		return;
1266 
1267 	if (!IS_DGFX(xe) || GRAPHICS_VER(xe) < 20)
1268 		return;
1269 
1270 	root_gt = xe_root_mmio_gt(xe);
1271 	if (!root_gt)
1272 		return;
1273 
1274 	if (XE_GT_WA(root_gt, 16023588340)) {
1275 		/* A transient flush is not sufficient: flush the L2 */
1276 		xe_device_l2_flush(xe, false);
1277 	} else {
1278 		xe_guc_pc_apply_flush_freq_limit(&root_gt->uc.guc.pc);
1279 		tdf_request_sync(xe);
1280 		xe_guc_pc_remove_flush_freq_limit(&root_gt->uc.guc.pc);
1281 	}
1282 }
1283 
1284 u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size)
1285 {
1286 	return xe_device_has_flat_ccs(xe) ?
1287 		DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0;
1288 }
1289 
1290 /**
1291  * xe_device_assert_mem_access - Inspect the current runtime_pm state.
1292  * @xe: xe device instance
1293  *
1294  * To be used before any kind of memory access. It will splat a debug warning
1295  * if the device is currently sleeping. But it doesn't guarantee in any way
1296  * that the device is going to remain awake. Xe PM runtime get and put
1297  * functions might be added to the outer bound of the memory access, while
1298  * this check is intended for inner usage to splat some warning if the worst
1299  * case has just happened.
1300  */
1301 void xe_device_assert_mem_access(struct xe_device *xe)
1302 {
1303 	xe_assert(xe, !xe_pm_runtime_suspended(xe));
1304 }
1305 
1306 void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p)
1307 {
1308 	struct xe_gt *gt;
1309 	u8 id;
1310 
1311 	drm_printf(p, "PCI ID: 0x%04x\n", xe->info.devid);
1312 	drm_printf(p, "PCI revision: 0x%02x\n", xe->info.revid);
1313 
1314 	for_each_gt(gt, xe, id) {
1315 		drm_printf(p, "GT id: %u\n", id);
1316 		drm_printf(p, "\tTile: %u\n", gt->tile->id);
1317 		drm_printf(p, "\tType: %s\n",
1318 			   gt->info.type == XE_GT_TYPE_MAIN ? "main" : "media");
1319 		drm_printf(p, "\tIP ver: %u.%u.%u\n",
1320 			   REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid),
1321 			   REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid),
1322 			   REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid));
1323 		drm_printf(p, "\tCS reference clock: %u\n", gt->info.reference_clock);
1324 	}
1325 }
1326 
1327 u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address)
1328 {
1329 	return sign_extend64(address, xe->info.va_bits - 1);
1330 }
1331 
1332 u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address)
1333 {
1334 	return address & GENMASK_ULL(xe->info.va_bits - 1, 0);
1335 }
1336 
1337 /**
1338  * DOC: Xe Device Wedging
1339  *
1340  * Xe driver uses drm device wedged uevent as documented in Documentation/gpu/drm-uapi.rst.
1341  * When device is in wedged state, every IOCTL will be blocked and GT cannot
1342  * be used. The conditions under which the driver declares the device wedged
1343  * depend on the wedged mode configuration (see &enum xe_wedged_mode). The
1344  * default recovery method for a wedged state is rebind/bus-reset.
1345  *
1346  * Another recovery method is vendor-specific. Below are the cases that send
1347  * ``WEDGED=vendor-specific`` recovery method in drm device wedged uevent.
1348  *
1349  * Case: Firmware Flash
1350  * --------------------
1351  *
1352  * Identification Hint
1353  * +++++++++++++++++++
1354  *
1355  * ``WEDGED=vendor-specific`` drm device wedged uevent with
1356  * :ref:`Runtime Survivability mode <xe-survivability-mode>` is used to notify
1357  * admin/userspace consumer about the need for a firmware flash.
1358  *
1359  * Recovery Procedure
1360  * ++++++++++++++++++
1361  *
1362  * Once ``WEDGED=vendor-specific`` drm device wedged uevent is received, follow
1363  * the below steps
1364  *
1365  * - Check Runtime Survivability mode sysfs.
1366  *   If enabled, firmware flash is required to recover the device.
1367  *
1368  *   /sys/bus/pci/devices/<device>/survivability_mode
1369  *
1370  * - Admin/userspace consumer can use firmware flashing tools like fwupd to flash
1371  *   firmware and restore device to normal operation.
1372  */
1373 
1374 /**
1375  * xe_device_set_wedged_method - Set wedged recovery method
1376  * @xe: xe device instance
1377  * @method: recovery method to set
1378  *
1379  * Set wedged recovery method to be sent in drm wedged uevent.
1380  */
1381 void xe_device_set_wedged_method(struct xe_device *xe, unsigned long method)
1382 {
1383 	xe->wedged.method = method;
1384 }
1385 
1386 #define WEDGED_URL	"https://docs.kernel.org/gpu/drm-uapi.html#device-wedging"
1387 #define XE_BUG_URL	"https://gitlab.freedesktop.org/drm/xe/kernel/issues/new"
1388 
1389 /**
1390  * xe_device_declare_wedged - Declare device wedged
1391  * @xe: xe device instance
1392  *
1393  * This is a final state that can only be cleared with the recovery method
1394  * specified in the drm wedged uevent. The method can be set using
1395  * xe_device_set_wedged_method before declaring the device as wedged. If no method
1396  * is set, reprobe (unbind/re-bind) will be sent by default.
1397  *
1398  * In this state every IOCTL will be blocked so the GT cannot be used.
1399  * In general it will be called upon any critical error such as gt reset
1400  * failure or guc loading failure. Userspace will be notified of this state
1401  * through device wedged uevent.
1402  * If xe.wedged module parameter is set to 2, this function will be called
1403  * on every single execution timeout (a.k.a. GPU hang) right after devcoredump
1404  * snapshot capture. In this mode, GT reset won't be attempted so the state of
1405  * the issue is preserved for further debugging.
1406  */
1407 void xe_device_declare_wedged(struct xe_device *xe)
1408 {
1409 	struct xe_gt *gt;
1410 	u8 id;
1411 
1412 	if (xe->wedged.mode == XE_WEDGED_MODE_NEVER) {
1413 		drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n");
1414 		return;
1415 	}
1416 
1417 	if (!atomic_xchg(&xe->wedged.flag, 1)) {
1418 		xe->needs_flr_on_fini = true;
1419 		xe_pm_runtime_get_noresume(xe);
1420 
1421 		xe_log_err_fatal(xe, WEDGED, -EIO, "Device declared wedged!\n");
1422 		xe_err_once(xe, "IOCTLs and executions are now blocked!\n"
1423 			    "For recovery procedure, refer to %s\n"
1424 			    "Please file a _new_ bug report at %s\n",
1425 			    WEDGED_URL, XE_BUG_URL);
1426 	}
1427 
1428 	for_each_gt(gt, xe, id)
1429 		xe_gt_declare_wedged(gt);
1430 
1431 	if (xe_device_wedged(xe)) {
1432 		/*
1433 		 * XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET is intended for debugging
1434 		 * hangs, so wedge the device with 'none' recovery method and have
1435 		 * it available to the user for debugging.
1436 		 */
1437 		if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET)
1438 			xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_NONE);
1439 		/* If no wedge recovery method is set, use default */
1440 		else if (!xe->wedged.method)
1441 			xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_REBIND |
1442 						    DRM_WEDGE_RECOVERY_BUS_RESET);
1443 
1444 		/* Notify userspace of wedged device */
1445 		drm_dev_wedged_event(&xe->drm, xe->wedged.method, NULL);
1446 	}
1447 }
1448 
1449 /**
1450  * xe_device_validate_wedged_mode - Check if given mode is supported
1451  * @xe: the &xe_device
1452  * @mode: requested mode to validate
1453  *
1454  * Check whether the provided wedged mode is supported.
1455  *
1456  * Return: 0 if mode is supported, error code otherwise.
1457  */
1458 int xe_device_validate_wedged_mode(struct xe_device *xe, unsigned int mode)
1459 {
1460 	if (mode > XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) {
1461 		drm_dbg(&xe->drm, "wedged_mode: invalid value (%u)\n", mode);
1462 		return -EINVAL;
1463 	} else if (mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET && (IS_SRIOV_VF(xe) ||
1464 		   (IS_SRIOV_PF(xe) && !IS_ENABLED(CONFIG_DRM_XE_DEBUG)))) {
1465 		drm_dbg(&xe->drm, "wedged_mode: (%u) %s mode is not supported for %s\n",
1466 			mode, xe_wedged_mode_to_string(mode),
1467 			xe_sriov_mode_to_string(xe_device_sriov_mode(xe)));
1468 		return -EPERM;
1469 	}
1470 
1471 	return 0;
1472 }
1473 
1474 /**
1475  * xe_wedged_mode_to_string - Convert enum value to string.
1476  * @mode: the &xe_wedged_mode to convert
1477  *
1478  * Returns: wedged mode as a user friendly string.
1479  */
1480 const char *xe_wedged_mode_to_string(enum xe_wedged_mode mode)
1481 {
1482 	switch (mode) {
1483 	case XE_WEDGED_MODE_NEVER:
1484 		return "never";
1485 	case XE_WEDGED_MODE_UPON_CRITICAL_ERROR:
1486 		return "upon-critical-error";
1487 	case XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET:
1488 		return "upon-any-hang-no-reset";
1489 	default:
1490 		return "<invalid>";
1491 	}
1492 }
1493 
1494 /**
1495  * xe_device_asid_to_vm() - Find VM from ASID
1496  * @xe: the &xe_device
1497  * @asid: Address space ID
1498  *
1499  * Find a VM from ASID and take a reference to VM which caller must drop.
1500  * Reclaim safe.
1501  *
1502  * Return: VM on success, ERR_PTR on failure
1503  */
1504 struct xe_vm *xe_device_asid_to_vm(struct xe_device *xe, u32 asid)
1505 {
1506 	struct xe_vm *vm;
1507 
1508 	down_read(&xe->usm.lock);
1509 	vm = xa_load(&xe->usm.asid_to_vm, asid);
1510 	if (vm)
1511 		xe_vm_get(vm);
1512 	else
1513 		vm = ERR_PTR(-EINVAL);
1514 	up_read(&xe->usm.lock);
1515 
1516 	return vm;
1517 }
1518